Merged hyracks_dev_next -r 601:1246
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_sort_join_opts@1247 123451ca-8445-de46-9d55-352943316053
diff --git a/.project b/.project
deleted file mode 100644
index cb5b36a..0000000
--- a/.project
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/.settings/org.maven.ide.eclipse.prefs b/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 1b2848c..0000000
--- a/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Sun Aug 29 19:38:09 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-admin-console/.classpath b/hyracks-admin-console/.classpath
deleted file mode 100644
index d93f560..0000000
--- a/hyracks-admin-console/.classpath
+++ /dev/null
@@ -1,13 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/hyracks-admin-console-0.1.8-SNAPSHOT/WEB-INF/classes" path="src/main/java"/>
- <classpathentry excluding="**" kind="src" output="target/hyracks-admin-console-0.1.8-SNAPSHOT/WEB-INF/classes" path="src/main/resources"/>
- <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
- <classpathentry excluding="**" kind="src" output="target/test-classes" path="src/test/resources"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="con" path="com.google.gwt.eclipse.core.GWT_CONTAINER"/>
- <classpathentry kind="con" path="org.eclipse.jst.j2ee.internal.web.container"/>
- <classpathentry kind="con" path="org.eclipse.jst.j2ee.internal.module.container"/>
- <classpathentry kind="output" path="target/hyracks-admin-console-0.1.8-SNAPSHOT/WEB-INF/classes"/>
-</classpath>
diff --git a/hyracks-admin-console/.project b/hyracks-admin-console/.project
deleted file mode 100644
index df60f9a..0000000
--- a/hyracks-admin-console/.project
+++ /dev/null
@@ -1,53 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-admin-console</name>
- <comment>hyracks-admin-console project</comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.wst.jsdt.core.javascriptValidator</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.eclipse.wst.common.project.facet.core.builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.eclipse.wst.validation.validationbuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>com.google.gdt.eclipse.core.webAppProjectValidator</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>com.google.gwt.eclipse.core.gwtProjectValidator</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jem.workbench.JavaEMFNature</nature>
- <nature>org.eclipse.wst.common.modulecore.ModuleCoreNature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.eclipse.wst.common.project.facet.core.nature</nature>
- <nature>org.eclipse.wst.jsdt.core.jsNature</nature>
- <nature>com.google.gwt.eclipse.core.gwtNature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-admin-console/.settings/.jsdtscope b/hyracks-admin-console/.settings/.jsdtscope
deleted file mode 100644
index ba3c245..0000000
--- a/hyracks-admin-console/.settings/.jsdtscope
+++ /dev/null
@@ -1,15 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" path="src/main/webapp" />
- <classpathentry kind="con"
- path="org.eclipse.wst.jsdt.launching.JRE_CONTAINER" />
- <classpathentry kind="con"
- path="org.eclipse.wst.jsdt.launching.WebProject">
- <attributes>
- <attribute name="hide" value="true" />
- </attributes>
- </classpathentry>
- <classpathentry kind="con"
- path="org.eclipse.wst.jsdt.launching.baseBrowserLibrary" />
- <classpathentry kind="output" path="" />
-</classpath>
diff --git a/hyracks-admin-console/.settings/com.google.appengine.eclipse.core.prefs b/hyracks-admin-console/.settings/com.google.appengine.eclipse.core.prefs
deleted file mode 100644
index a60576c..0000000
--- a/hyracks-admin-console/.settings/com.google.appengine.eclipse.core.prefs
+++ /dev/null
@@ -1,3 +0,0 @@
-#Thu Jun 16 10:18:26 CEST 2011
-eclipse.preferences.version=1
-filesCopiedToWebInfLib=
diff --git a/hyracks-admin-console/.settings/com.google.gdt.eclipse.core.prefs b/hyracks-admin-console/.settings/com.google.gdt.eclipse.core.prefs
deleted file mode 100644
index 8019224..0000000
--- a/hyracks-admin-console/.settings/com.google.gdt.eclipse.core.prefs
+++ /dev/null
@@ -1,5 +0,0 @@
-#Thu Sep 02 10:55:28 CEST 2010
-eclipse.preferences.version=1
-jarsExcludedFromWebInfLib=
-warSrcDir=src/main/webapp
-warSrcDirIsOutput=true
diff --git a/hyracks-admin-console/.settings/com.google.gwt.eclipse.core.prefs b/hyracks-admin-console/.settings/com.google.gwt.eclipse.core.prefs
deleted file mode 100644
index c803c44..0000000
--- a/hyracks-admin-console/.settings/com.google.gwt.eclipse.core.prefs
+++ /dev/null
@@ -1,5 +0,0 @@
-#Thu Jun 16 11:14:17 CEST 2011
-eclipse.preferences.version=1
-entryPointModules=
-filesCopiedToWebInfLib=gwt-servlet.jar
-gwtCompileSettings=PGd3dC1jb21waWxlLXNldHRpbmdzPjxsb2ctbGV2ZWw+SU5GTzwvbG9nLWxldmVsPjxvdXRwdXQtc3R5bGU+T0JGVVNDQVRFRDwvb3V0cHV0LXN0eWxlPjxleHRyYS1hcmdzPjwhW0NEQVRBWy13YXIgc3JjL21haW4vd2ViYXBwXV0+PC9leHRyYS1hcmdzPjx2bS1hcmdzPjwhW0NEQVRBWy1YbXg1MTJtXV0+PC92bS1hcmdzPjxlbnRyeS1wb2ludC1tb2R1bGU+Y29tLmNvbXBhbnkuU29tZU1vZHVsZTwvZW50cnktcG9pbnQtbW9kdWxlPjwvZ3d0LWNvbXBpbGUtc2V0dGluZ3M+
diff --git a/hyracks-admin-console/.settings/org.eclipse.jdt.core.prefs b/hyracks-admin-console/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 96e4c8c..0000000
--- a/hyracks-admin-console/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Wed Aug 24 10:20:07 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-admin-console/.settings/org.eclipse.wst.common.component b/hyracks-admin-console/.settings/org.eclipse.wst.common.component
deleted file mode 100644
index 8b16bad..0000000
--- a/hyracks-admin-console/.settings/org.eclipse.wst.common.component
+++ /dev/null
@@ -1,11 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project-modules id="moduleCoreId" project-version="1.5.0">
- <wb-module deploy-name="${module}">
- <wb-resource deploy-path="/" source-path="/src/main/webapp"/>
- <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
- <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/test/java"/>
- <wb-resource deploy-path="/WEB-INF/classes" source-path="/target/generated-sources/gwt"/>
- <property name="context-root" value="${module}"/>
- <property name="java-output-path" value="/${module}/target/www/WEB-INF/classes"/>
- </wb-module>
-</project-modules>
diff --git a/hyracks-admin-console/.settings/org.eclipse.wst.common.project.facet.core.xml b/hyracks-admin-console/.settings/org.eclipse.wst.common.project.facet.core.xml
deleted file mode 100644
index 18c1b9a..0000000
--- a/hyracks-admin-console/.settings/org.eclipse.wst.common.project.facet.core.xml
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<faceted-project>
- <fixed facet="wst.jsdt.web" />
- <installed facet="java" version="1.5" />
- <installed facet="jst.web" version="2.3" />
- <installed facet="wst.jsdt.web" version="1.0" />
-</faceted-project>
diff --git a/hyracks-admin-console/.settings/org.eclipse.wst.jsdt.ui.superType.container b/hyracks-admin-console/.settings/org.eclipse.wst.jsdt.ui.superType.container
deleted file mode 100644
index 3bd5d0a..0000000
--- a/hyracks-admin-console/.settings/org.eclipse.wst.jsdt.ui.superType.container
+++ /dev/null
@@ -1 +0,0 @@
-org.eclipse.wst.jsdt.launching.baseBrowserLibrary
\ No newline at end of file
diff --git a/hyracks-admin-console/.settings/org.maven.ide.eclipse.prefs b/hyracks-admin-console/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index c74c58e..0000000
--- a/hyracks-admin-console/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Sep 02 10:42:12 CEST 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-admin-console/pom.xml b/hyracks-admin-console/pom.xml
deleted file mode 100644
index ca200e5..0000000
--- a/hyracks-admin-console/pom.xml
+++ /dev/null
@@ -1,115 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project
- xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
- <groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-admin-console</artifactId>
- <packaging>war</packaging>
- <version>0.1.8-SNAPSHOT</version>
-
- <parent>
- <groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
- </parent>
-
- <properties>
- <gwtVersion>2.3.0</gwtVersion>
- <webappDirectory>${project.build.directory}/${project.build.finalName}</webappDirectory>
- <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>com.google.gwt</groupId>
- <artifactId>gwt-user</artifactId>
- <version>${gwtVersion}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.7</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- <version>1.0.0.GA</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>javax.validation</groupId>
- <artifactId>validation-api</artifactId>
- <version>1.0.0.GA</version>
- <classifier>sources</classifier>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>com.googlecode.gchart</groupId>
- <artifactId>gchart</artifactId>
- <version>2.6</version>
- </dependency>
- </dependencies>
-
- <build>
- <!-- Generate compiled stuff in the folder used for developing mode -->
- <outputDirectory>${webappDirectory}/WEB-INF/classes</outputDirectory>
-
- <plugins>
-
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>gwt-maven-plugin</artifactId>
- <version>2.3.0-1</version>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>test</goal>
- <goal>i18n</goal>
- <goal>generateAsync</goal>
- </goals>
- </execution>
- </executions>
- <!-- Plugin configuration. There are many available options, see
- gwt-maven-plugin documentation at codehaus.org -->
- <configuration>
- <runTarget>HyracksAdminConsole.html</runTarget>
- <hostedWebapp>${webappDirectory}</hostedWebapp>
- <i18nMessagesBundle>edu.uci.ics.hyracks.adminconsole.client.Messages</i18nMessagesBundle>
- </configuration>
- </plugin>
-
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-war-plugin</artifactId>
- <version>2.1.1</version>
- <executions>
- <execution>
- <phase>compile</phase>
- <goals>
- <goal>exploded</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <webappDirectory>${webappDirectory}</webappDirectory>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
- <version>2.3.2</version>
- <configuration>
- <source>1.6</source>
- <target>1.6</target>
- </configuration>
- </plugin>
- </plugins>
- </build>
-
-</project>
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/HyracksAdminConsole.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/HyracksAdminConsole.java
deleted file mode 100644
index 09e86b0..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/HyracksAdminConsole.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client;
-
-import com.google.gwt.core.client.EntryPoint;
-import com.google.gwt.core.client.GWT;
-import com.google.gwt.uibinder.client.UiBinder;
-import com.google.gwt.uibinder.client.UiField;
-import com.google.gwt.user.client.Window;
-import com.google.gwt.user.client.ui.DockLayoutPanel;
-import com.google.gwt.user.client.ui.RootLayoutPanel;
-
-import edu.uci.ics.hyracks.adminconsole.client.connection.ServerConnection;
-import edu.uci.ics.hyracks.adminconsole.client.panels.ApplicationsPanel;
-import edu.uci.ics.hyracks.adminconsole.client.panels.DashboardPanel;
-import edu.uci.ics.hyracks.adminconsole.client.panels.JobsPanel;
-import edu.uci.ics.hyracks.adminconsole.client.panels.NodeControllersPanel;
-import edu.uci.ics.hyracks.adminconsole.client.panels.TopPanel;
-
-public class HyracksAdminConsole implements EntryPoint {
- public static HyracksAdminConsole INSTANCE;
-
- private final Messages messages = GWT.create(Messages.class);
-
- private ServerConnection serverConnection;
-
- @UiField
- TopPanel topPanel;
-
- @UiField
- DashboardPanel dashboardPanel;
-
- @UiField
- ApplicationsPanel applicationsPanel;
-
- @UiField
- NodeControllersPanel nodeControllersPanel;
-
- @UiField
- JobsPanel jobsPanel;
-
- interface Binder extends UiBinder<DockLayoutPanel, HyracksAdminConsole> {
- }
-
- private final static Binder binder = GWT.create(Binder.class);
-
- public ServerConnection getServerConnection() {
- return serverConnection;
- }
-
- public void onModuleLoad() {
- INSTANCE = this;
- serverConnection = new ServerConnection();
- String serverURLPrefix = Window.Location.getParameter("cclocation");
- serverConnection.setServerURLPrefix(serverURLPrefix);
-
- DockLayoutPanel lp = binder.createAndBindUi(this);
-
- // Get rid of scrollbars, and clear out the window's built-in margin,
- // because we want to take advantage of the entire client area.
- Window.enableScrolling(false);
- Window.setMargin("0px");
-
- // Add the outer panel to the RootLayoutPanel, so that it will be
- // displayed.
- RootLayoutPanel root = RootLayoutPanel.get();
- root.add(lp);
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/HyracksAdminConsole.ui.xml b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/HyracksAdminConsole.ui.xml
deleted file mode 100644
index 929e1de..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/HyracksAdminConsole.ui.xml
+++ /dev/null
@@ -1,44 +0,0 @@
-<!DOCTYPE ui:UiBinder
- SYSTEM "http://dl.google.com/gwt/DTD/xhtml.ent"
->
-<ui:UiBinder
- xmlns:ui='urn:ui:com.google.gwt.uibinder'
- xmlns:g='urn:import:com.google.gwt.user.client.ui'
- xmlns:hac.panels='urn:import:edu.uci.ics.hyracks.adminconsole.client.panels'>
-
- <g:DockLayoutPanel unit='EM'>
- <g:north size='5'>
- <hac.panels:TopPanel ui:field='topPanel' />
- </g:north>
-
- <g:center>
- <g:TabLayoutPanel barUnit="EM" barHeight="2">
- <g:tab>
- <g:header>
- Dashboard
- </g:header>
- <hac.panels:DashboardPanel ui:field='dashboardPanel' />
- </g:tab>
- <g:tab>
- <g:header>
- Applications
- </g:header>
- <hac.panels:ApplicationsPanel ui:field='applicationsPanel' />
- </g:tab>
- <g:tab>
- <g:header>
- Node Controllers
- </g:header>
- <hac.panels:NodeControllersPanel ui:field='nodeControllersPanel' />
- </g:tab>
- <g:tab>
- <g:header>
- Jobs
- </g:header>
- <hac.panels:JobsPanel ui:field='jobsPanel' />
- </g:tab>
- </g:TabLayoutPanel>
- </g:center>
- </g:DockLayoutPanel>
-
-</ui:UiBinder>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/beans/JobSummary.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/beans/JobSummary.java
deleted file mode 100644
index 1cf8acd..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/beans/JobSummary.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.beans;
-
-import com.google.gwt.core.client.JavaScriptObject;
-import com.google.gwt.view.client.ProvidesKey;
-
-public final class JobSummary extends JavaScriptObject {
- public static final ProvidesKey<JobSummary> KEY_PROVIDER = new ProvidesKey<JobSummary>() {
- @Override
- public Object getKey(JobSummary item) {
- return item.getJobId();
- }
- };
-
- protected JobSummary() {
- }
-
- public native String getJobId()
- /*-{
- return this["job-id"];
- }-*/;
-
- public native String getApplicationName()
- /*-{
- return this["application-name"];
- }-*/;
-
- public native String getStatus()
- /*-{
- return this.status;
- }-*/;
-
- public native Long getCreateTime()
- /*-{
- return this["create-time"];
- }-*/;
-
- public native Long getStartTime()
- /*-{
- return this["start-time"];
- }-*/;
-
- public native Long getEndTime()
- /*-{
- return this["end-time"];
- }-*/;
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/beans/NodeDetails.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/beans/NodeDetails.java
deleted file mode 100644
index 5e002f0..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/beans/NodeDetails.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.beans;
-
-import com.google.gwt.core.client.JavaScriptObject;
-import com.google.gwt.core.client.JsArrayInteger;
-import com.google.gwt.core.client.JsArrayNumber;
-
-public final class NodeDetails extends JavaScriptObject {
- protected NodeDetails() {
-
- }
-
- public native String getNodeId()
- /*-{
- return this["node-id"];
- }-*/;
-
- public native String getOSName()
- /*-{
- return this["os-name"];
- }-*/;
-
- public native String getArch()
- /*-{
- return this["arch"];
- }-*/;
-
- public native String getOSVersion()
- /*-{
- return this["os-version"];
- }-*/;
-
- public native int getNProcessors()
- /*-{
- return this["num-processors"];
- }-*/;
-
- public native int getRRDPtr()
- /*-{
- return this["rrd-ptr"];
- }-*/;
-
- public native JsArrayNumber getHeartbeatTimes()
- /*-{
- return this["heartbeat-times"];
- }-*/;
-
- public native JsArrayNumber getHeapInitSizes()
- /*-{
- return this["heap-init-sizes"];
- }-*/;
-
- public native JsArrayNumber getHeapUsedSizes()
- /*-{
- return this["heap-used-sizes"];
- }-*/;
-
- public native JsArrayNumber getHeapCommittedSizes()
- /*-{
- return this["heap-committed-sizes"];
- }-*/;
-
- public native JsArrayNumber getHeapMaxSizes()
- /*-{
- return this["heap-max-sizes"];
- }-*/;
-
- public native JsArrayNumber getNonHeapInitSizes()
- /*-{
- return this["nonheap-init-sizes"];
- }-*/;
-
- public native JsArrayNumber getNonHeapUsedSizes()
- /*-{
- return this["nonheap-used-sizes"];
- }-*/;
-
- public native JsArrayNumber getNonHeapCommittedSizes()
- /*-{
- return this["nonheap-committed-sizes"];
- }-*/;
-
- public native JsArrayNumber getNonHeapMaxSizes()
- /*-{
- return this["nonheap-max-sizes"];
- }-*/;
-
- public native JsArrayInteger getThreadCounts()
- /*-{
- return this["thread-counts"];
- }-*/;
-
- public native JsArrayInteger getPeakThreadCounts()
- /*-{
- return this["peak-thread-counts"];
- }-*/;
-
- public native JsArrayNumber getSystemLoadAverages()
- /*-{
- return this["system-load-averages"];
- }-*/;
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/beans/NodeSummary.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/beans/NodeSummary.java
deleted file mode 100644
index 45436ef..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/beans/NodeSummary.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.beans;
-
-import com.google.gwt.core.client.JavaScriptObject;
-import com.google.gwt.core.client.JsArray;
-import com.google.gwt.view.client.ProvidesKey;
-
-public final class NodeSummary extends JavaScriptObject {
- public static final ProvidesKey<NodeSummary> KEY_PROVIDER = new ProvidesKey<NodeSummary>() {
- @Override
- public Object getKey(NodeSummary item) {
- return item.getNodeId();
- }
- };
-
- protected NodeSummary() {
- }
-
- public native String getNodeId()
- /*-{
- return this["node-id"];
- }-*/;
-
- public native Double getSystemLoadAverage()
- /*-{
- return this["system-load-average"];
- }-*/;
-
- public native Long getHeapUsed()
- /*-{
- return this["heap-used"];
- }-*/;
-
- public static native JsArray<NodeSummary> parseNodeSummariesResult(String json)
- /*-{
- return eval(json)[0].result;
- }-*/;
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/connection/ServerConnection.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/connection/ServerConnection.java
deleted file mode 100644
index 1e5ab09..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/connection/ServerConnection.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.connection;
-
-public class ServerConnection {
- private String serverURLPrefix;
-
- public ServerConnection() {
- }
-
- public String getServerURLPrefix() {
- return serverURLPrefix;
- }
-
- public void setServerURLPrefix(String serverURLPrefix) {
- this.serverURLPrefix = serverURLPrefix;
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/details/node/charts/MemoryUsageChart.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/details/node/charts/MemoryUsageChart.java
deleted file mode 100644
index 523380c..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/details/node/charts/MemoryUsageChart.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.details.node.charts;
-
-import com.google.gwt.core.client.JsArrayNumber;
-import com.googlecode.gchart.client.GChart;
-
-public class MemoryUsageChart extends GChart {
- private String prefix;
-
- public MemoryUsageChart() {
- setChartSize(720, 200);
- setPadding("30px");
- }
-
- public void setPrefix(String prefix) {
- this.prefix = prefix;
- }
-
- public void reset(int rrdPtr, JsArrayNumber times, JsArrayNumber initSizes, JsArrayNumber usedSizes,
- JsArrayNumber committedSizes, JsArrayNumber maxSizes) {
- clearCurves();
- addCurve();
- getCurve().setLegendLabel(prefix + " Initial Size");
- getCurve().getSymbol().setWidth(0);
- getCurve().getSymbol().setHeight(0);
- getCurve().getSymbol().setSymbolType(SymbolType.LINE);
- addCurve();
- getCurve().setLegendLabel(prefix + " Used Size");
- getCurve().getSymbol().setWidth(0);
- getCurve().getSymbol().setHeight(0);
- getCurve().getSymbol().setSymbolType(SymbolType.LINE);
- addCurve();
- getCurve().setLegendLabel(prefix + " Committed Size");
- getCurve().getSymbol().setWidth(0);
- getCurve().getSymbol().setHeight(0);
- getCurve().getSymbol().setSymbolType(SymbolType.LINE);
- addCurve();
- getCurve().setLegendLabel(prefix + " Maximum Size");
- getCurve().getSymbol().setWidth(0);
- getCurve().getSymbol().setHeight(0);
- getCurve().getSymbol().setSymbolType(SymbolType.LINE);
- int ptr = rrdPtr;
- for (int i = 0; i < times.length(); ++i) {
- getCurve(0).addPoint(i, initSizes.get(ptr));
- getCurve(1).addPoint(i, usedSizes.get(ptr));
- getCurve(2).addPoint(i, committedSizes.get(ptr));
- getCurve(3).addPoint(i, maxSizes.get(ptr));
- ptr = (ptr + 1) % times.length();
- }
- update();
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/details/node/charts/SystemLoadAverageChart.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/details/node/charts/SystemLoadAverageChart.java
deleted file mode 100644
index 83b3632..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/details/node/charts/SystemLoadAverageChart.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.details.node.charts;
-
-import com.google.gwt.core.client.JsArrayNumber;
-import com.googlecode.gchart.client.GChart;
-
-public class SystemLoadAverageChart extends GChart {
- public SystemLoadAverageChart() {
- setChartSize(720, 200);
- setPadding("30px");
- }
-
- public void reset(int rrdPtr, JsArrayNumber times, JsArrayNumber systemLoadAverages) {
- clearCurves();
- addCurve();
- getCurve().setLegendLabel("System Load Average");
- getCurve().getSymbol().setWidth(0);
- getCurve().getSymbol().setHeight(0);
- getCurve().getSymbol().setSymbolType(SymbolType.LINE);
- int ptr = rrdPtr;
- for (int i = 0; i < times.length(); ++i) {
- getCurve().addPoint(i, systemLoadAverages.get(ptr));
- ptr = (ptr + 1) % times.length();
- }
- update();
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/details/node/charts/ThreadCountChart.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/details/node/charts/ThreadCountChart.java
deleted file mode 100644
index 5982fcf..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/details/node/charts/ThreadCountChart.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.details.node.charts;
-
-import com.google.gwt.core.client.JsArrayInteger;
-import com.google.gwt.core.client.JsArrayNumber;
-import com.googlecode.gchart.client.GChart;
-
-public class ThreadCountChart extends GChart {
- public ThreadCountChart() {
- setChartSize(720, 200);
- setPadding("30px");
- }
-
- public void reset(int rrdPtr, JsArrayNumber times, JsArrayInteger threadCounts, JsArrayInteger peakThreadCounts) {
- clearCurves();
- addCurve();
- getCurve().setLegendLabel("Thread Count");
- getCurve().getSymbol().setWidth(0);
- getCurve().getSymbol().setHeight(0);
- getCurve().getSymbol().setSymbolType(SymbolType.LINE);
- addCurve();
- getCurve().setLegendLabel("Peak Thread Count");
- getCurve().getSymbol().setWidth(0);
- getCurve().getSymbol().setHeight(0);
- getCurve().getSymbol().setSymbolType(SymbolType.LINE);
- int ptr = rrdPtr;
- for (int i = 0; i < times.length(); ++i) {
- getCurve(0).addPoint(i, threadCounts.get(ptr));
- getCurve(1).addPoint(i, peakThreadCounts.get(ptr));
- ptr = (ptr + 1) % times.length();
- }
- update();
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/ApplicationsPanel.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/ApplicationsPanel.java
deleted file mode 100644
index 6dc307f..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/ApplicationsPanel.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.panels;
-
-import com.google.gwt.core.client.GWT;
-import com.google.gwt.uibinder.client.UiBinder;
-import com.google.gwt.user.client.ui.Composite;
-import com.google.gwt.user.client.ui.Widget;
-
-public class ApplicationsPanel extends Composite {
- interface Binder extends UiBinder<Widget, ApplicationsPanel> {
- }
-
- private final static Binder binder = GWT.create(Binder.class);
-
- public ApplicationsPanel() {
- initWidget(binder.createAndBindUi(this));
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/ApplicationsPanel.ui.xml b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/ApplicationsPanel.ui.xml
deleted file mode 100644
index 74669b0..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/ApplicationsPanel.ui.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-<!DOCTYPE ui:UiBinder
- SYSTEM "http://dl.google.com/gwt/DTD/xhtml.ent"
->
-<ui:UiBinder
- xmlns:ui='urn:ui:com.google.gwt.uibinder'
- xmlns:g='urn:import:com.google.gwt.user.client.ui'
- xmlns:hac.panels='urn:import:edu.uci.ics.hyracks.adminconsole.client.panels'>
- <g:HTML></g:HTML>
-</ui:UiBinder>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/DashboardPanel.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/DashboardPanel.java
deleted file mode 100644
index eb3c9d5..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/DashboardPanel.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.panels;
-
-import com.google.gwt.core.client.GWT;
-import com.google.gwt.uibinder.client.UiBinder;
-import com.google.gwt.user.client.ui.Composite;
-import com.google.gwt.user.client.ui.Widget;
-
-public class DashboardPanel extends Composite {
- interface Binder extends UiBinder<Widget, DashboardPanel> {
- }
-
- private final static Binder binder = GWT.create(Binder.class);
-
- public DashboardPanel() {
- initWidget(binder.createAndBindUi(this));
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/DashboardPanel.ui.xml b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/DashboardPanel.ui.xml
deleted file mode 100644
index 74669b0..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/DashboardPanel.ui.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-<!DOCTYPE ui:UiBinder
- SYSTEM "http://dl.google.com/gwt/DTD/xhtml.ent"
->
-<ui:UiBinder
- xmlns:ui='urn:ui:com.google.gwt.uibinder'
- xmlns:g='urn:import:com.google.gwt.user.client.ui'
- xmlns:hac.panels='urn:import:edu.uci.ics.hyracks.adminconsole.client.panels'>
- <g:HTML></g:HTML>
-</ui:UiBinder>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/JobsPanel.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/JobsPanel.java
deleted file mode 100644
index 8e8557b..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/JobsPanel.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.panels;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.gwt.core.client.GWT;
-import com.google.gwt.core.client.JsArray;
-import com.google.gwt.http.client.RequestException;
-import com.google.gwt.uibinder.client.UiBinder;
-import com.google.gwt.uibinder.client.UiField;
-import com.google.gwt.user.client.Timer;
-import com.google.gwt.user.client.ui.Composite;
-import com.google.gwt.user.client.ui.SplitLayoutPanel;
-import com.google.gwt.user.client.ui.Widget;
-import com.google.gwt.view.client.AsyncDataProvider;
-
-import edu.uci.ics.hyracks.adminconsole.client.beans.JobSummary;
-import edu.uci.ics.hyracks.adminconsole.client.rest.AbstractRestFunction;
-import edu.uci.ics.hyracks.adminconsole.client.rest.GetJobSummariesFunction;
-import edu.uci.ics.hyracks.adminconsole.client.widgets.JobsTableWidget;
-
-public class JobsPanel extends Composite implements JobsTableWidget.IRefreshRequestHandler {
- interface Binder extends UiBinder<Widget, JobsPanel> {
- }
-
- private final static Binder binder = GWT.create(Binder.class);
-
- @UiField
- SplitLayoutPanel split;
-
- @UiField
- JobsTableWidget jobs;
-
- @UiField
- Widget details;
-
- private int callCounter;
-
- public JobsPanel() {
- initWidget(binder.createAndBindUi(this));
-
- jobs.setRefreshRequestHandler(this);
-
- Timer timer = new Timer() {
- @Override
- public void run() {
- refresh();
- }
- };
- refresh();
- timer.scheduleRepeating(5000);
- }
-
- @Override
- public void refresh() {
- try {
- final int counter = ++callCounter;
- GetJobSummariesFunction.INSTANCE.call(new AbstractRestFunction.ResultCallback<JsArray<JobSummary>>() {
- @Override
- public void onSuccess(JsArray<JobSummary> result) {
- if (counter == callCounter) {
- AsyncDataProvider<JobSummary> dataProvider = jobs.getDataProvider();
- List<JobSummary> data = new ArrayList<JobSummary>();
- for (int i = 0; i < result.length(); ++i) {
- data.add(result.get(i));
- }
- dataProvider.updateRowData(0, data);
- dataProvider.updateRowCount(result.length(), true);
- }
- }
-
- @Override
- public void onError(Throwable exception) {
-
- }
- });
- } catch (RequestException e) {
- e.printStackTrace();
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/JobsPanel.ui.xml b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/JobsPanel.ui.xml
deleted file mode 100644
index c12640f..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/JobsPanel.ui.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-<!DOCTYPE ui:UiBinder
- SYSTEM "http://dl.google.com/gwt/DTD/xhtml.ent"
->
-<ui:UiBinder
- xmlns:ui='urn:ui:com.google.gwt.uibinder'
- xmlns:g='urn:import:com.google.gwt.user.client.ui'
- xmlns:widgets='urn:import:edu.uci.ics.hyracks.adminconsole.client.widgets'>
- <g:SplitLayoutPanel ui:field="split">
- <g:north size="300">
- <g:ScrollPanel>
- <widgets:JobsTableWidget ui:field="jobs" width="100%" height="300px"></widgets:JobsTableWidget>
- </g:ScrollPanel>
- </g:north>
- <g:center>
- <g:HTML ui:field="details"></g:HTML>
- </g:center>
- </g:SplitLayoutPanel>
-</ui:UiBinder>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllerDetailsPanel.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllerDetailsPanel.java
deleted file mode 100644
index d64a3ad..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllerDetailsPanel.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.panels;
-
-import com.google.gwt.core.client.GWT;
-import com.google.gwt.http.client.RequestException;
-import com.google.gwt.uibinder.client.UiBinder;
-import com.google.gwt.uibinder.client.UiField;
-import com.google.gwt.user.client.Timer;
-import com.google.gwt.user.client.ui.Composite;
-import com.google.gwt.user.client.ui.Widget;
-
-import edu.uci.ics.hyracks.adminconsole.client.beans.NodeDetails;
-import edu.uci.ics.hyracks.adminconsole.client.details.node.charts.MemoryUsageChart;
-import edu.uci.ics.hyracks.adminconsole.client.details.node.charts.SystemLoadAverageChart;
-import edu.uci.ics.hyracks.adminconsole.client.details.node.charts.ThreadCountChart;
-import edu.uci.ics.hyracks.adminconsole.client.rest.AbstractRestFunction;
-import edu.uci.ics.hyracks.adminconsole.client.rest.GetNodeDetailsFunction;
-
-public class NodeControllerDetailsPanel extends Composite {
- interface Binder extends UiBinder<Widget, NodeControllerDetailsPanel> {
- }
-
- private final static Binder binder = GWT.create(Binder.class);
-
- private final String nodeId;
-
- @UiField
- MemoryUsageChart heapUsage;
-
- @UiField
- MemoryUsageChart nonheapUsage;
-
- @UiField
- SystemLoadAverageChart loadAverage;
-
- @UiField
- ThreadCountChart threadCount;
-
- private int callCounter;
-
- private Timer timer;
-
- public NodeControllerDetailsPanel(String nodeId) {
- initWidget(binder.createAndBindUi(this));
-
- this.nodeId = nodeId;
- heapUsage.setPrefix("Heap ");
- nonheapUsage.setPrefix("Non-Heap ");
-
- timer = new Timer() {
- @Override
- public void run() {
- refresh();
- }
- };
- refresh();
- timer.scheduleRepeating(10000);
- }
-
- public void destroy() {
- timer.cancel();
- }
-
- private void refresh() {
- try {
- final int counter = ++callCounter;
- new GetNodeDetailsFunction(nodeId).call(new AbstractRestFunction.ResultCallback<NodeDetails>() {
- @Override
- public void onSuccess(NodeDetails result) {
- if (counter == callCounter) {
- loadAverage.reset(result.getRRDPtr(), result.getHeartbeatTimes(),
- result.getSystemLoadAverages());
- heapUsage.reset(result.getRRDPtr(), result.getHeartbeatTimes(), result.getHeapInitSizes(),
- result.getHeapUsedSizes(), result.getHeapCommittedSizes(), result.getHeapMaxSizes());
- nonheapUsage.reset(result.getRRDPtr(), result.getHeartbeatTimes(),
- result.getNonHeapInitSizes(), result.getNonHeapUsedSizes(),
- result.getNonHeapCommittedSizes(), result.getNonHeapMaxSizes());
- threadCount.reset(result.getRRDPtr(), result.getHeartbeatTimes(), result.getThreadCounts(),
- result.getPeakThreadCounts());
- }
- }
-
- @Override
- public void onError(Throwable exception) {
-
- }
- });
- } catch (RequestException e) {
- e.printStackTrace();
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllerDetailsPanel.ui.xml b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllerDetailsPanel.ui.xml
deleted file mode 100644
index 4506f2b..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllerDetailsPanel.ui.xml
+++ /dev/null
@@ -1,37 +0,0 @@
-<!DOCTYPE ui:UiBinder
- SYSTEM "http://dl.google.com/gwt/DTD/xhtml.ent"
->
-<ui:UiBinder
- xmlns:ui='urn:ui:com.google.gwt.uibinder'
- xmlns:g='urn:import:com.google.gwt.user.client.ui'
- xmlns:charts='urn:import:edu.uci.ics.hyracks.adminconsole.client.details.node.charts'>
- <g:StackLayoutPanel unit="EM">
- <g:stack>
- <g:header size="2">
- System Load
- </g:header>
- <g:ScrollPanel>
- <charts:SystemLoadAverageChart ui:field="loadAverage"/>
- </g:ScrollPanel>
- </g:stack>
- <g:stack>
- <g:header size="2">
- Memory Usage
- </g:header>
- <g:ScrollPanel>
- <g:FlowPanel>
- <charts:MemoryUsageChart ui:field="heapUsage"/>
- <charts:MemoryUsageChart ui:field="nonheapUsage"/>
- </g:FlowPanel>
- </g:ScrollPanel>
- </g:stack>
- <g:stack>
- <g:header size="2">
- Thread Count
- </g:header>
- <g:ScrollPanel>
- <charts:ThreadCountChart ui:field="threadCount"/>
- </g:ScrollPanel>
- </g:stack>
- </g:StackLayoutPanel>
-</ui:UiBinder>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllersPanel.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllersPanel.java
deleted file mode 100644
index 708c05d..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllersPanel.java
+++ /dev/null
@@ -1,106 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.panels;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import com.google.gwt.core.client.GWT;
-import com.google.gwt.core.client.JsArray;
-import com.google.gwt.http.client.RequestException;
-import com.google.gwt.uibinder.client.UiBinder;
-import com.google.gwt.uibinder.client.UiField;
-import com.google.gwt.user.client.Timer;
-import com.google.gwt.user.client.ui.Composite;
-import com.google.gwt.user.client.ui.SplitLayoutPanel;
-import com.google.gwt.user.client.ui.Widget;
-import com.google.gwt.view.client.AsyncDataProvider;
-
-import edu.uci.ics.hyracks.adminconsole.client.beans.NodeSummary;
-import edu.uci.ics.hyracks.adminconsole.client.rest.AbstractRestFunction;
-import edu.uci.ics.hyracks.adminconsole.client.rest.GetNodeSummariesFunction;
-import edu.uci.ics.hyracks.adminconsole.client.widgets.NodesTableWidget;
-
-public class NodeControllersPanel extends Composite implements NodesTableWidget.IRefreshRequestHandler {
- interface Binder extends UiBinder<Widget, NodeControllersPanel> {
- }
-
- private final static Binder binder = GWT.create(Binder.class);
-
- @UiField
- SplitLayoutPanel split;
-
- @UiField
- NodesTableWidget nodes;
-
- @UiField
- Widget details;
-
- private int callCounter;
-
- public NodeControllersPanel() {
- initWidget(binder.createAndBindUi(this));
-
- nodes.setRefreshRequestHandler(this);
- nodes.setClickListener(new NodesTableWidget.IClickListener() {
- @Override
- public void click(String nodeId) {
- if (details instanceof NodeControllerDetailsPanel) {
- ((NodeControllerDetailsPanel) details).destroy();
- }
- split.remove(details);
- details = new NodeControllerDetailsPanel(nodeId);
- split.add(details);
- }
- });
-
- Timer timer = new Timer() {
- @Override
- public void run() {
- refresh();
- }
- };
- refresh();
- timer.scheduleRepeating(10000);
- }
-
- @Override
- public void refresh() {
- try {
- final int counter = ++callCounter;
- GetNodeSummariesFunction.INSTANCE.call(new AbstractRestFunction.ResultCallback<JsArray<NodeSummary>>() {
- @Override
- public void onSuccess(JsArray<NodeSummary> result) {
- if (counter == callCounter) {
- AsyncDataProvider<NodeSummary> dataProvider = nodes.getDataProvider();
- List<NodeSummary> data = new ArrayList<NodeSummary>();
- for (int i = 0; i < result.length(); ++i) {
- data.add(result.get(i));
- }
- dataProvider.updateRowData(0, data);
- dataProvider.updateRowCount(result.length(), true);
- }
- }
-
- @Override
- public void onError(Throwable exception) {
-
- }
- });
- } catch (RequestException e) {
- e.printStackTrace();
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllersPanel.ui.xml b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllersPanel.ui.xml
deleted file mode 100644
index 80b4148..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/NodeControllersPanel.ui.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-<!DOCTYPE ui:UiBinder
- SYSTEM "http://dl.google.com/gwt/DTD/xhtml.ent"
->
-<ui:UiBinder
- xmlns:ui='urn:ui:com.google.gwt.uibinder'
- xmlns:g='urn:import:com.google.gwt.user.client.ui'
- xmlns:widgets='urn:import:edu.uci.ics.hyracks.adminconsole.client.widgets'>
- <g:SplitLayoutPanel ui:field="split">
- <g:west size="300">
- <g:ScrollPanel>
- <widgets:NodesTableWidget ui:field="nodes" width="100%"></widgets:NodesTableWidget>
- </g:ScrollPanel>
- </g:west>
- <g:center>
- <g:HTML ui:field="details"></g:HTML>
- </g:center>
- </g:SplitLayoutPanel>
-</ui:UiBinder>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/TopPanel.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/TopPanel.java
deleted file mode 100644
index 3946fce..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/TopPanel.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.panels;
-
-import com.google.gwt.core.client.GWT;
-import com.google.gwt.uibinder.client.UiBinder;
-import com.google.gwt.user.client.ui.Composite;
-import com.google.gwt.user.client.ui.Widget;
-
-public class TopPanel extends Composite {
- interface Binder extends UiBinder<Widget, TopPanel> {
- }
-
- private final static Binder binder = GWT.create(Binder.class);
-
- public TopPanel() {
- initWidget(binder.createAndBindUi(this));
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/TopPanel.ui.xml b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/TopPanel.ui.xml
deleted file mode 100644
index 71c7b24..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/panels/TopPanel.ui.xml
+++ /dev/null
@@ -1,15 +0,0 @@
-<!DOCTYPE ui:UiBinder
- SYSTEM "http://dl.google.com/gwt/DTD/xhtml.ent"
->
-<ui:UiBinder
- xmlns:ui='urn:ui:com.google.gwt.uibinder'
- xmlns:g='urn:import:com.google.gwt.user.client.ui'
- xmlns:mail='urn:import:com.google.gwt.sample.mail.client'>
-
- <ui:style>
-
- </ui:style>
-
- <g:HTMLPanel>
- </g:HTMLPanel>
-</ui:UiBinder>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/AbstractRestFunction.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/AbstractRestFunction.java
deleted file mode 100644
index 742f19e..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/AbstractRestFunction.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.rest;
-
-import com.google.gwt.http.client.Request;
-import com.google.gwt.http.client.RequestBuilder;
-import com.google.gwt.http.client.RequestCallback;
-import com.google.gwt.http.client.RequestException;
-import com.google.gwt.http.client.Response;
-import com.google.gwt.http.client.URL;
-
-import edu.uci.ics.hyracks.adminconsole.client.HyracksAdminConsole;
-
-public abstract class AbstractRestFunction<T> {
- protected abstract void appendURLPath(StringBuilder buffer);
-
- protected native T packageResults(String result)
- /*-{
- return eval("[" + result + "]")[0].result;
- }-*/;
-
- public interface ResultCallback<T> {
- public void onSuccess(T result);
-
- public void onError(Throwable exception);
- }
-
- public void call(final ResultCallback<T> callback) throws RequestException {
- StringBuilder buffer = new StringBuilder();
- buffer.append(HyracksAdminConsole.INSTANCE.getServerConnection().getServerURLPrefix());
- buffer.append("/rest");
- appendURLPath(buffer);
- RequestBuilder builder = new RequestBuilder(RequestBuilder.GET, URL.encode(buffer.toString()));
-
- builder.sendRequest(null, new RequestCallback() {
- public void onError(Request request, Throwable exception) {
- callback.onError(exception);
- }
-
- public void onResponseReceived(Request request, Response response) {
- if (200 == response.getStatusCode()) {
- callback.onSuccess(packageResults(response.getText()));
- } else {
- callback.onError(null);
- }
- }
- });
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/GetJobSummariesFunction.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/GetJobSummariesFunction.java
deleted file mode 100644
index 4ae4b55..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/GetJobSummariesFunction.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.rest;
-
-import com.google.gwt.core.client.JsArray;
-
-import edu.uci.ics.hyracks.adminconsole.client.beans.JobSummary;
-
-public class GetJobSummariesFunction extends AbstractRestFunction<JsArray<JobSummary>> {
- public static final GetJobSummariesFunction INSTANCE = new GetJobSummariesFunction();
-
- private GetJobSummariesFunction() {
- }
-
- @Override
- protected void appendURLPath(StringBuilder buffer) {
- buffer.append("/jobs");
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/GetNodeDetailsFunction.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/GetNodeDetailsFunction.java
deleted file mode 100644
index e3ee78c..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/GetNodeDetailsFunction.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.rest;
-
-import edu.uci.ics.hyracks.adminconsole.client.beans.NodeDetails;
-
-public class GetNodeDetailsFunction extends AbstractRestFunction<NodeDetails> {
- private String nodeId;
-
- public GetNodeDetailsFunction(String nodeId) {
- this.nodeId = nodeId;
- }
-
- @Override
- protected void appendURLPath(StringBuilder buffer) {
- buffer.append("/nodes/").append(nodeId);
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/GetNodeSummariesFunction.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/GetNodeSummariesFunction.java
deleted file mode 100644
index 8f762f0..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/rest/GetNodeSummariesFunction.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.rest;
-
-import com.google.gwt.core.client.JsArray;
-
-import edu.uci.ics.hyracks.adminconsole.client.beans.NodeSummary;
-
-public class GetNodeSummariesFunction extends AbstractRestFunction<JsArray<NodeSummary>> {
- public static final GetNodeSummariesFunction INSTANCE = new GetNodeSummariesFunction();
-
- private GetNodeSummariesFunction() {
- }
-
- @Override
- protected void appendURLPath(StringBuilder buffer) {
- buffer.append("/nodes");
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/JobsTableWidget.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/JobsTableWidget.java
deleted file mode 100644
index 91ab87f..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/JobsTableWidget.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.widgets;
-
-import com.google.gwt.core.client.GWT;
-import com.google.gwt.uibinder.client.UiBinder;
-import com.google.gwt.uibinder.client.UiField;
-import com.google.gwt.user.cellview.client.CellTable;
-import com.google.gwt.user.cellview.client.TextColumn;
-import com.google.gwt.user.client.ui.Composite;
-import com.google.gwt.user.client.ui.Widget;
-import com.google.gwt.view.client.AsyncDataProvider;
-import com.google.gwt.view.client.HasData;
-
-import edu.uci.ics.hyracks.adminconsole.client.beans.JobSummary;
-
-public class JobsTableWidget extends Composite {
- public interface IRefreshRequestHandler {
- public void refresh();
- }
-
- interface Binder extends UiBinder<Widget, JobsTableWidget> {
- }
-
- private final static Binder binder = GWT.create(Binder.class);
-
- @UiField
- CellTable<JobSummary> table;
-
- private AsyncDataProvider<JobSummary> jobSummaryProvider;
-
- private IRefreshRequestHandler refreshRequestHandler;
-
- public JobsTableWidget() {
- initWidget(binder.createAndBindUi(this));
-
- TextColumn<JobSummary> idCol = new TextColumn<JobSummary>() {
- @Override
- public String getValue(JobSummary object) {
- return object.getJobId();
- }
- };
- idCol.setSortable(true);
-
- TextColumn<JobSummary> appCol = new TextColumn<JobSummary>() {
- @Override
- public String getValue(JobSummary object) {
- return object.getApplicationName();
- }
- };
- appCol.setSortable(true);
-
- TextColumn<JobSummary> statusCol = new TextColumn<JobSummary>() {
- @Override
- public String getValue(JobSummary object) {
- return object.getStatus();
- }
- };
- statusCol.setSortable(true);
-
- /*
- TextColumn<JobSummary> createTimeCol = new TextColumn<JobSummary>() {
- @Override
- public String getValue(JobSummary object) {
- return renderTime(object.getCreateTime());
- }
- };
- createTimeCol.setSortable(true);
-
- TextColumn<JobSummary> startTimeCol = new TextColumn<JobSummary>() {
- @Override
- public String getValue(JobSummary object) {
- return renderTime(object.getStartTime());
- }
- };
- startTimeCol.setSortable(true);
-
- TextColumn<JobSummary> endTimeCol = new TextColumn<JobSummary>() {
- @Override
- public String getValue(JobSummary object) {
- return renderTime(object.getEndTime());
- }
- };
- endTimeCol.setSortable(true);
- */
- table.addColumn(idCol, "Job Id");
- table.addColumn(appCol, "Application Name");
- table.addColumn(statusCol, "Status");
- /*
- table.addColumn(createTimeCol, "Created At");
- table.addColumn(startTimeCol, "Started At");
- table.addColumn(endTimeCol, "Finished At");
- */
-
- jobSummaryProvider = new AsyncDataProvider<JobSummary>(JobSummary.KEY_PROVIDER) {
- @Override
- protected void onRangeChanged(HasData<JobSummary> display) {
- if (refreshRequestHandler != null) {
- refreshRequestHandler.refresh();
- }
- }
- };
- jobSummaryProvider.addDataDisplay(table);
- }
-
- public AsyncDataProvider<JobSummary> getDataProvider() {
- return jobSummaryProvider;
- }
-
- public void setRefreshRequestHandler(IRefreshRequestHandler refreshRequestHandler) {
- this.refreshRequestHandler = refreshRequestHandler;
- }
-
- private static String renderTime(Long time) {
- if (time < 0) {
- return "";
- }
- return time.toString();
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/JobsTableWidget.ui.xml b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/JobsTableWidget.ui.xml
deleted file mode 100644
index 9c48d77..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/JobsTableWidget.ui.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-<!DOCTYPE ui:UiBinder
- SYSTEM "http://dl.google.com/gwt/DTD/xhtml.ent"
->
-<ui:UiBinder
- xmlns:ui='urn:ui:com.google.gwt.uibinder'
- xmlns:g='urn:import:com.google.gwt.user.client.ui'
- xmlns:c='urn:import:com.google.gwt.user.cellview.client'>
- <c:CellTable ui:field="table" width="100%"></c:CellTable>
-</ui:UiBinder>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/NodesTableWidget.java b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/NodesTableWidget.java
deleted file mode 100644
index 983b0fb..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/NodesTableWidget.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.adminconsole.client.widgets;
-
-import com.google.gwt.cell.client.ClickableTextCell;
-import com.google.gwt.cell.client.FieldUpdater;
-import com.google.gwt.core.client.GWT;
-import com.google.gwt.uibinder.client.UiBinder;
-import com.google.gwt.uibinder.client.UiField;
-import com.google.gwt.user.cellview.client.CellTable;
-import com.google.gwt.user.cellview.client.Column;
-import com.google.gwt.user.cellview.client.TextColumn;
-import com.google.gwt.user.client.ui.Composite;
-import com.google.gwt.user.client.ui.Widget;
-import com.google.gwt.view.client.AsyncDataProvider;
-import com.google.gwt.view.client.HasData;
-
-import edu.uci.ics.hyracks.adminconsole.client.beans.NodeSummary;
-
-public class NodesTableWidget extends Composite {
- public interface IRefreshRequestHandler {
- public void refresh();
- }
-
- public interface IClickListener {
- public void click(String nodeId);
- }
-
- interface Binder extends UiBinder<Widget, NodesTableWidget> {
- }
-
- private final static Binder binder = GWT.create(Binder.class);
-
- @UiField
- CellTable<NodeSummary> table;
-
- private AsyncDataProvider<NodeSummary> nodeSummaryProvider;
-
- private IRefreshRequestHandler refreshRequestHandler;
-
- private IClickListener cl;
-
- public NodesTableWidget() {
- initWidget(binder.createAndBindUi(this));
-
- Column<NodeSummary, String> idCol = new Column<NodeSummary, String>(new ClickableTextCell()) {
- @Override
- public String getValue(NodeSummary object) {
- return object.getNodeId();
- }
- };
- idCol.setFieldUpdater(new FieldUpdater<NodeSummary, String>() {
- @Override
- public void update(int index, NodeSummary object, String value) {
- if (cl != null) {
- cl.click(value);
- }
- }
- });
- idCol.setSortable(true);
-
- TextColumn<NodeSummary> heapUsedCol = new TextColumn<NodeSummary>() {
- @Override
- public String getValue(NodeSummary object) {
- return String.valueOf(object.getHeapUsed());
- }
- };
- heapUsedCol.setSortable(true);
-
- TextColumn<NodeSummary> systemLoadAvgCol = new TextColumn<NodeSummary>() {
- @Override
- public String getValue(NodeSummary object) {
- return String.valueOf(object.getSystemLoadAverage());
- }
- };
- systemLoadAvgCol.setSortable(true);
-
- table.addColumn(idCol, "Node Id");
- table.addColumn(heapUsedCol, "Heap Used");
- table.addColumn(systemLoadAvgCol, "System Load Average");
-
- nodeSummaryProvider = new AsyncDataProvider<NodeSummary>(NodeSummary.KEY_PROVIDER) {
- @Override
- protected void onRangeChanged(HasData<NodeSummary> display) {
- if (refreshRequestHandler != null) {
- refreshRequestHandler.refresh();
- }
- }
- };
- nodeSummaryProvider.addDataDisplay(table);
- }
-
- public void setClickListener(IClickListener cl) {
- this.cl = cl;
- }
-
- public AsyncDataProvider<NodeSummary> getDataProvider() {
- return nodeSummaryProvider;
- }
-
- public void setRefreshRequestHandler(IRefreshRequestHandler refreshRequestHandler) {
- this.refreshRequestHandler = refreshRequestHandler;
- }
-}
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/NodesTableWidget.ui.xml b/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/NodesTableWidget.ui.xml
deleted file mode 100644
index 9c48d77..0000000
--- a/hyracks-admin-console/src/main/java/edu/uci/ics/hyracks/adminconsole/client/widgets/NodesTableWidget.ui.xml
+++ /dev/null
@@ -1,9 +0,0 @@
-<!DOCTYPE ui:UiBinder
- SYSTEM "http://dl.google.com/gwt/DTD/xhtml.ent"
->
-<ui:UiBinder
- xmlns:ui='urn:ui:com.google.gwt.uibinder'
- xmlns:g='urn:import:com.google.gwt.user.client.ui'
- xmlns:c='urn:import:com.google.gwt.user.cellview.client'>
- <c:CellTable ui:field="table" width="100%"></c:CellTable>
-</ui:UiBinder>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/resources/edu/uci/ics/hyracks/adminconsole/HyracksAdminConsole.gwt.xml b/hyracks-admin-console/src/main/resources/edu/uci/ics/hyracks/adminconsole/HyracksAdminConsole.gwt.xml
deleted file mode 100644
index 765ad1b..0000000
--- a/hyracks-admin-console/src/main/resources/edu/uci/ics/hyracks/adminconsole/HyracksAdminConsole.gwt.xml
+++ /dev/null
@@ -1,22 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<module rename-to='HyracksAdminConsole'>
- <!-- Inherit the core Web Toolkit stuff. -->
- <inherits name='com.google.gwt.user.User' />
- <inherits name='com.google.gwt.http.HTTP' />
- <inherits name='com.googlecode.gchart.GChart' />
-
- <inherits name='com.google.gwt.user.theme.standard.Standard' />
- <!-- <inherits name='com.google.gwt.user.theme.chrome.Chrome'/> -->
- <!-- <inherits name='com.google.gwt.user.theme.dark.Dark'/> -->
-
- <!-- Other module inherits -->
-
- <!-- Specify the app entry point class. -->
- <entry-point class='edu.uci.ics.hyracks.adminconsole.client.HyracksAdminConsole' />
-
- <!-- Specify the paths for translatable code -->
- <source path='client' />
-
- <!--set-property name="user.agent" value="safari"></set-property-->
- <set-configuration-property name="UiBinder.useSafeHtmlTemplates" value="true"/>
-</module>
diff --git a/hyracks-admin-console/src/main/resources/edu/uci/ics/hyracks/adminconsole/client/Messages.properties b/hyracks-admin-console/src/main/resources/edu/uci/ics/hyracks/adminconsole/client/Messages.properties
deleted file mode 100644
index c222555..0000000
--- a/hyracks-admin-console/src/main/resources/edu/uci/ics/hyracks/adminconsole/client/Messages.properties
+++ /dev/null
@@ -1,2 +0,0 @@
-sendButton = Send
-nameField = Enter your name
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/webapp/HyracksAdminConsole.css b/hyracks-admin-console/src/main/webapp/HyracksAdminConsole.css
deleted file mode 100644
index 7aca7ac..0000000
--- a/hyracks-admin-console/src/main/webapp/HyracksAdminConsole.css
+++ /dev/null
@@ -1,34 +0,0 @@
-/** Add css rules here for your application. */
-
-
-/** Example rules used by the template application (remove for your app) */
-h1 {
- font-size: 2em;
- font-weight: bold;
- color: #777777;
- margin: 40px 0px 70px;
- text-align: center;
-}
-
-.sendButton {
- display: block;
- font-size: 16pt;
-}
-
-/** Most GWT widgets already have a style name defined */
-.gwt-DialogBox {
- width: 400px;
-}
-
-.dialogVPanel {
- margin: 5px;
-}
-
-.serverResponseLabelError {
- color: red;
-}
-
-/** Set ids using widget.getElement().setId("idOfElement") */
-#closeButton {
- margin: 15px 6px 6px;
-}
diff --git a/hyracks-admin-console/src/main/webapp/HyracksAdminConsole.html b/hyracks-admin-console/src/main/webapp/HyracksAdminConsole.html
deleted file mode 100644
index b909e08..0000000
--- a/hyracks-admin-console/src/main/webapp/HyracksAdminConsole.html
+++ /dev/null
@@ -1,18 +0,0 @@
-<!doctype html>
-<html>
- <head>
- <meta http-equiv="content-type" content="text/html; charset=UTF-8">
- <link type="text/css" rel="stylesheet" href="HyracksAdminConsole.css">
- <title>Hyracks Administration Console</title>
- <script type="text/javascript" language="javascript" src="HyracksAdminConsole/HyracksAdminConsole.nocache.js"></script>
- </head>
- <body>
- <iframe src="javascript:''" id="__gwt_historyFrame" tabIndex='-1' style="position:absolute;width:0;height:0;border:0"></iframe>
- <noscript>
- <div style="width: 22em; position: absolute; left: 50%; margin-left: -11em; color: red; background-color: white; border: 1px solid red; padding: 4px; font-family: sans-serif">
- Your web browser must have JavaScript enabled
- in order for this application to display correctly.
- </div>
- </noscript>
- </body>
-</html>
\ No newline at end of file
diff --git a/hyracks-admin-console/src/main/webapp/WEB-INF/web.xml b/hyracks-admin-console/src/main/webapp/WEB-INF/web.xml
deleted file mode 100644
index 8c1ab37..0000000
--- a/hyracks-admin-console/src/main/webapp/WEB-INF/web.xml
+++ /dev/null
@@ -1,24 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE web-app
- PUBLIC "-//Sun Microsystems, Inc.//DTD Web Application 2.3//EN"
- "http://java.sun.com/dtd/web-app_2_3.dtd">
-
-<web-app>
-
- <!-- Servlets -->
- <!--servlet>
- <servlet-name>greetServlet</servlet-name>
- <servlet-class>edu.uci.ics.hyracks.adminconsole.server.GreetingServiceImpl</servlet-class>
- </servlet>
-
- <servlet-mapping>
- <servlet-name>greetServlet</servlet-name>
- <url-pattern>/HyracksAdminConsole/greet</url-pattern>
- </servlet-mapping-->
-
- <!-- Default page to serve -->
- <welcome-file-list>
- <welcome-file>HyracksAdminConsole.html</welcome-file>
- </welcome-file-list>
-
-</web-app>
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/pom.xml b/hyracks-algebricks/hyracks-algebricks-compiler/pom.xml
new file mode 100644
index 0000000..ca8c534
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/pom.xml
@@ -0,0 +1,36 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-algebricks-compiler</artifactId>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-rewriter</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-core</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
new file mode 100644
index 0000000..276398d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/AbstractCompilerFactoryBuilder.java
@@ -0,0 +1,219 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.api;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.ITypeTraitProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+
+public abstract class AbstractCompilerFactoryBuilder {
+
+ protected List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites;
+ protected List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites;
+ protected ITypeTraitProvider typeTraitProvider;
+ protected ISerializerDeserializerProvider serializerDeserializerProvider;
+ protected IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider;
+ protected IBinaryComparatorFactoryProvider comparatorFactoryProvider;
+ protected IBinaryBooleanInspector binaryBooleanInspector;
+ protected IBinaryIntegerInspector binaryIntegerInspector;
+ protected IPrinterFactoryProvider printerProvider;
+ protected ILogicalExpressionJobGen exprJobGen;
+ protected IExpressionTypeComputer expressionTypeComputer;
+ protected INullableTypeComputer nullableTypeComputer;
+ protected IExpressionEvalSizeComputer expressionEvalSizeComputer;
+ protected INullWriterFactory nullWriterFactory;
+ protected INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider;
+ protected IPartialAggregationTypeComputer partialAggregationTypeComputer;
+ protected IMergeAggregationExpressionFactory mergeAggregationExpressionFactory;
+ protected PhysicalOptimizationConfig physicalOptimizationConfig = new PhysicalOptimizationConfig();
+ protected AlgebricksPartitionConstraint clusterLocations;
+ protected int frameSize = -1;
+
+ public abstract ICompilerFactory create();
+
+ public void setLogicalRewrites(List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites) {
+ this.logicalRewrites = logicalRewrites;
+ }
+
+ public void setPhysicalRewrites(List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites) {
+ this.physicalRewrites = physicalRewrites;
+ }
+
+ public void setTypeTraitProvider(ITypeTraitProvider typeTraitProvider) {
+ this.typeTraitProvider = typeTraitProvider;
+ }
+
+ public ITypeTraitProvider getTypeTraitProvider() {
+ return typeTraitProvider;
+ }
+
+ public void setSerializerDeserializerProvider(ISerializerDeserializerProvider serializerDeserializerProvider) {
+ this.serializerDeserializerProvider = serializerDeserializerProvider;
+ }
+
+ public ISerializerDeserializerProvider getSerializerDeserializerProvider() {
+ return serializerDeserializerProvider;
+ }
+
+ public void setHashFunctionFactoryProvider(IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider) {
+ this.hashFunctionFactoryProvider = hashFunctionFactoryProvider;
+ }
+
+ public IBinaryHashFunctionFactoryProvider getHashFunctionFactoryProvider() {
+ return hashFunctionFactoryProvider;
+ }
+
+ public void setComparatorFactoryProvider(IBinaryComparatorFactoryProvider comparatorFactoryProvider) {
+ this.comparatorFactoryProvider = comparatorFactoryProvider;
+ }
+
+ public IBinaryComparatorFactoryProvider getComparatorFactoryProvider() {
+ return comparatorFactoryProvider;
+ }
+
+ public void setBinaryBooleanInspector(IBinaryBooleanInspector binaryBooleanInspector) {
+ this.binaryBooleanInspector = binaryBooleanInspector;
+ }
+
+ public IBinaryBooleanInspector getBinaryBooleanInspector() {
+ return binaryBooleanInspector;
+ }
+
+ public void setBinaryIntegerInspector(IBinaryIntegerInspector binaryIntegerInspector) {
+ this.binaryIntegerInspector = binaryIntegerInspector;
+ }
+
+ public IBinaryIntegerInspector getBinaryIntegerInspector() {
+ return binaryIntegerInspector;
+ }
+
+ public void setPrinterProvider(IPrinterFactoryProvider printerProvider) {
+ this.printerProvider = printerProvider;
+ }
+
+ public IPrinterFactoryProvider getPrinterProvider() {
+ return printerProvider;
+ }
+
+ public void setExprJobGen(ILogicalExpressionJobGen exprJobGen) {
+ this.exprJobGen = exprJobGen;
+ }
+
+ public ILogicalExpressionJobGen getExprJobGen() {
+ return exprJobGen;
+ }
+
+ public void setExpressionTypeComputer(IExpressionTypeComputer expressionTypeComputer) {
+ this.expressionTypeComputer = expressionTypeComputer;
+ }
+
+ public IExpressionTypeComputer getExpressionTypeComputer() {
+ return expressionTypeComputer;
+ }
+
+ public void setClusterLocations(AlgebricksPartitionConstraint clusterLocations) {
+ this.clusterLocations = clusterLocations;
+ }
+
+ public AlgebricksPartitionConstraint getClusterLocations() {
+ return clusterLocations;
+ }
+
+ public void setNullWriterFactory(INullWriterFactory nullWriterFactory) {
+ this.nullWriterFactory = nullWriterFactory;
+ }
+
+ public INullWriterFactory getNullWriterFactory() {
+ return nullWriterFactory;
+ }
+
+ public void setExpressionEvalSizeComputer(IExpressionEvalSizeComputer expressionEvalSizeComputer) {
+ this.expressionEvalSizeComputer = expressionEvalSizeComputer;
+ }
+
+ public IExpressionEvalSizeComputer getExpressionEvalSizeComputer() {
+ return expressionEvalSizeComputer;
+ }
+
+ public void setNormalizedKeyComputerFactoryProvider(
+ INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider) {
+ this.normalizedKeyComputerFactoryProvider = normalizedKeyComputerFactoryProvider;
+ }
+
+ public INormalizedKeyComputerFactoryProvider getNormalizedKeyComputerFactoryProvider() {
+ return normalizedKeyComputerFactoryProvider;
+ }
+
+ public void setFrameSize(int frameSize) {
+ this.frameSize = frameSize;
+ }
+
+ public int getFrameSize() {
+ return frameSize;
+ }
+
+ public IPartialAggregationTypeComputer getPartialAggregationTypeComputer() {
+ return partialAggregationTypeComputer;
+ }
+
+ public void setPartialAggregationTypeComputer(IPartialAggregationTypeComputer partialAggregationTypeComputer) {
+ this.partialAggregationTypeComputer = partialAggregationTypeComputer;
+ }
+
+ public IMergeAggregationExpressionFactory getIMergeAggregationExpressionFactory() {
+ return mergeAggregationExpressionFactory;
+ }
+
+ public void setIMergeAggregationExpressionFactory(
+ IMergeAggregationExpressionFactory mergeAggregationExpressionFactory) {
+ this.mergeAggregationExpressionFactory = mergeAggregationExpressionFactory;
+ }
+
+ public PhysicalOptimizationConfig getPhysicalOptimizationConfig() {
+ return physicalOptimizationConfig;
+ }
+
+ public void setPhysicalOptimizationConfig(PhysicalOptimizationConfig physicalOptimizationConfig) {
+ this.physicalOptimizationConfig = physicalOptimizationConfig;
+ }
+
+ public void setNullableTypeComputer(INullableTypeComputer nullableTypeComputer) {
+ this.nullableTypeComputer = nullableTypeComputer;
+ }
+
+ public INullableTypeComputer getNullableTypeComputer() {
+ return nullableTypeComputer;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
new file mode 100644
index 0000000..9245772
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.api;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.PlanCompiler;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AlgebricksOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class HeuristicCompilerFactoryBuilder extends AbstractCompilerFactoryBuilder {
+
+ public static class DefaultOptimizationContextFactory implements IOptimizationContextFactory {
+
+ public static final DefaultOptimizationContextFactory INSTANCE = new DefaultOptimizationContextFactory();
+
+ private DefaultOptimizationContextFactory() {
+ }
+
+ @Override
+ public IOptimizationContext createOptimizationContext(int varCounter, int frameSize,
+ IExpressionEvalSizeComputer expressionEvalSizeComputer,
+ IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
+ IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
+ PhysicalOptimizationConfig physicalOptimizationConfig) {
+ return new AlgebricksOptimizationContext(varCounter, frameSize, expressionEvalSizeComputer,
+ mergeAggregationExpressionFactory, expressionTypeComputer, nullableTypeComputer,
+ physicalOptimizationConfig);
+ }
+ }
+
+ private IOptimizationContextFactory optCtxFactory;
+
+ public HeuristicCompilerFactoryBuilder() {
+ this.optCtxFactory = DefaultOptimizationContextFactory.INSTANCE;
+ }
+
+ public HeuristicCompilerFactoryBuilder(IOptimizationContextFactory optCtxFactory) {
+ this.optCtxFactory = optCtxFactory;
+ }
+
+ @Override
+ public ICompilerFactory create() {
+ return new ICompilerFactory() {
+ @Override
+ public ICompiler createCompiler(final ILogicalPlan plan, final IMetadataProvider<?, ?> metadata,
+ int varCounter) {
+ final IOptimizationContext oc = optCtxFactory.createOptimizationContext(varCounter, frameSize,
+ expressionEvalSizeComputer, mergeAggregationExpressionFactory, expressionTypeComputer,
+ nullableTypeComputer, physicalOptimizationConfig);
+ oc.setMetadataDeclarations(metadata);
+ final HeuristicOptimizer opt = new HeuristicOptimizer(plan, logicalRewrites, physicalRewrites, oc);
+ return new ICompiler() {
+
+ @Override
+ public void optimize() throws AlgebricksException {
+ opt.optimize();
+ }
+
+ @Override
+ public JobSpecification createJob(Object appContext) throws AlgebricksException {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting Job Generation.\n");
+ JobGenContext context = new JobGenContext(null, metadata, appContext,
+ serializerDeserializerProvider, hashFunctionFactoryProvider, comparatorFactoryProvider,
+ typeTraitProvider, binaryBooleanInspector, binaryIntegerInspector, printerProvider,
+ nullWriterFactory, normalizedKeyComputerFactoryProvider, exprJobGen,
+ expressionTypeComputer, nullableTypeComputer, oc, expressionEvalSizeComputer,
+ partialAggregationTypeComputer, frameSize, clusterLocations);
+ PlanCompiler pc = new PlanCompiler(context);
+ return pc.compilePlan(plan, null);
+ }
+ };
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java
new file mode 100644
index 0000000..654337a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompiler.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.api;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public interface ICompiler {
+ public void optimize() throws AlgebricksException;
+
+ public JobSpecification createJob(Object appContext) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompilerFactory.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompilerFactory.java
new file mode 100644
index 0000000..f831e8f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/api/ICompilerFactory.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.api;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public interface ICompilerFactory {
+ ICompiler createCompiler(ILogicalPlan plan, IMetadataProvider<?, ?> metadata, int varCounter);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
new file mode 100644
index 0000000..129ec26
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/PrioritizedRuleController.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers;
+
+import java.util.Collection;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ *
+ * Runs each rule until it produces no changes. Then the whole collection of
+ * rules is run again until no change is made.
+ *
+ *
+ * @author Nicola
+ *
+ */
+
+public class PrioritizedRuleController extends AbstractRuleController {
+
+ public PrioritizedRuleController() {
+ super();
+ }
+
+ @Override
+ public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root, Collection<IAlgebraicRewriteRule> rules)
+ throws AlgebricksException {
+ boolean anyRuleFired = false;
+ boolean anyChange = false;
+ do {
+ anyChange = false;
+ for (IAlgebraicRewriteRule r : rules) {
+ while (true) {
+ boolean ruleFired = rewriteOperatorRef(root, r);
+ if (ruleFired) {
+ anyChange = true;
+ anyRuleFired = true;
+ } else {
+ break; // go to next rule
+ }
+ }
+ }
+ } while (anyChange);
+ return anyRuleFired;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
new file mode 100644
index 0000000..8be65f4
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialFixpointRuleController.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers;
+
+import java.util.Collection;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ *
+ * Runs rules sequentially (round-robin), until one iteration over all rules
+ * produces no change.
+ *
+ *
+ * @author Nicola
+ *
+ */
+public class SequentialFixpointRuleController extends AbstractRuleController {
+
+ private boolean fullDfs;
+
+ public SequentialFixpointRuleController(boolean fullDfs) {
+ super();
+ this.fullDfs = fullDfs;
+ }
+
+ @Override
+ public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root,
+ Collection<IAlgebraicRewriteRule> ruleCollection) throws AlgebricksException {
+ boolean anyRuleFired = false;
+ boolean anyChange = false;
+ do {
+ anyChange = false;
+ for (IAlgebraicRewriteRule rule : ruleCollection) {
+ boolean ruleFired = rewriteOperatorRef(root, rule, true, fullDfs);
+ if (ruleFired) {
+ anyChange = true;
+ anyRuleFired = true;
+ }
+ }
+ } while (anyChange);
+ return anyRuleFired;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
new file mode 100644
index 0000000..5cf62e0
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-compiler/src/main/java/edu/uci/ics/hyracks/algebricks/compiler/rewriter/rulecontrollers/SequentialOnceRuleController.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers;
+
+import java.util.Collection;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class SequentialOnceRuleController extends AbstractRuleController {
+
+ private final boolean enterNestedPlans;
+
+ public SequentialOnceRuleController(boolean enterNestedPlans) {
+ super();
+ this.enterNestedPlans = enterNestedPlans;
+ }
+
+ @Override
+ public boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root, Collection<IAlgebraicRewriteRule> rules)
+ throws AlgebricksException {
+ boolean fired = false;
+ for (IAlgebraicRewriteRule rule : rules) {
+ if (rewriteOperatorRef(root, rule, enterNestedPlans, true)) {
+ fired = true;
+ }
+ }
+ return fired;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/pom.xml b/hyracks-algebricks/hyracks-algebricks-core/pom.xml
new file mode 100644
index 0000000..b6132bd
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/pom.xml
@@ -0,0 +1,41 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-algebricks-core</artifactId>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-btree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-rtree</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-dataflow-std</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/Counter.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/Counter.java
new file mode 100644
index 0000000..b27bf84
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/Counter.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public class Counter {
+ private int counter = 0;
+
+ public Counter() {
+ }
+
+ public Counter(int initial) {
+ counter = initial;
+ }
+
+ public int get() {
+ return counter;
+ }
+
+ public void inc() {
+ ++counter;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/EquivalenceClass.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/EquivalenceClass.java
new file mode 100644
index 0000000..fe3511e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/EquivalenceClass.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public final class EquivalenceClass {
+ private List<LogicalVariable> members;
+ private ILogicalExpression constRepresentative;
+ private LogicalVariable variableRepresentative;
+ private boolean representativeIsConst;
+
+ public EquivalenceClass(List<LogicalVariable> members, ILogicalExpression constRepresentative) {
+ this.members = members;
+ this.constRepresentative = constRepresentative;
+ representativeIsConst = true;
+ }
+
+ public EquivalenceClass(List<LogicalVariable> members, LogicalVariable variableRepresentative) {
+ this.members = members;
+ this.variableRepresentative = variableRepresentative;
+ representativeIsConst = false;
+ }
+
+ public boolean representativeIsConst() {
+ return representativeIsConst;
+ }
+
+ public List<LogicalVariable> getMembers() {
+ return members;
+ }
+
+ public boolean contains(LogicalVariable var) {
+ return members.contains(var);
+ }
+
+ public ILogicalExpression getConstRepresentative() {
+ return constRepresentative;
+ }
+
+ public LogicalVariable getVariableRepresentative() {
+ return variableRepresentative;
+ }
+
+ public void setConstRepresentative(ILogicalExpression constRepresentative) {
+ this.constRepresentative = constRepresentative;
+ this.representativeIsConst = true;
+ }
+
+ public void setVariableRepresentative(LogicalVariable variableRepresentative) {
+ this.variableRepresentative = variableRepresentative;
+ this.representativeIsConst = false;
+ }
+
+ public void merge(EquivalenceClass ec2) {
+ members.addAll(ec2.getMembers());
+ if (!representativeIsConst && ec2.representativeIsConst()) {
+ representativeIsConst = true;
+ constRepresentative = ec2.getConstRepresentative();
+ }
+ }
+
+ public void addMember(LogicalVariable v) {
+ members.add(v);
+ }
+
+ public EquivalenceClass cloneEquivalenceClass() {
+ List<LogicalVariable> membersClone = new LinkedList<LogicalVariable>();
+ membersClone.addAll(members);
+ EquivalenceClass ec;
+ if (representativeIsConst()) {
+ ec = new EquivalenceClass(membersClone, constRepresentative);
+ } else {
+ ec = new EquivalenceClass(membersClone, variableRepresentative);
+ }
+ return ec;
+ }
+
+ @Override
+ public String toString() {
+ return "(<" + (representativeIsConst ? constRepresentative : variableRepresentative) + "> " + members + ")";
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof EquivalenceClass)) {
+ return false;
+ } else {
+ EquivalenceClass ec = (EquivalenceClass) obj;
+ if (!members.equals(ec.getMembers())) {
+ return false;
+ }
+ if (representativeIsConst) {
+ return ec.representativeIsConst() && (constRepresentative.equals(ec.getConstRepresentative()));
+ } else {
+ return !ec.representativeIsConst() && (variableRepresentative.equals(ec.getVariableRepresentative()));
+ }
+ }
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
new file mode 100644
index 0000000..19c85c6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IHyracksJobBuilder.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public interface IHyracksJobBuilder {
+ public enum TargetConstraint {
+ ONE,
+ SAME_COUNT
+ }
+
+ public void contributeHyracksOperator(ILogicalOperator op, IOperatorDescriptor opDesc);
+
+ public void contributeAlgebricksPartitionConstraint(IOperatorDescriptor opDesc, AlgebricksPartitionConstraint apc);
+
+ public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc);
+
+ public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc,
+ AlgebricksPartitionConstraint pc);
+
+ /**
+ * inputs are numbered starting from 0
+ */
+ public void contributeGraphEdge(ILogicalOperator src, int srcOutputIndex, ILogicalOperator dest, int destInputIndex);
+
+ public void contributeConnector(ILogicalOperator exchgOp, IConnectorDescriptor conn);
+
+ public void contributeConnectorWithTargetConstraint(ILogicalOperator exchgOp, IConnectorDescriptor conn,
+ TargetConstraint numberOfTargetPartitions);
+
+ public JobSpecification getJobSpec();
+
+ /**
+ * to be called only after all the graph information is added
+ */
+ public void buildSpec(List<ILogicalOperator> roots) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
new file mode 100644
index 0000000..a54a88d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalExpression.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface ILogicalExpression {
+
+ public LogicalExpressionTag getExpressionTag();
+
+ public abstract <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException;
+
+ public void getUsedVariables(Collection<LogicalVariable> vars);
+
+ public void substituteVar(LogicalVariable v1, LogicalVariable v2);
+
+ // constraints (e.g., FDs, equivalences)
+
+ /**
+ * @param fds
+ * Output argument: functional dependencies that can be inferred
+ * from this expression.
+ * @param equivClasses
+ * Output argument: Equivalence classes that can be inferred from
+ * this expression.
+ */
+ public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
+ Map<LogicalVariable, EquivalenceClass> equivClasses);
+
+ /**
+ * @param fds
+ * Output argument: functional dependencies that can be inferred
+ * from this expression.
+ * @param outerVars
+ * Input argument: variables coming from outer branch(es), e.g.,
+ * the left branch of a left outer join.
+ */
+ public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds, Collection<LogicalVariable> outerVars);
+
+ /**
+ * @param conjs
+ * Output argument: a list of expression whose conjunction, in
+ * any order, can replace the current expression.
+ * @return true if the expression can be broken in at least two conjuncts,
+ * false otherwise.
+ */
+ public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs);
+
+ public abstract ILogicalExpression cloneExpression();
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
new file mode 100644
index 0000000..719b5df
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalOperator.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface ILogicalOperator {
+
+ public List<Mutable<ILogicalOperator>> getInputs();
+
+ boolean hasInputs();
+
+ public void recomputeSchema() throws AlgebricksException;
+
+ public List<LogicalVariable> getSchema();
+
+ /*
+ *
+ * support for visitors
+ */
+
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException;
+
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException;
+
+ public boolean isMap();
+
+ public Map<String, Object> getAnnotations();
+
+ public void removeAnnotation(String annotationName);
+
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException;
+
+ // variables
+
+ /**
+ * Get the variable propogation policy from this operator's input to its
+ * output.
+ *
+ * @return The VariablePropogationPolicy.
+ */
+ public VariablePropagationPolicy getVariablePropagationPolicy();
+
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException;
+
+ public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException;
+
+ // structural properties
+
+ /**
+ * @return for each child, one vector of required physical properties
+ */
+
+ public PhysicalRequirements getRequiredPhysicalPropertiesForChildren(IPhysicalPropertiesVector requiredProperties);
+
+ /**
+ * @return the physical properties that this operator delivers, based on
+ * what its children deliver
+ */
+
+ public IPhysicalPropertiesVector getDeliveredPhysicalProperties();
+
+ public void computeDeliveredPhysicalProperties(IOptimizationContext context) throws AlgebricksException;
+
+ public void setHostQueryContext(Object context);
+
+ public Object getHostQueryContext();
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
new file mode 100644
index 0000000..c94a9c1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlan.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+public interface ILogicalPlan {
+ public List<Mutable<ILogicalOperator>> getRoots();
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
new file mode 100644
index 0000000..5a616c6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/ILogicalPlanAndMetadata.java
@@ -0,0 +1,26 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+
+public interface ILogicalPlanAndMetadata {
+ public ILogicalPlan getPlan();
+
+ public IMetadataProvider<?, ?> getMetadataProvider();
+
+ public AlgebricksPartitionConstraint getClusterLocations();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
new file mode 100644
index 0000000..4daaceb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IOptimizationContext.java
@@ -0,0 +1,88 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+
+public interface IOptimizationContext extends ITypingContext {
+
+ public abstract int getVarCounter();
+
+ // -1 if unknown
+ public abstract int getFrameSize();
+
+ public abstract void setVarCounter(int varCounter);
+
+ public abstract LogicalVariable newVar();
+
+ public abstract IMetadataProvider<?, ?> getMetadataProvider();
+
+ public abstract void setMetadataDeclarations(IMetadataProvider<?, ?> metadataProvider);
+
+ public abstract boolean checkIfInDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op);
+
+ public abstract void addToDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op);
+
+ /*
+ * returns true if op1 and op2 have already been compared
+ */
+ public abstract boolean checkAndAddToAlreadyCompared(ILogicalOperator op1, ILogicalOperator op2);
+
+ public abstract void addNotToBeInlinedVar(LogicalVariable var);
+
+ public abstract boolean shouldNotBeInlined(LogicalVariable var);
+
+ public abstract void addPrimaryKey(FunctionalDependency pk);
+
+ public abstract List<LogicalVariable> findPrimaryKey(LogicalVariable recordVar);
+
+ public abstract void putEquivalenceClassMap(ILogicalOperator op, Map<LogicalVariable, EquivalenceClass> eqClassMap);
+
+ public abstract Map<LogicalVariable, EquivalenceClass> getEquivalenceClassMap(ILogicalOperator op);
+
+ public abstract void putFDList(ILogicalOperator op, List<FunctionalDependency> fdList);
+
+ public abstract List<FunctionalDependency> getFDList(ILogicalOperator op);
+
+ public abstract void putLogicalPropertiesVector(ILogicalOperator op, ILogicalPropertiesVector v);
+
+ public abstract ILogicalPropertiesVector getLogicalPropertiesVector(ILogicalOperator op);
+
+ public abstract IExpressionEvalSizeComputer getExpressionEvalSizeComputer();
+
+ public abstract IVariableEvalSizeEnvironment getVariableEvalSizeEnvironment();
+
+ public abstract IMergeAggregationExpressionFactory getMergeAggregationExpressionFactory();
+
+ public abstract PhysicalOptimizationConfig getPhysicalOptimizationConfig();
+
+ public abstract void invalidateTypeEnvironmentForOperator(ILogicalOperator op);
+
+ public abstract void computeAndSetTypeEnvironmentForOperator(ILogicalOperator op) throws AlgebricksException;
+
+ public abstract void updatePrimaryKeys(Map<LogicalVariable, LogicalVariable> mappedVars);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
new file mode 100644
index 0000000..46d2dc3
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/IPhysicalOperator.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IPhysicalOperator {
+
+ public PhysicalOperatorTag getOperatorTag();
+
+ /**
+ * @param op
+ * the logical operator this physical operator annotates
+ * @param reqdByParent
+ * parent's requirements, which are not enforced for now, as we
+ * only explore one plan
+ * @return for each child, one vector of required physical properties
+ */
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent);
+
+ /**
+ * @return the physical properties that this operator delivers, based on
+ * what its children deliver
+ */
+ public IPhysicalPropertiesVector getDeliveredProperties();
+
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException;
+
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException;
+
+ public void disableJobGenBelowMe();
+
+ public boolean isJobGenDisabledBelowMe();
+
+ public boolean isMicroOperator();
+
+ public void setHostQueryContext(Object context);
+
+ public Object getHostQueryContext();
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java
new file mode 100644
index 0000000..166b357
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalExpressionTag.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public enum LogicalExpressionTag {
+ FUNCTION_CALL, VARIABLE, CONSTANT
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
new file mode 100644
index 0000000..01fa9da
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalOperatorTag.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public enum LogicalOperatorTag {
+ AGGREGATE,
+ ASSIGN,
+ CLUSTER,
+ DATASOURCESCAN,
+ DISTINCT,
+ GROUP,
+ EMPTYTUPLESOURCE,
+ EXCHANGE,
+ INNERJOIN,
+ LEFTOUTERJOIN,
+ LIMIT,
+ DIE,
+ NESTEDTUPLESOURCE,
+ ORDER,
+ PROJECT,
+ PARTITIONINGSPLIT,
+ REPLICATE,
+ RUNNINGAGGREGATE,
+ SCRIPT,
+ SELECT,
+ SINK,
+ SUBPLAN,
+ UNIONALL,
+ UNNEST,
+ UNNEST_MAP,
+ WRITE,
+ WRITE_RESULT,
+ INSERT_DELETE,
+ INDEX_INSERT_DELETE,
+ UPDATE
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalVariable.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalVariable.java
new file mode 100644
index 0000000..0a88459
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/LogicalVariable.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+/**
+ * Represents a logical variable in an asterix logical plan.
+ *
+ * @author Vinayak R. Borkar [vborkar@ics.uci.edu]
+ */
+public final class LogicalVariable {
+ private final int id;
+
+ public LogicalVariable(int id) {
+ this.id = id;
+ }
+
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public String toString() {
+ return "$$" + id;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof LogicalVariable)) {
+ return false;
+ } else {
+ return id == ((LogicalVariable) obj).getId();
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return id;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java
new file mode 100644
index 0000000..655645f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/OperatorAnnotations.java
@@ -0,0 +1,12 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public interface OperatorAnnotations {
+ // hints
+ public static final String USE_HASH_GROUP_BY = "USE_HASH_GROUP_BY"; // -->
+ public static final String USE_EXTERNAL_GROUP_BY = "USE_EXTERNAL_GROUP_BY"; // -->
+ // Boolean
+ public static final String CARDINALITY = "CARDINALITY"; // -->
+ // Integer
+ public static final String MAX_NUMBER_FRAMES = "MAX_NUMBER_FRAMES"; // -->
+ // Integer
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
new file mode 100644
index 0000000..48f320b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/base/PhysicalOperatorTag.java
@@ -0,0 +1,45 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.base;
+
+public enum PhysicalOperatorTag {
+ AGGREGATE,
+ ASSIGN,
+ BROADCAST_EXCHANGE,
+ BTREE_SEARCH,
+ DATASOURCE_SCAN,
+ EMPTY_TUPLE_SOURCE,
+ EXTERNAL_GROUP_BY,
+ IN_MEMORY_HASH_JOIN,
+ HASH_GROUP_BY,
+ HASH_PARTITION_EXCHANGE,
+ HASH_PARTITION_MERGE_EXCHANGE,
+ HYBRID_HASH_JOIN,
+ HDFS_READER,
+ IN_MEMORY_STABLE_SORT,
+ MICRO_PRE_CLUSTERED_GROUP_BY,
+ NESTED_LOOP,
+ NESTED_TUPLE_SOURCE,
+ ONE_TO_ONE_EXCHANGE,
+ PRE_SORTED_DISTINCT_BY,
+ PRE_CLUSTERED_GROUP_BY,
+ RANGE_PARTITION_EXCHANGE,
+ RANDOM_MERGE_EXCHANGE,
+ RTREE_SEARCH,
+ RUNNING_AGGREGATE,
+ SORT_MERGE_EXCHANGE,
+ SINK,
+ SINK_WRITE,
+ SPLIT,
+ STABLE_SORT,
+ STREAM_LIMIT,
+ STREAM_DIE,
+ STREAM_SELECT,
+ STREAM_PROJECT,
+ STRING_STREAM_SCRIPT,
+ SUBPLAN,
+ UNION_ALL,
+ UNNEST,
+ WRITE_RESULT,
+ INSERT_DELETE,
+ INDEX_INSERT_DELETE,
+ UPDATE
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriter.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriter.java
new file mode 100644
index 0000000..d29fab6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriter.java
@@ -0,0 +1,11 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+
+public interface IAWriter {
+
+ public void init() throws AlgebricksException;
+
+ public void printTuple(FrameTupleAccessor tAccess, int tIdx) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriterFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriterFactory.java
new file mode 100644
index 0000000..200f58f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IAWriterFactory.java
@@ -0,0 +1,11 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import java.io.PrintStream;
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public interface IAWriterFactory extends Serializable {
+ public IAWriter createWriter(int[] fields, PrintStream ps, IPrinterFactory[] printerFactories,
+ RecordDescriptor inputRecordDescriptor);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryBooleanInspector.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryBooleanInspector.java
new file mode 100644
index 0000000..60e5050
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryBooleanInspector.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import java.io.Serializable;
+
+public interface IBinaryBooleanInspector extends Serializable {
+ public boolean getBooleanValue(byte[] bytes, int offset, int length);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryComparatorFactoryProvider.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryComparatorFactoryProvider.java
new file mode 100644
index 0000000..bb4b734
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryComparatorFactoryProvider.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+
+public interface IBinaryComparatorFactoryProvider {
+ /**
+ * only ASC and DESC are currently supported for orderKind (FUNCTIONCALL can
+ * be ignored)
+ */
+ public IBinaryComparatorFactory getBinaryComparatorFactory(Object type, OrderKind orderKind)
+ throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryHashFunctionFactoryProvider.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryHashFunctionFactoryProvider.java
new file mode 100644
index 0000000..2e1f0fa
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryHashFunctionFactoryProvider.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+
+public interface IBinaryHashFunctionFactoryProvider {
+ public IBinaryHashFunctionFactory getBinaryHashFunctionFactory(Object type) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryIntegerInspector.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryIntegerInspector.java
new file mode 100644
index 0000000..3a2fe29
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IBinaryIntegerInspector.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import java.io.Serializable;
+
+public interface IBinaryIntegerInspector extends Serializable {
+ public int getIntegerValue(byte[] bytes, int offset, int length);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/INormalizedKeyComputerFactoryProvider.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/INormalizedKeyComputerFactoryProvider.java
new file mode 100644
index 0000000..1883ef5
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/INormalizedKeyComputerFactoryProvider.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+
+public interface INormalizedKeyComputerFactoryProvider {
+ public INormalizedKeyComputerFactory getNormalizedKeyComputerFactory(Object type, OrderKind order);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinter.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinter.java
new file mode 100644
index 0000000..89de415
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinter.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IPrinter {
+ public void init() throws AlgebricksException;
+
+ public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
new file mode 100644
index 0000000..871fe43
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactory.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import java.io.Serializable;
+
+public interface IPrinterFactory extends Serializable {
+ public IPrinter createPrinter();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactoryProvider.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactoryProvider.java
new file mode 100644
index 0000000..e2e2f1c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/IPrinterFactoryProvider.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IPrinterFactoryProvider {
+ public IPrinterFactory getPrinterFactory(Object type) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ISerializerDeserializerProvider.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ISerializerDeserializerProvider.java
new file mode 100644
index 0000000..66d696c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ISerializerDeserializerProvider.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+
+public interface ISerializerDeserializerProvider {
+ @SuppressWarnings("unchecked")
+ public ISerializerDeserializer getSerializerDeserializer(Object type) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ITypeTraitProvider.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ITypeTraitProvider.java
new file mode 100644
index 0000000..6ad3079
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/data/ITypeTraitProvider.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+
+public interface ITypeTraitProvider {
+ public ITypeTraits getTypeTrait(Object type);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
new file mode 100644
index 0000000..dae055c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractFunctionCallExpression.java
@@ -0,0 +1,326 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+
+public abstract class AbstractFunctionCallExpression extends AbstractLogicalExpression {
+
+ public enum FunctionKind {
+ SCALAR,
+ STATEFUL,
+ AGGREGATE,
+ UNNEST
+ }
+
+ protected IFunctionInfo finfo;
+ final private List<Mutable<ILogicalExpression>> arguments;
+ private Object[] opaqueParameters;
+ private final FunctionKind kind;
+ private Map<Object, IExpressionAnnotation> annotationMap = new HashMap<Object, IExpressionAnnotation>();
+
+ public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo,
+ List<Mutable<ILogicalExpression>> arguments) {
+ this.kind = kind;
+ this.finfo = finfo;
+ this.arguments = arguments;
+ }
+
+ public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo) {
+ this.kind = kind;
+ this.finfo = finfo;
+ this.arguments = new ArrayList<Mutable<ILogicalExpression>>();
+ }
+
+ public AbstractFunctionCallExpression(FunctionKind kind, IFunctionInfo finfo,
+ Mutable<ILogicalExpression>... expressions) {
+ this(kind, finfo);
+ for (Mutable<ILogicalExpression> e : expressions) {
+ this.arguments.add(e);
+ }
+ }
+
+ public void setOpaqueParameters(Object[] opaqueParameters) {
+ this.opaqueParameters = opaqueParameters;
+ }
+
+ public Object[] getOpaqueParameters() {
+ return opaqueParameters;
+ }
+
+ public FunctionKind getKind() {
+ return kind;
+ }
+
+ protected List<Mutable<ILogicalExpression>> cloneArguments() {
+ List<Mutable<ILogicalExpression>> clonedArgs = new ArrayList<Mutable<ILogicalExpression>>(arguments.size());
+ for (Mutable<ILogicalExpression> e : arguments) {
+ ILogicalExpression e2 = ((AbstractLogicalExpression) e.getValue()).cloneExpression();
+ clonedArgs.add(new MutableObject<ILogicalExpression>(e2));
+ }
+ return clonedArgs;
+ }
+
+ public FunctionIdentifier getFunctionIdentifier() {
+ return finfo.getFunctionIdentifier();
+ }
+
+ public IFunctionInfo getFunctionInfo() {
+ return finfo;
+ }
+
+ public void setFunctionInfo(IFunctionInfo finfo) {
+ this.finfo = finfo;
+ }
+
+ public List<Mutable<ILogicalExpression>> getArguments() {
+ return arguments;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("function-call: " + finfo.getFunctionIdentifier() + ", Args:[");
+ // + arguments;
+ boolean first = true;
+ for (Mutable<ILogicalExpression> ref : arguments) {
+ if (first) {
+ first = false;
+ } else {
+ sb.append(", ");
+ }
+ sb.append(ref.getValue());
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ @Override
+ public LogicalExpressionTag getExpressionTag() {
+ return LogicalExpressionTag.FUNCTION_CALL;
+ }
+
+ @Override
+ public void getUsedVariables(Collection<LogicalVariable> vars) {
+ for (Mutable<ILogicalExpression> arg : arguments) {
+ arg.getValue().getUsedVariables(vars);
+ }
+ }
+
+ @Override
+ public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
+ for (Mutable<ILogicalExpression> arg : arguments) {
+ arg.getValue().substituteVar(v1, v2);
+ }
+ }
+
+ @Override
+ public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
+ Map<LogicalVariable, EquivalenceClass> equivClasses) {
+ FunctionIdentifier funId = getFunctionIdentifier();
+ if (funId == AlgebricksBuiltinFunctions.AND) {
+ for (Mutable<ILogicalExpression> a : arguments) {
+ a.getValue().getConstraintsAndEquivClasses(fds, equivClasses);
+ }
+ } else if (funId == AlgebricksBuiltinFunctions.EQ) {
+ ILogicalExpression opLeft = arguments.get(0).getValue();
+ ILogicalExpression opRight = arguments.get(1).getValue();
+ if (opLeft.getExpressionTag() == LogicalExpressionTag.CONSTANT
+ && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ ConstantExpression op1 = (ConstantExpression) opLeft;
+ VariableReferenceExpression op2 = (VariableReferenceExpression) opRight;
+ getFDsAndEquivClassesForEqWithConstant(op1, op2, fds, equivClasses);
+ } else if (opLeft.getExpressionTag() == LogicalExpressionTag.VARIABLE
+ && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression op1 = (VariableReferenceExpression) opLeft;
+ VariableReferenceExpression op2 = (VariableReferenceExpression) opRight;
+ getFDsAndEquivClassesForColumnEq(op1, op2, fds, equivClasses);
+ }
+ }
+ }
+
+ @Override
+ public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds, Collection<LogicalVariable> outerVars) {
+ FunctionIdentifier funId = getFunctionIdentifier();
+ if (funId == AlgebricksBuiltinFunctions.AND) {
+ for (Mutable<ILogicalExpression> a : arguments) {
+ a.getValue().getConstraintsForOuterJoin(fds, outerVars);
+ }
+ } else if (funId == AlgebricksBuiltinFunctions.EQ) {
+ ILogicalExpression opLeft = arguments.get(0).getValue();
+ ILogicalExpression opRight = arguments.get(1).getValue();
+ if (opLeft.getExpressionTag() == LogicalExpressionTag.VARIABLE
+ && opRight.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ LogicalVariable var1 = ((VariableReferenceExpression) opLeft).getVariableReference();
+ LogicalVariable var2 = ((VariableReferenceExpression) opRight).getVariableReference();
+ if (outerVars.contains(var1)) {
+ addFD(fds, var1, var2);
+ }
+ if (outerVars.contains(var2)) {
+ addFD(fds, var2, var1);
+ }
+ }
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof AbstractFunctionCallExpression)) {
+ return false;
+ } else {
+ AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) obj;
+ boolean equal = getFunctionIdentifier().equals(fce.getFunctionIdentifier());
+ if (!equal)
+ return false;
+ for (int i = 0; i < arguments.size(); i++) {
+ ILogicalExpression argument = arguments.get(i).getValue();
+ ILogicalExpression fceArgument = fce.getArguments().get(i).getValue();
+ if (!argument.equals(fceArgument))
+ return false;
+ }
+ return true;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ int h = finfo.hashCode();
+ for (Mutable<ILogicalExpression> e : arguments) {
+ h = h * 41 + e.getValue().hashCode();
+ }
+ return h;
+ }
+
+ @Override
+ public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
+ if (getFunctionIdentifier() != AlgebricksBuiltinFunctions.AND || arguments.size() <= 1) {
+ return false;
+ } else {
+ conjs.addAll(arguments);
+ return true;
+ }
+ }
+
+ public Map<Object, IExpressionAnnotation> getAnnotations() {
+ return annotationMap;
+ }
+
+ protected Map<Object, IExpressionAnnotation> cloneAnnotations() {
+ Map<Object, IExpressionAnnotation> m = new HashMap<Object, IExpressionAnnotation>();
+ for (Object k : annotationMap.keySet()) {
+ IExpressionAnnotation annot2 = annotationMap.get(k).copy();
+ m.put(k, annot2);
+ }
+ return m;
+ }
+
+ private final static void addFD(Collection<FunctionalDependency> fds, LogicalVariable var1, LogicalVariable var2) {
+ LinkedList<LogicalVariable> set1 = new LinkedList<LogicalVariable>();
+ set1.add(var1);
+ LinkedList<LogicalVariable> set2 = new LinkedList<LogicalVariable>();
+ set2.add(var2);
+ FunctionalDependency fd1 = new FunctionalDependency(set1, set2);
+ fds.add(fd1);
+ }
+
+ private final static void getFDsAndEquivClassesForEqWithConstant(ConstantExpression c,
+ VariableReferenceExpression v, Collection<FunctionalDependency> fds,
+ Map<LogicalVariable, EquivalenceClass> equivClasses) {
+ LogicalVariable var = v.getVariableReference();
+ LinkedList<LogicalVariable> head = new LinkedList<LogicalVariable>();
+ // empty set in the head
+ LinkedList<LogicalVariable> tail = new LinkedList<LogicalVariable>();
+ tail.add(var);
+ FunctionalDependency fd = new FunctionalDependency(head, tail);
+ fds.add(fd);
+
+ EquivalenceClass ec = equivClasses.get(var);
+ if (ec == null) {
+ LinkedList<LogicalVariable> members = new LinkedList<LogicalVariable>();
+ members.add(var);
+ EquivalenceClass eclass = new EquivalenceClass(members, c);
+ equivClasses.put(var, eclass);
+ } else {
+ if (ec.representativeIsConst()) {
+ ILogicalExpression c1 = ec.getConstRepresentative();
+ if (!c1.equals(c)) {
+ // here I could also rewrite to FALSE
+ return;
+ }
+ }
+ ec.setConstRepresentative(c);
+ }
+ }
+
+ /*
+ * Obs.: mgmt. of equiv. classes should use a more efficient data
+ * structure,if we are to implem. cost-bazed optim.
+ */
+ private final static void getFDsAndEquivClassesForColumnEq(VariableReferenceExpression v1,
+ VariableReferenceExpression v2, Collection<FunctionalDependency> fds,
+ Map<LogicalVariable, EquivalenceClass> equivClasses) {
+ LogicalVariable var1 = v1.getVariableReference();
+ LogicalVariable var2 = v2.getVariableReference();
+ LinkedList<LogicalVariable> set1 = new LinkedList<LogicalVariable>();
+ set1.add(var1);
+ LinkedList<LogicalVariable> set2 = new LinkedList<LogicalVariable>();
+ set2.add(var2);
+ FunctionalDependency fd1 = new FunctionalDependency(set1, set2);
+ FunctionalDependency fd2 = new FunctionalDependency(set2, set1);
+ fds.add(fd1);
+ fds.add(fd2);
+
+ EquivalenceClass ec1 = equivClasses.get(var1);
+ EquivalenceClass ec2 = equivClasses.get(var2);
+ if (ec1 == null && ec2 == null) {
+ LinkedList<LogicalVariable> members = new LinkedList<LogicalVariable>();
+ members.add(var1);
+ members.add(var2);
+ EquivalenceClass ec = new EquivalenceClass(members, var1);
+ equivClasses.put(var1, ec);
+ equivClasses.put(var2, ec);
+ } else if (ec1 == null && ec2 != null) {
+ ec2.addMember(var1);
+ equivClasses.put(var1, ec2);
+ } else if (ec2 == null && ec1 != null) {
+ ec1.addMember(var2);
+ equivClasses.put(var2, ec1);
+ } else {
+ ec1.merge(ec2);
+ for (LogicalVariable w : equivClasses.keySet()) {
+ if (ec2.getMembers().contains(w)) {
+ equivClasses.put(w, ec1);
+ }
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java
new file mode 100644
index 0000000..f9b30cc
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AbstractLogicalExpression.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.Collection;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+
+public abstract class AbstractLogicalExpression implements ILogicalExpression {
+
+ @Override
+ public void getConstraintsAndEquivClasses(Collection<FunctionalDependency> fds,
+ Map<LogicalVariable, EquivalenceClass> equivClasses) {
+ // do nothing
+ }
+
+ @Override
+ public void getConstraintsForOuterJoin(Collection<FunctionalDependency> fds,
+ Collection<LogicalVariable> innerSideVars) {
+ // do nothing
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
new file mode 100644
index 0000000..ac77ee2
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/AggregateFunctionCallExpression.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+/**
+ *
+ * An aggregate function may be executed in a "two step" mode. First the
+ * "step-one" aggregates are run and then the results are passed to the
+ * "step-two" aggregators. The convention is the following:
+ *
+ * 1. The step-one aggregate must be able to accept the same arguments as the
+ * original aggregate function call.
+ *
+ * 2. The step-two aggregate must be a unary function that accepts as input the
+ * output of the step-one aggregate.
+ *
+ */
+
+public class AggregateFunctionCallExpression extends AbstractFunctionCallExpression {
+
+ private boolean twoStep;
+ private IFunctionInfo stepOneAggregate;
+ private IFunctionInfo stepTwoAggregate;
+
+ public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep) {
+ super(FunctionKind.AGGREGATE, finfo);
+ this.twoStep = isTwoStep;
+ }
+
+ public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep,
+ List<Mutable<ILogicalExpression>> arguments) {
+ super(FunctionKind.AGGREGATE, finfo, arguments);
+ this.twoStep = isTwoStep;
+ }
+
+ public AggregateFunctionCallExpression(IFunctionInfo finfo, boolean isTwoStep,
+ Mutable<ILogicalExpression>... expressions) {
+ super(FunctionKind.AGGREGATE, finfo, expressions);
+ this.twoStep = isTwoStep;
+ }
+
+ public boolean isTwoStep() {
+ return twoStep;
+ }
+
+ public void setTwoStep(boolean twoStep) {
+ this.twoStep = twoStep;
+ }
+
+ @Override
+ public AggregateFunctionCallExpression cloneExpression() {
+ cloneAnnotations();
+ List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
+ AggregateFunctionCallExpression fun = new AggregateFunctionCallExpression(finfo, twoStep, clonedArgs);
+ fun.setStepTwoAggregate(stepTwoAggregate);
+ fun.setStepOneAggregate(stepOneAggregate);
+ return fun;
+ }
+
+ public void setStepOneAggregate(IFunctionInfo stepOneAggregate) {
+ this.stepOneAggregate = stepOneAggregate;
+ }
+
+ public IFunctionInfo getStepOneAggregate() {
+ return stepOneAggregate;
+ }
+
+ public void setStepTwoAggregate(IFunctionInfo stepTwoAggregate) {
+ this.stepTwoAggregate = stepTwoAggregate;
+ }
+
+ public IFunctionInfo getStepTwoAggregate() {
+ return stepTwoAggregate;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitAggregateFunctionCallExpression(this, arg);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
new file mode 100644
index 0000000..ff88912
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/BroadcastExpressionAnnotation.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public class BroadcastExpressionAnnotation implements IExpressionAnnotation {
+
+ public static final String BROADCAST_ANNOTATION_KEY = "broadcast";
+
+ public enum BroadcastSide {
+ LEFT,
+ RIGHT
+ };
+
+ private BroadcastSide side;
+
+ @Override
+ public Object getObject() {
+ return side;
+ }
+
+ @Override
+ public void setObject(Object side) {
+ this.side = (BroadcastSide) side;
+ }
+
+ @Override
+ public IExpressionAnnotation copy() {
+ BroadcastExpressionAnnotation bcast = new BroadcastExpressionAnnotation();
+ bcast.side = side;
+ return bcast;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
new file mode 100644
index 0000000..2fa0637
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ConstantExpression.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public final class ConstantExpression extends AbstractLogicalExpression {
+ private IAlgebricksConstantValue value;
+
+ public final static ConstantExpression TRUE = new ConstantExpression(new IAlgebricksConstantValue() {
+
+ @Override
+ public boolean isTrue() {
+ return true;
+ }
+
+ @Override
+ public boolean isNull() {
+ return false;
+ }
+
+ @Override
+ public boolean isFalse() {
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "TRUE";
+ }
+ });
+ public final static ConstantExpression FALSE = new ConstantExpression(new IAlgebricksConstantValue() {
+
+ @Override
+ public boolean isTrue() {
+ return false;
+ }
+
+ @Override
+ public boolean isNull() {
+ return false;
+ }
+
+ @Override
+ public boolean isFalse() {
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "FALSE";
+ }
+ });
+ public final static ConstantExpression NULL = new ConstantExpression(new IAlgebricksConstantValue() {
+
+ @Override
+ public boolean isTrue() {
+ return false;
+ }
+
+ @Override
+ public boolean isNull() {
+ return true;
+ }
+
+ @Override
+ public boolean isFalse() {
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "NULL";
+ }
+ });
+
+ private Map<Object, IExpressionAnnotation> annotationMap = new HashMap<Object, IExpressionAnnotation>();
+
+ public ConstantExpression(IAlgebricksConstantValue value) {
+ this.value = value;
+ }
+
+ public IAlgebricksConstantValue getValue() {
+ return value;
+ }
+
+ public void setValue(IAlgebricksConstantValue value) {
+ this.value = value;
+ }
+
+ @Override
+ public LogicalExpressionTag getExpressionTag() {
+ return LogicalExpressionTag.CONSTANT;
+ }
+
+ @Override
+ public String toString() {
+ return value.toString();
+ }
+
+ @Override
+ public void getUsedVariables(Collection<LogicalVariable> vars) {
+ // do nothing
+ }
+
+ @Override
+ public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
+ // do nothing
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof ConstantExpression)) {
+ return false;
+ } else {
+ return value.equals(((ConstantExpression) obj).getValue());
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return value.hashCode();
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitConstantExpression(this, arg);
+ }
+
+ @Override
+ public AbstractLogicalExpression cloneExpression() {
+ Map<Object, IExpressionAnnotation> m = new HashMap<Object, IExpressionAnnotation>();
+ for (Object k : annotationMap.keySet()) {
+ IExpressionAnnotation annot2 = annotationMap.get(k).copy();
+ m.put(k, annot2);
+ }
+ ConstantExpression c = new ConstantExpression(value);
+ c.annotationMap = m;
+ return c;
+ }
+
+ public Map<Object, IExpressionAnnotation> getAnnotations() {
+ return annotationMap;
+ }
+
+ @Override
+ public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java
new file mode 100644
index 0000000..6870a38
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ExpressionAnnotationNoCopyImpl.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public class ExpressionAnnotationNoCopyImpl implements IExpressionAnnotation {
+
+ private Object object;
+
+ @Override
+ public IExpressionAnnotation copy() {
+ return this;
+ }
+
+ @Override
+ public Object getObject() {
+ return object;
+ }
+
+ @Override
+ public void setObject(Object object) {
+ this.object = object;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java
new file mode 100644
index 0000000..b6c7dbd
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IAlgebricksConstantValue.java
@@ -0,0 +1,9 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public interface IAlgebricksConstantValue {
+ public boolean isNull();
+
+ public boolean isTrue();
+
+ public boolean isFalse();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java
new file mode 100644
index 0000000..15e543b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionAnnotation.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+public interface IExpressionAnnotation {
+ public Object getObject();
+
+ public void setObject(Object object);
+
+ public IExpressionAnnotation copy();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
new file mode 100644
index 0000000..8dd807e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionEvalSizeComputer.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IExpressionEvalSizeComputer {
+ // size in bytes, or -1 if unknown
+ public int getEvalSize(ILogicalExpression expr, IVariableEvalSizeEnvironment env) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
new file mode 100644
index 0000000..72861ae
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IExpressionTypeComputer.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IExpressionTypeComputer {
+ public Object getType(ILogicalExpression expr, IMetadataProvider<?, ?> metadataProvider,
+ IVariableTypeEnvironment env) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
new file mode 100644
index 0000000..babfede
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ILogicalExpressionJobGen.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface ILogicalExpressionJobGen {
+
+ public IEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+ IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException;
+
+ public IAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException;
+
+ public ISerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+ AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
+ JobGenContext context) throws AlgebricksException;
+
+ public IRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException;
+
+ public IUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException;
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
new file mode 100644
index 0000000..36a414c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IMergeAggregationExpressionFactory.java
@@ -0,0 +1,10 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IMergeAggregationExpressionFactory {
+ ILogicalExpression createMergeAggregation(ILogicalExpression expr, IOptimizationContext env)
+ throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
new file mode 100644
index 0000000..3290e35
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/INullableTypeComputer.java
@@ -0,0 +1,7 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface INullableTypeComputer {
+ public Object makeNullableType(Object type) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
new file mode 100644
index 0000000..88e34e9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IPartialAggregationTypeComputer.java
@@ -0,0 +1,10 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IPartialAggregationTypeComputer {
+ public Object getType(ILogicalExpression expr, IVariableTypeEnvironment env,
+ IMetadataProvider<?, ?> metadataProvider) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java
new file mode 100644
index 0000000..f6799b0
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableEvalSizeEnvironment.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IVariableEvalSizeEnvironment {
+ // size in bytes or -1 if unknown
+ public int getVariableEvalSize(LogicalVariable var);
+
+ public void setVariableEvalSize(LogicalVariable var, int size);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
new file mode 100644
index 0000000..0827b97
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/IVariableTypeEnvironment.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IVariableTypeEnvironment {
+ public Object getVarType(LogicalVariable var) throws AlgebricksException;
+
+ public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables) throws AlgebricksException;
+
+ public void setVarType(LogicalVariable var, Object type);
+
+ public Object getType(ILogicalExpression expr) throws AlgebricksException;
+
+ public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
new file mode 100644
index 0000000..fa8838b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/ScalarFunctionCallExpression.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class ScalarFunctionCallExpression extends AbstractFunctionCallExpression {
+
+ public ScalarFunctionCallExpression(IFunctionInfo finfo) {
+ super(FunctionKind.SCALAR, finfo);
+ }
+
+ public ScalarFunctionCallExpression(IFunctionInfo finfo, List<Mutable<ILogicalExpression>> arguments) {
+ super(FunctionKind.SCALAR, finfo, arguments);
+ }
+
+ public ScalarFunctionCallExpression(IFunctionInfo finfo, Mutable<ILogicalExpression>... expressions) {
+ super(FunctionKind.SCALAR, finfo, expressions);
+ }
+
+ @Override
+ public ScalarFunctionCallExpression cloneExpression() {
+ cloneAnnotations();
+ List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
+ return new ScalarFunctionCallExpression(finfo, clonedArgs);
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitScalarFunctionCallExpression(this, arg);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
new file mode 100644
index 0000000..c5cd897
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/StatefulFunctionCallExpression.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPropertiesComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class StatefulFunctionCallExpression extends AbstractFunctionCallExpression {
+
+ private final IPropertiesComputer propertiesComputer;
+
+ public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer) {
+ super(FunctionKind.STATEFUL, finfo);
+ this.propertiesComputer = propertiesComputer;
+ }
+
+ public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer,
+ List<Mutable<ILogicalExpression>> arguments) {
+ super(FunctionKind.STATEFUL, finfo, arguments);
+ this.propertiesComputer = propertiesComputer;
+ }
+
+ public StatefulFunctionCallExpression(IFunctionInfo finfo, IPropertiesComputer propertiesComputer,
+ Mutable<ILogicalExpression>... expressions) {
+ super(FunctionKind.STATEFUL, finfo, expressions);
+ this.propertiesComputer = propertiesComputer;
+ }
+
+ @Override
+ public StatefulFunctionCallExpression cloneExpression() {
+ cloneAnnotations();
+ List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
+ return new StatefulFunctionCallExpression(finfo, propertiesComputer, clonedArgs);
+ }
+
+ // can be null
+ public IPartitioningProperty getRequiredPartitioningProperty() {
+ return propertiesComputer.computePartitioningProperty(this);
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitStatefulFunctionCallExpression(this, arg);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
new file mode 100644
index 0000000..5f3de4b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/UnnestingFunctionCallExpression.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class UnnestingFunctionCallExpression extends AbstractFunctionCallExpression {
+
+ private boolean returnsUniqueValues;
+
+ public UnnestingFunctionCallExpression(IFunctionInfo finfo) {
+ super(FunctionKind.UNNEST, finfo);
+ }
+
+ public UnnestingFunctionCallExpression(IFunctionInfo finfo, List<Mutable<ILogicalExpression>> arguments) {
+ super(FunctionKind.UNNEST, finfo, arguments);
+ }
+
+ public UnnestingFunctionCallExpression(IFunctionInfo finfo, Mutable<ILogicalExpression>... expressions) {
+ super(FunctionKind.UNNEST, finfo, expressions);
+ }
+
+ @Override
+ public UnnestingFunctionCallExpression cloneExpression() {
+ cloneAnnotations();
+ List<Mutable<ILogicalExpression>> clonedArgs = cloneArguments();
+ UnnestingFunctionCallExpression ufce = new UnnestingFunctionCallExpression(finfo, clonedArgs);
+ ufce.setReturnsUniqueValues(returnsUniqueValues);
+ return ufce;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitUnnestingFunctionCallExpression(this, arg);
+ }
+
+ public void setReturnsUniqueValues(boolean returnsUniqueValues) {
+ this.returnsUniqueValues = returnsUniqueValues;
+ }
+
+ public boolean returnsUniqueValues() {
+ return returnsUniqueValues;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
new file mode 100644
index 0000000..ecd993f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/expressions/VariableReferenceExpression.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.expressions;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class VariableReferenceExpression extends AbstractLogicalExpression {
+ private LogicalVariable variable;
+
+ public VariableReferenceExpression(LogicalVariable variable) {
+ this.variable = variable;
+ }
+
+ public LogicalVariable getVariableReference() {
+ return variable;
+ }
+
+ public void setVariable(LogicalVariable variable) {
+ this.variable = variable;
+ }
+
+ @Override
+ public LogicalExpressionTag getExpressionTag() {
+ return LogicalExpressionTag.VARIABLE;
+ }
+
+ @Override
+ public String toString() {
+ return variable.toString();
+ }
+
+ @Override
+ public void getUsedVariables(Collection<LogicalVariable> vars) {
+ // if (!vars.contains(variable)) {
+ vars.add(variable);
+ // }
+ }
+
+ @Override
+ public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
+ if (variable.equals(v1)) {
+ variable = v2;
+ }
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof VariableReferenceExpression)) {
+ return false;
+ } else {
+ return variable.equals(((VariableReferenceExpression) obj).getVariableReference());
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return variable.getId();
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalExpressionVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitVariableReferenceExpression(this, arg);
+ }
+
+ @Override
+ public AbstractLogicalExpression cloneExpression() {
+ return new VariableReferenceExpression(variable);
+ }
+
+ @Override
+ public boolean splitIntoConjuncts(List<Mutable<ILogicalExpression>> conjs) {
+ return false;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
new file mode 100644
index 0000000..7511155
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/AlgebricksBuiltinFunctions.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+public class AlgebricksBuiltinFunctions {
+ public enum ComparisonKind {
+ EQ,
+ LE,
+ GE,
+ LT,
+ GT,
+ NEQ
+ }
+
+ private static Map<FunctionIdentifier, FunctionIdentifier> algebricksBuiltinFunctions = new HashMap<FunctionIdentifier, FunctionIdentifier>();
+
+ private static Map<FunctionIdentifier, IFunctionInfo> _finfos = new HashMap<FunctionIdentifier, IFunctionInfo>();
+
+ public static final String ALGEBRICKS_NS = "algebricks";
+
+ // comparisons
+ public final static FunctionIdentifier EQ = new FunctionIdentifier(ALGEBRICKS_NS, "eq", 2, true);
+ public final static FunctionIdentifier LE = new FunctionIdentifier(ALGEBRICKS_NS, "le", 2, true);
+ public final static FunctionIdentifier GE = new FunctionIdentifier(ALGEBRICKS_NS, "ge", 2, true);
+ public final static FunctionIdentifier LT = new FunctionIdentifier(ALGEBRICKS_NS, "lt", 2, true);
+ public final static FunctionIdentifier GT = new FunctionIdentifier(ALGEBRICKS_NS, "gt", 2, true);
+ public final static FunctionIdentifier NEQ = new FunctionIdentifier(ALGEBRICKS_NS, "neq", 2, true);
+
+ // booleans
+ public final static FunctionIdentifier NOT = new FunctionIdentifier(ALGEBRICKS_NS, "not", 1, true);
+ public final static FunctionIdentifier AND = new FunctionIdentifier(ALGEBRICKS_NS, "and",
+ FunctionIdentifier.VARARGS, true);
+ public final static FunctionIdentifier OR = new FunctionIdentifier(ALGEBRICKS_NS, "or", FunctionIdentifier.VARARGS,
+ true);
+
+ // numerics
+ public final static FunctionIdentifier NUMERIC_ADD = new FunctionIdentifier(ALGEBRICKS_NS, "numeric-add", 2, true);
+
+ // nulls
+ public final static FunctionIdentifier IS_NULL = new FunctionIdentifier(ALGEBRICKS_NS, "is-null", 1, true);
+
+ static {
+ // comparisons
+ add(EQ);
+ add(LE);
+ add(GE);
+ add(LT);
+ add(GT);
+ add(NEQ);
+ // booleans
+ add(NOT);
+ add(AND);
+ add(OR);
+ // numerics
+ add(NUMERIC_ADD);
+ // nulls
+ add(IS_NULL);
+ for (FunctionIdentifier fi : algebricksBuiltinFunctions.values()) {
+ _finfos.put(fi, new FunctionInfoImpl(fi));
+ }
+ }
+
+ private static void add(FunctionIdentifier fi) {
+ algebricksBuiltinFunctions.put(fi, fi);
+ }
+
+ public static final boolean isAlgebricksBuiltinFunction(FunctionIdentifier fi) {
+ return algebricksBuiltinFunctions.get(fi) != null;
+ }
+
+ public static final Collection<FunctionIdentifier> getAlgebricksBuiltinFunctions() {
+ return algebricksBuiltinFunctions.values();
+ }
+
+ public static final FunctionIdentifier getBuiltinFunctionIdentifier(FunctionIdentifier fi) {
+ return algebricksBuiltinFunctions.get(fi);
+ }
+
+ private static final Map<FunctionIdentifier, ComparisonKind> comparisonFunctions = new HashMap<FunctionIdentifier, ComparisonKind>();
+ static {
+ comparisonFunctions.put(AlgebricksBuiltinFunctions.EQ, ComparisonKind.EQ);
+ comparisonFunctions.put(AlgebricksBuiltinFunctions.LE, ComparisonKind.LE);
+ comparisonFunctions.put(AlgebricksBuiltinFunctions.GE, ComparisonKind.GE);
+ comparisonFunctions.put(AlgebricksBuiltinFunctions.LT, ComparisonKind.LT);
+ comparisonFunctions.put(AlgebricksBuiltinFunctions.GT, ComparisonKind.GT);
+ comparisonFunctions.put(AlgebricksBuiltinFunctions.NEQ, ComparisonKind.NEQ);
+ }
+
+ public static ComparisonKind getComparisonType(FunctionIdentifier fi) {
+ return comparisonFunctions.get(fi);
+ }
+
+ public static boolean isComparisonFunction(FunctionIdentifier fi) {
+ return comparisonFunctions.get(fi) != null;
+ }
+
+ public static IFunctionInfo getBuiltinFunctionInfo(FunctionIdentifier fi) {
+ return _finfos.get(fi);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
new file mode 100644
index 0000000..252f23c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionIdentifier.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+
+public final class FunctionIdentifier {
+ final private String namespace;
+ final private String name;
+ final private boolean isBuiltin;
+ final private int arity;
+
+ public final static int VARARGS = -1;
+
+ public FunctionIdentifier(String namespace, String name, boolean isBuiltin) {
+ this.namespace = namespace;
+ this.name = name;
+ this.arity = VARARGS;
+ this.isBuiltin = isBuiltin;
+ }
+
+ public FunctionIdentifier(String namespace, String name, int arity, boolean isBuiltin) {
+ this.namespace = namespace;
+ this.name = name;
+ this.arity = arity;
+ this.isBuiltin = isBuiltin;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public boolean isBuiltin() {
+ return isBuiltin;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (super.equals(o)) {
+ return true;
+ }
+ if (o instanceof FunctionIdentifier) {
+ FunctionIdentifier ofi = (FunctionIdentifier) o;
+ return ofi.getNamespace().equals(getNamespace()) && ofi.name.equals(name);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode() + namespace.hashCode();
+ }
+
+ public String toString() {
+ return getNamespace() + ":" + name;
+ }
+
+ public int getArity() {
+ return arity;
+ }
+
+ public String getNamespace() {
+ return namespace;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionInfoImpl.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionInfoImpl.java
new file mode 100644
index 0000000..15521d6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/FunctionInfoImpl.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+
+public class FunctionInfoImpl implements IFunctionInfo {
+
+ private final FunctionIdentifier fid;
+
+ public FunctionInfoImpl(FunctionIdentifier fid) {
+ this.fid = fid;
+ }
+
+ @Override
+ public FunctionIdentifier getFunctionIdentifier() {
+ return fid;
+ }
+
+ @Override
+ public Object getInfo() {
+ throw new IllegalStateException();
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
new file mode 100644
index 0000000..57b7262
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/functions/IFunctionInfo.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.functions;
+
+public interface IFunctionInfo {
+ FunctionIdentifier getFunctionIdentifier();
+
+ Object getInfo();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSink.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSink.java
new file mode 100644
index 0000000..00b3554
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSink.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+
+public interface IDataSink {
+ public Object getId();
+
+ public Object[] getSchemaTypes();
+
+ public IPartitioningProperty getPartitioningProperty();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSource.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSource.java
new file mode 100644
index 0000000..22f9feb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSource.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+
+public interface IDataSource<T> {
+ public T getId();
+
+ public Object[] getSchemaTypes();
+
+ public IDataSourcePropertiesProvider getPropertiesProvider();
+
+ public void computeFDs(List<LogicalVariable> scanVariables, List<FunctionalDependency> fdList);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java
new file mode 100644
index 0000000..90c8536
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourceIndex.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+public interface IDataSourceIndex<I, T> {
+ public I getId();
+
+ public IDataSource<T> getDataSource();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java
new file mode 100644
index 0000000..500b968
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IDataSourcePropertiesProvider.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+
+public interface IDataSourcePropertiesProvider {
+ public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
new file mode 100644
index 0000000..221992e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.metadata;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public interface IMetadataProvider<S, I> {
+ public IDataSource<S> findDataSource(S id) throws AlgebricksException;
+
+ /**
+ * Obs: A scanner may choose to contribute a null
+ * AlgebricksPartitionConstraint and implement
+ * contributeSchedulingConstraints instead.
+ */
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<S> dataSource,
+ List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+ JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
+
+ public boolean scannerOperatorIsLeaf(IDataSource<S> dataSource);
+
+ public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
+ int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc)
+ throws AlgebricksException;
+
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(IDataSource<S> dataSource,
+ IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
+ JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
+
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<S> dataSource,
+ IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
+ RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
+
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
+ IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
+ RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
+
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
+ IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, List<LogicalVariable> primaryKeys,
+ List<LogicalVariable> secondaryKeys, RecordDescriptor recordDesc, JobGenContext context,
+ JobSpecification spec) throws AlgebricksException;
+
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
+ IDataSourceIndex<I, S> dataSource, IOperatorSchema propagatedSchema, List<LogicalVariable> primaryKeys,
+ List<LogicalVariable> secondaryKeys, RecordDescriptor recordDesc, JobGenContext context,
+ JobSpecification spec) throws AlgebricksException;
+
+ public IDataSourceIndex<I, S> findDataSourceIndex(I indexId, S dataSourceId) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
new file mode 100644
index 0000000..456010a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractAssignOperator.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+/**
+ * @author Nicola
+ *
+ */
+public abstract class AbstractAssignOperator extends AbstractLogicalOperator {
+ protected final List<LogicalVariable> variables;
+ protected final List<Mutable<ILogicalExpression>> expressions;
+
+ public AbstractAssignOperator() {
+ this.variables = new ArrayList<LogicalVariable>();
+ this.expressions = new ArrayList<Mutable<ILogicalExpression>>();
+ }
+
+ public AbstractAssignOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
+ this.variables = variables;
+ this.expressions = expressions;
+ }
+
+ public List<LogicalVariable> getVariables() {
+ return variables;
+ }
+
+ public List<Mutable<ILogicalExpression>> getExpressions() {
+ return expressions;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ schema.addAll(variables);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean modif = false;
+ for (int i = 0; i < expressions.size(); i++) {
+ if (visitor.transform(expressions.get(i))) {
+ modif = true;
+ }
+ }
+ return modif;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
new file mode 100644
index 0000000..c2911d4
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractBinaryJoinOperator.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public abstract class AbstractBinaryJoinOperator extends AbstractLogicalOperator {
+ protected final Mutable<ILogicalExpression> condition;
+ protected JoinKind joinKind;
+
+ public enum JoinKind {
+ INNER,
+ LEFT_OUTER
+ }
+
+ public AbstractBinaryJoinOperator(JoinKind joinKind, Mutable<ILogicalExpression> condition) {
+ this.joinKind = joinKind;
+ this.condition = condition;
+ }
+
+ public AbstractBinaryJoinOperator(JoinKind joinKind, Mutable<ILogicalExpression> condition,
+ Mutable<ILogicalOperator> input1, Mutable<ILogicalOperator> input2) {
+ this(joinKind, condition);
+ inputs.add(input1);
+ inputs.add(input2);
+ }
+
+ public Mutable<ILogicalExpression> getCondition() {
+ return condition;
+ }
+
+ public JoinKind getJoinKind() {
+ return joinKind;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ schema.addAll(inputs.get(1).getValue().getSchema());
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.concat(VariablePropagationPolicy.ALL, VariablePropagationPolicy.ALL);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ return visitor.transform(condition);
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
new file mode 100644
index 0000000..88fd0d5
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
@@ -0,0 +1,194 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public abstract class AbstractLogicalOperator implements ILogicalOperator {
+
+ /*********************************************************************
+ * UNPARTITIONED, the input data is not partitioned
+ * PARTITIONED, the input data is partitioned, the operator is executed on
+ * each partition and may receive input from other partitions (e.g. if it is
+ * a join or an aggregate)
+ * LOCAL, the input data is partitioned, the operator is executed on each
+ * partition and only processes data from that partition
+ */
+
+ public static enum ExecutionMode {
+ UNPARTITIONED,
+ PARTITIONED,
+ LOCAL
+ }
+
+ private AbstractLogicalOperator.ExecutionMode mode = AbstractLogicalOperator.ExecutionMode.UNPARTITIONED;
+ protected IPhysicalOperator physicalOperator;
+ private final Map<String, Object> annotations = new HashMap<String, Object>();
+ private boolean bJobGenEnabled = true;
+ private Object hostQueryContext;
+
+ final protected List<Mutable<ILogicalOperator>> inputs;
+ // protected List<LogicalOperatorReference> outputs;
+ protected List<LogicalVariable> schema;
+
+ public AbstractLogicalOperator() {
+ inputs = new ArrayList<Mutable<ILogicalOperator>>();
+ // outputs = new ArrayList<LogicalOperatorReference>();
+ }
+
+ public abstract LogicalOperatorTag getOperatorTag();
+
+ public ExecutionMode getExecutionMode() {
+ return mode;
+ }
+
+ public void setExecutionMode(ExecutionMode mode) {
+ this.mode = mode;
+ }
+
+ @Override
+ public List<LogicalVariable> getSchema() {
+ return schema;
+ }
+
+ public void setPhysicalOperator(IPhysicalOperator physicalOp) {
+ this.physicalOperator = physicalOp;
+ }
+
+ public IPhysicalOperator getPhysicalOperator() {
+ return physicalOperator;
+ }
+
+ /**
+ * @return for each child, one vector of required physical properties
+ */
+
+ @Override
+ public final PhysicalRequirements getRequiredPhysicalPropertiesForChildren(
+ IPhysicalPropertiesVector requiredProperties) {
+ return physicalOperator.getRequiredPropertiesForChildren(this, requiredProperties);
+ }
+
+ /**
+ * @return the physical properties that this operator delivers, based on
+ * what its children deliver
+ */
+
+ @Override
+ public final IPhysicalPropertiesVector getDeliveredPhysicalProperties() {
+ return physicalOperator.getDeliveredProperties();
+ }
+
+ @Override
+ public final void computeDeliveredPhysicalProperties(IOptimizationContext context) throws AlgebricksException {
+ physicalOperator.computeDeliveredProperties(this, context);
+ }
+
+ @Override
+ public final List<Mutable<ILogicalOperator>> getInputs() {
+ return inputs;
+ }
+
+ // @Override
+ // public final List<LogicalOperatorReference> getOutputs() {
+ // return outputs;
+ // }
+
+ @Override
+ public final boolean hasInputs() {
+ return !inputs.isEmpty();
+ }
+
+ public boolean hasNestedPlans() {
+ return false;
+ }
+
+ @Override
+ public Map<String, Object> getAnnotations() {
+ return annotations;
+ }
+
+ @Override
+ public void removeAnnotation(String annotationName) {
+ annotations.remove(annotationName);
+ }
+
+ @Override
+ public final void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ if (bJobGenEnabled) {
+ if (physicalOperator == null) {
+ throw new AlgebricksException("Physical operator not set for operator: " + this);
+ }
+ physicalOperator.contributeRuntimeOperator(builder, context, this, propagatedSchema, inputSchemas,
+ outerPlanSchema);
+ }
+ }
+
+ public void disableJobGen() {
+ bJobGenEnabled = false;
+ }
+
+ public boolean isJobGenEnabled() {
+ return bJobGenEnabled;
+ }
+
+ public void setHostQueryContext(Object context) {
+ this.hostQueryContext = context;
+ }
+
+ public Object getHostQueryContext() {
+ return hostQueryContext;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+ protected IVariableTypeEnvironment createPropagatingAllInputsTypeEnvironment(ITypingContext ctx) {
+ // return createPropagatingAllInputsTypeEnvironment(ctx);
+ int n = inputs.size();
+ ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
+ for (int i = 0; i < n; i++) {
+ envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx);
+ }
+ return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
+ ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
new file mode 100644
index 0000000..ea4f3e0
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractOperatorWithNestedPlans.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public abstract class AbstractOperatorWithNestedPlans extends AbstractLogicalOperator {
+ protected final List<ILogicalPlan> nestedPlans;
+
+ public AbstractOperatorWithNestedPlans() {
+ nestedPlans = new ArrayList<ILogicalPlan>();
+ }
+
+ public AbstractOperatorWithNestedPlans(List<ILogicalPlan> nestedPlans) {
+ this.nestedPlans = nestedPlans;
+ }
+
+ public List<ILogicalPlan> getNestedPlans() {
+ return nestedPlans;
+ }
+
+ @Override
+ public boolean hasNestedPlans() {
+ return true;
+ }
+
+ public LinkedList<Mutable<ILogicalOperator>> allRootsInReverseOrder() {
+ LinkedList<Mutable<ILogicalOperator>> allRoots = new LinkedList<Mutable<ILogicalOperator>>();
+ for (ILogicalPlan p : nestedPlans) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ allRoots.addFirst(r);
+ }
+ }
+ return allRoots;
+ }
+
+ //
+ // @Override
+ // public void computeConstraintsAndEquivClasses() {
+ // for (ILogicalPlan p : nestedPlans) {
+ // for (LogicalOperatorReference r : p.getRoots()) {
+ // AbstractLogicalOperator op = (AbstractLogicalOperator) r.getOperator();
+ // equivalenceClasses.putAll(op.getEquivalenceClasses());
+ // functionalDependencies.addAll(op.getFDs());
+ // }
+ // }
+ // }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ for (ILogicalPlan p : nestedPlans) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ schema.addAll(r.getValue().getSchema());
+ }
+ }
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ public abstract void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars);
+
+ public abstract void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars);
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
new file mode 100644
index 0000000..e4e7ec8
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractScanOperator.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public abstract class AbstractScanOperator extends AbstractLogicalOperator {
+ protected List<LogicalVariable> variables;
+
+ public AbstractScanOperator(List<LogicalVariable> variables) {
+ this.variables = variables;
+ }
+
+ public List<LogicalVariable> getVariables() {
+ return variables;
+ }
+
+ public void setVariables(List<LogicalVariable> variables) {
+ this.variables = variables;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ schema.addAll(variables);
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new VariablePropagationPolicy() {
+
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ if (sources.length > 0) {
+ target.addAllVariables(sources[0]);
+ }
+ for (LogicalVariable v : variables) {
+ target.addVariable(v);
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
new file mode 100644
index 0000000..22f73ff
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AbstractUnnestOperator.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public abstract class AbstractUnnestOperator extends AbstractScanOperator {
+
+ protected final Mutable<ILogicalExpression> expression;
+
+ public AbstractUnnestOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression) {
+ super(variables);
+ this.expression = expression;
+ }
+
+ public Mutable<ILogicalExpression> getExpressionRef() {
+ return expression;
+ }
+
+ @Override
+ public boolean isMap() {
+ return true;
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ return visitor.transform(expression);
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
new file mode 100644
index 0000000..1555a76
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AggregateOperator.java
@@ -0,0 +1,84 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class AggregateOperator extends AbstractAssignOperator {
+
+ // private ArrayList<AggregateFunctionCallExpression> expressions;
+ // TODO type safe list of expressions
+ private List<Mutable<ILogicalExpression>> mergeExpressions;
+
+ public AggregateOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
+ super(variables, expressions);
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.AGGREGATE;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitAggregateOperator(this, arg);
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new VariablePropagationPolicy() {
+
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ for (LogicalVariable v : variables) {
+ target.addVariable(v);
+ }
+ }
+ };
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(variables);
+ }
+
+ public void setMergeExpressions(List<Mutable<ILogicalExpression>> merges) {
+ mergeExpressions = merges;
+ }
+
+ public List<Mutable<ILogicalExpression>> getMergeExpressions() {
+ return mergeExpressions;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+ ctx.getMetadataProvider());
+ IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(inputs.get(0).getValue());
+ int n = variables.size();
+ for (int i = 0; i < n; i++) {
+ Object t = ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(),
+ ctx.getMetadataProvider(), env2);
+ env.setVarType(variables.get(i), t);
+ }
+ return env;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
new file mode 100644
index 0000000..eedd3a2
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/AssignOperator.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+/**
+ *
+ * It corresponds to the Map operator in other algebras.
+ *
+ * @author Nicola
+ *
+ */
+
+public class AssignOperator extends AbstractAssignOperator {
+
+ public AssignOperator(List<LogicalVariable> vars, List<Mutable<ILogicalExpression>> exprs) {
+ super(vars, exprs);
+ }
+
+ public AssignOperator(LogicalVariable var, Mutable<ILogicalExpression> expr) {
+ super();
+ this.variables.add(var);
+ this.expressions.add(expr);
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.ASSIGN;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitAssignOperator(this, arg);
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new VariablePropagationPolicy() {
+
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ target.addAllVariables(sources[0]);
+ for (LogicalVariable v : variables) {
+ target.addVariable(v);
+ }
+ }
+ };
+
+ }
+
+ @Override
+ public boolean isMap() {
+ return true;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
+ int n = variables.size();
+ for (int i = 0; i < n; i++) {
+ env.setVarType(
+ variables.get(i),
+ ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(), ctx.getMetadataProvider(),
+ env));
+ }
+ return env;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
new file mode 100644
index 0000000..bcc1192
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DataSourceScanOperator.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class DataSourceScanOperator extends AbstractScanOperator {
+ private IDataSource<?> dataSource;
+
+ private List<LogicalVariable> projectVars;
+
+ private boolean projectPushed = false;
+
+ public DataSourceScanOperator(List<LogicalVariable> variables, IDataSource<?> dataSource) {
+ super(variables);
+ this.dataSource = dataSource;
+ projectVars = new ArrayList<LogicalVariable>();
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.DATASOURCESCAN;
+ }
+
+ public IDataSource<?> getDataSource() {
+ return dataSource;
+ }
+
+ @Override
+ public <R, S> R accept(ILogicalOperatorVisitor<R, S> visitor, S arg) throws AlgebricksException {
+ return visitor.visitDataScanOperator(this, arg);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ public void addProjectVariables(Collection<LogicalVariable> vars) {
+ projectVars.addAll(vars);
+ projectPushed = true;
+ }
+
+ public List<LogicalVariable> getProjectVariables() {
+ return projectVars;
+ }
+
+ public boolean isProjectPushed() {
+ return projectPushed;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new VariablePropagationPolicy() {
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ if (sources.length > 0) {
+ target.addAllVariables(sources[0]);
+ }
+ List<LogicalVariable> outputVariables = projectPushed ? projectVars : variables;
+ for (LogicalVariable v : outputVariables) {
+ target.addVariable(v);
+ }
+ }
+ };
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
+ Object[] types = dataSource.getSchemaTypes();
+ int i = 0;
+ for (LogicalVariable v : variables) {
+ env.setVarType(v, types[i]);
+ ++i;
+ }
+ return env;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java
new file mode 100644
index 0000000..2e330cf
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DieOperator.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class DieOperator extends AbstractLogicalOperator {
+
+ private final Mutable<ILogicalExpression> afterObjects; // mandatory
+
+ public DieOperator(ILogicalExpression maxObjectsExpr) {
+ this.afterObjects = new MutableObject<ILogicalExpression>(maxObjectsExpr);
+ }
+
+ public Mutable<ILogicalExpression> getAfterObjects() {
+ return afterObjects;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitDieOperator(this, arg);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean b = false;
+ if (visitor.transform(afterObjects)) {
+ b = true;
+ }
+ return b;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.DIE;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean isMap() {
+ return true;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
new file mode 100644
index 0000000..4d479e1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/DistinctOperator.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class DistinctOperator extends AbstractLogicalOperator {
+ private final List<Mutable<ILogicalExpression>> expressions;
+
+ public DistinctOperator(List<Mutable<ILogicalExpression>> expressions) {
+ this.expressions = expressions;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.DISTINCT;
+ }
+
+ public List<Mutable<ILogicalExpression>> getExpressions() {
+ return expressions;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean changed = false;
+ for (Mutable<ILogicalExpression> e : expressions) {
+ if (visitor.transform(e)) {
+ changed = true;
+ }
+ }
+ return changed;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitDistinctOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ public List<LogicalVariable> getDistinctByVarList() {
+ List<LogicalVariable> varList = new ArrayList<LogicalVariable>(expressions.size());
+ for (Mutable<ILogicalExpression> eRef : expressions) {
+ ILogicalExpression e = eRef.getValue();
+ if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression v = (VariableReferenceExpression) e;
+ varList.add(v.getVariableReference());
+ }
+ }
+ return varList;
+ }
+
+ public boolean isDistinctByVar(LogicalVariable var) {
+ for (Mutable<ILogicalExpression> eRef : expressions) {
+ ILogicalExpression e = eRef.getValue();
+ if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression v = (VariableReferenceExpression) e;
+ if (v.getVariableReference() == var) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
new file mode 100644
index 0000000..732dfbe
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/EmptyTupleSourceOperator.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class EmptyTupleSourceOperator extends AbstractLogicalOperator {
+
+ // public final static EmptyTupleSourceOperator INSTANCE = new
+ // EmptyTupleSourceOperator();
+
+ public EmptyTupleSourceOperator() {
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.EMPTYTUPLESOURCE;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.NONE;
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+ // do nothing
+ return false;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitEmptyTupleSourceOperator(this, arg);
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException {
+ return new IVariableTypeEnvironment() {
+
+ @Override
+ public void setVarType(LogicalVariable var, Object type) {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Object getType(ILogicalExpression expr) throws AlgebricksException {
+ return ctx.getExpressionTypeComputer().getType(expr, ctx.getMetadataProvider(), this);
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables)
+ throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2)
+ throws AlgebricksException {
+ return false;
+ }
+ };
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
new file mode 100644
index 0000000..3e046b5
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ExchangeOperator.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class ExchangeOperator extends AbstractLogicalOperator {
+
+ public ExchangeOperator() {
+ super();
+ setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitExchangeOperator(this, arg);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+ // do nothing
+ return false;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.EXCHANGE;
+ }
+
+ @Override
+ public void recomputeSchema() throws AlgebricksException {
+ AbstractLogicalOperator cld = (AbstractLogicalOperator) inputs.get(0).getValue();
+ OperatorPropertiesUtil.computeSchemaRecIfNull(cld);
+ List<LogicalVariable> inputSchema = inputs.get(0).getValue().getSchema();
+ schema = new ArrayList<LogicalVariable>(inputSchema);
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean isMap() {
+ return true;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
new file mode 100644
index 0000000..762f034
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/GroupByOperator.java
@@ -0,0 +1,277 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class GroupByOperator extends AbstractOperatorWithNestedPlans {
+ // If the LogicalVariable in a pair is null, it means that the GroupBy is
+ // only grouping by the expression, without producing a new variable.
+ private final List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList;
+ private final List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList;
+
+ // In decorList, if the variable (first member of the pair) is null, the
+ // second member of the pair is variable reference which is propagated.
+
+ public GroupByOperator() {
+ super();
+ gByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+ decorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+ }
+
+ public GroupByOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList,
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList, List<ILogicalPlan> nestedPlans) {
+ super(nestedPlans);
+ this.decorList = decorList;
+ this.gByList = groupByList;
+ }
+
+ public void addGbyExpression(LogicalVariable variable, ILogicalExpression expression) {
+ this.gByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable,
+ new MutableObject<ILogicalExpression>(expression)));
+ }
+
+ public void addDecorExpression(LogicalVariable variable, ILogicalExpression expression) {
+ this.decorList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(variable,
+ new MutableObject<ILogicalExpression>(expression)));
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.GROUP;
+ }
+
+ public List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> getGroupByList() {
+ return gByList;
+ }
+
+ public String gByListToString() {
+ return veListToString(gByList);
+ }
+
+ public String decorListToString() {
+ return veListToString(decorList);
+ }
+
+ public List<LogicalVariable> getGbyVarList() {
+ List<LogicalVariable> varList = new ArrayList<LogicalVariable>(gByList.size());
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gByList) {
+ ILogicalExpression expr = ve.second.getValue();
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression v = (VariableReferenceExpression) expr;
+ varList.add(v.getVariableReference());
+ }
+ }
+ return varList;
+ }
+
+ public static String veListToString(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairList) {
+ StringBuilder sb = new StringBuilder();
+ sb.append("[");
+ boolean fst = true;
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : vePairList) {
+ if (fst) {
+ fst = false;
+ } else {
+ sb.append("; ");
+ }
+ if (ve.first != null) {
+ sb.append(ve.first + " := " + ve.second);
+ } else {
+ sb.append(ve.second.getValue());
+ }
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ @Override
+ public void recomputeSchema() {
+ super.recomputeSchema();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+ schema.add(p.first);
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+ schema.add(getDecorVariable(p));
+ }
+ }
+
+ @Override
+ public void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
+ // super.getProducedVariables(vars);
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+ if (p.first != null) {
+ vars.add(p.first);
+ }
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+ if (p.first != null) {
+ vars.add(p.first);
+ }
+ }
+ }
+
+ @Override
+ public void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : gByList) {
+ g.second.getValue().getUsedVariables(vars);
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : decorList) {
+ g.second.getValue().getUsedVariables(vars);
+ }
+ // super.getUsedVariables(vars);
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new VariablePropagationPolicy() {
+
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+ ILogicalExpression expr = p.second.getValue();
+ if (p.first != null) {
+ target.addVariable(p.first);
+ } else {
+ if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new AlgebricksException("hash group-by expects variable references.");
+ }
+ VariableReferenceExpression v = (VariableReferenceExpression) expr;
+ target.addVariable(v.getVariableReference());
+ }
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+ ILogicalExpression expr = p.second.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new AlgebricksException("pre-sorted group-by expects variable references.");
+ }
+ VariableReferenceExpression v = (VariableReferenceExpression) expr;
+ LogicalVariable decor = v.getVariableReference();
+ if (p.first != null) {
+ target.addVariable(p.first);
+ } else {
+ target.addVariable(decor);
+ }
+ }
+
+ }
+ };
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean b = false;
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+ if (visitor.transform(p.second)) {
+ b = true;
+ }
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+ if (visitor.transform(p.second)) {
+ b = true;
+ }
+ }
+ return b;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitGroupByOperator(this, arg);
+ }
+
+ public static LogicalVariable getDecorVariable(Pair<LogicalVariable, Mutable<ILogicalExpression>> p) {
+ if (p.first != null) {
+ return p.first;
+ } else {
+ VariableReferenceExpression e = (VariableReferenceExpression) p.second.getValue();
+ return e.getVariableReference();
+ }
+ }
+
+ public List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> getDecorList() {
+ return decorList;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ int n = 0;
+ for (ILogicalPlan p : nestedPlans) {
+ n += p.getRoots().size();
+ }
+ ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
+ int i = 0;
+ for (ILogicalPlan p : nestedPlans) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ envPointers[i] = new OpRefTypeEnvPointer(r, ctx);
+ i++;
+ }
+ }
+ IVariableTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+ ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
+ ILogicalOperator child = inputs.get(0).getValue();
+ IVariableTypeEnvironment env2 = ctx.getOutputTypeEnvironment(child);
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : getGroupByList()) {
+ ILogicalExpression expr = p.second.getValue();
+ if (p.first != null) {
+ env.setVarType(p.first, env2.getType(expr));
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ LogicalVariable v1 = ((VariableReferenceExpression) expr).getVariableReference();
+ env.setVarType(v1, env2.getVarType(v1));
+ }
+ } else {
+ VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getValue();
+ LogicalVariable v2 = vre.getVariableReference();
+ env.setVarType(v2, env2.getVarType(v2));
+ }
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : getDecorList()) {
+ ILogicalExpression expr = p.second.getValue();
+ if (p.first != null) {
+ env.setVarType(p.first, env2.getType(expr));
+ } else {
+ VariableReferenceExpression vre = (VariableReferenceExpression) p.second.getValue();
+ LogicalVariable v2 = vre.getVariableReference();
+ env.setVarType(v2, env2.getVarType(v2));
+ }
+ }
+ return env;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java
new file mode 100644
index 0000000..043ac8e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IOperatorSchema.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IOperatorSchema extends Iterable<LogicalVariable> {
+ public void addAllVariables(IOperatorSchema source);
+
+ public void addAllNewVariables(IOperatorSchema source);
+
+ public int addVariable(LogicalVariable var);
+
+ public int findVariable(LogicalVariable var);
+
+ public LogicalVariable getVariable(int index);
+
+ public int getSize();
+
+ public void clear();
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
new file mode 100644
index 0000000..7264763
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/IndexInsertDeleteOperator.java
@@ -0,0 +1,99 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class IndexInsertDeleteOperator extends AbstractLogicalOperator {
+
+ private final IDataSourceIndex<?, ?> dataSourceIndex;
+ private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
+ private final List<Mutable<ILogicalExpression>> secondaryKeyExprs;
+ private final Kind operation;
+
+ public IndexInsertDeleteOperator(IDataSourceIndex<?, ?> dataSourceIndex,
+ List<Mutable<ILogicalExpression>> primaryKeyExprs, List<Mutable<ILogicalExpression>> secondaryKeyExprs,
+ Kind operation) {
+ this.dataSourceIndex = dataSourceIndex;
+ this.primaryKeyExprs = primaryKeyExprs;
+ this.secondaryKeyExprs = secondaryKeyExprs;
+ this.operation = operation;
+ }
+
+ @Override
+ public void recomputeSchema() throws AlgebricksException {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean b = false;
+ for (int i = 0; i < primaryKeyExprs.size(); i++) {
+ if (visitor.transform(primaryKeyExprs.get(i))) {
+ b = true;
+ }
+ }
+ for (int i = 0; i < secondaryKeyExprs.size(); i++) {
+ if (visitor.transform(secondaryKeyExprs.get(i))) {
+ b = true;
+ }
+ }
+ return b;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitIndexInsertDeleteOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.INDEX_INSERT_DELETE;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+ public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
+ return primaryKeyExprs;
+ }
+
+ public IDataSourceIndex<?, ?> getDataSourceIndex() {
+ return dataSourceIndex;
+ }
+
+ public List<Mutable<ILogicalExpression>> getSecondaryKeyExpressions() {
+ return secondaryKeyExprs;
+ }
+
+ public Kind getOperation() {
+ return operation;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
new file mode 100644
index 0000000..c2f0161
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InnerJoinOperator.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class InnerJoinOperator extends AbstractBinaryJoinOperator {
+
+ public InnerJoinOperator(Mutable<ILogicalExpression> condition) {
+ super(JoinKind.INNER, condition);
+ }
+
+ public InnerJoinOperator(Mutable<ILogicalExpression> condition, Mutable<ILogicalOperator> input1,
+ Mutable<ILogicalOperator> input2) {
+ super(JoinKind.INNER, condition, input1, input2);
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.INNERJOIN;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitInnerJoinOperator(this, arg);
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
new file mode 100644
index 0000000..6699228
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/InsertDeleteOperator.java
@@ -0,0 +1,98 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class InsertDeleteOperator extends AbstractLogicalOperator {
+
+ public enum Kind {
+ INSERT,
+ DELETE
+ }
+
+ private final IDataSource<?> dataSource;
+ private final Mutable<ILogicalExpression> payloadExpr;
+ private final List<Mutable<ILogicalExpression>> primaryKeyExprs;
+ private final Kind operation;
+
+ public InsertDeleteOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payload,
+ List<Mutable<ILogicalExpression>> primaryKeyExprs, Kind operation) {
+ this.dataSource = dataSource;
+ this.payloadExpr = payload;
+ this.primaryKeyExprs = primaryKeyExprs;
+ this.operation = operation;
+ }
+
+ @Override
+ public void recomputeSchema() throws AlgebricksException {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean b = false;
+ b = visitor.transform(payloadExpr);
+ for (int i = 0; i < primaryKeyExprs.size(); i++) {
+ if (visitor.transform(primaryKeyExprs.get(i))) {
+ b = true;
+ }
+ }
+ return b;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitInsertDeleteOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.INSERT_DELETE;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+ public List<Mutable<ILogicalExpression>> getPrimaryKeyExpressions() {
+ return primaryKeyExprs;
+ }
+
+ public IDataSource<?> getDataSource() {
+ return dataSource;
+ }
+
+ public Mutable<ILogicalExpression> getPayloadExpression() {
+ return payloadExpr;
+ }
+
+ public Kind getOperation() {
+ return operation;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
new file mode 100644
index 0000000..7c5be7e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LeftOuterJoinOperator.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class LeftOuterJoinOperator extends AbstractBinaryJoinOperator {
+
+ public LeftOuterJoinOperator(Mutable<ILogicalExpression> condition) {
+ super(JoinKind.LEFT_OUTER, condition);
+ }
+
+ public LeftOuterJoinOperator(Mutable<ILogicalExpression> condition, Mutable<ILogicalOperator> input1,
+ Mutable<ILogicalOperator> input2) {
+ super(JoinKind.LEFT_OUTER, condition, input1, input2);
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.LEFTOUTERJOIN;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitLeftOuterJoinOperator(this, arg);
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ int n = inputs.size();
+ ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n];
+ for (int i = 0; i < n; i++) {
+ envPointers[i] = new OpRefTypeEnvPointer(inputs.get(i), ctx);
+ }
+ return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
+ ctx.getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
new file mode 100644
index 0000000..21738f6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/LimitOperator.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class LimitOperator extends AbstractLogicalOperator {
+
+ private final Mutable<ILogicalExpression> maxObjects; // mandatory
+ private final Mutable<ILogicalExpression> offset; // optional
+ private boolean topmost;
+
+ public LimitOperator(ILogicalExpression maxObjectsExpr, ILogicalExpression offsetExpr, boolean topmost) {
+ this.maxObjects = new MutableObject<ILogicalExpression>(maxObjectsExpr);
+ this.offset = new MutableObject<ILogicalExpression>(offsetExpr);
+ this.topmost = topmost;
+ }
+
+ public LimitOperator(ILogicalExpression maxObjectsExpr, boolean topmost) {
+ this(maxObjectsExpr, null, topmost);
+ }
+
+ public LimitOperator(ILogicalExpression maxObjects, ILogicalExpression offset) {
+ this(maxObjects, offset, true);
+ }
+
+ public LimitOperator(ILogicalExpression maxObjects) {
+ this(maxObjects, null, true);
+ }
+
+ public Mutable<ILogicalExpression> getMaxObjects() {
+ return maxObjects;
+ }
+
+ public Mutable<ILogicalExpression> getOffset() {
+ return offset;
+ }
+
+ public boolean isTopmostLimitOp() {
+ return topmost;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitLimitOperator(this, arg);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean b = false;
+ if (visitor.transform(maxObjects)) {
+ b = true;
+ }
+ if (offset.getValue() != null) {
+ if (visitor.transform(offset)) {
+ b = true;
+ }
+ }
+ return b;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.LIMIT;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean isMap() {
+ return true;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
new file mode 100644
index 0000000..189a410
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/NestedTupleSourceOperator.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class NestedTupleSourceOperator extends AbstractLogicalOperator {
+ private final Mutable<ILogicalOperator> dataSourceReference;
+
+ public NestedTupleSourceOperator(Mutable<ILogicalOperator> dataSourceReference) {
+ this.dataSourceReference = dataSourceReference;
+ }
+
+ public ILogicalOperator getSourceOperator() {
+ return dataSourceReference.getValue().getInputs().get(0).getValue();
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.NESTEDTUPLESOURCE;
+ }
+
+ public Mutable<ILogicalOperator> getDataSourceReference() {
+ return dataSourceReference;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ ILogicalOperator topOp = dataSourceReference.getValue();
+ for (Mutable<ILogicalOperator> i : topOp.getInputs()) {
+ schema.addAll(i.getValue().getSchema());
+ }
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+ // do nothing
+ return false;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitNestedTupleSourceOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(final ITypingContext ctx) throws AlgebricksException {
+ ITypeEnvPointer[] p = new ITypeEnvPointer[1];
+ p[0] = new ITypeEnvPointer() {
+
+ @Override
+ public IVariableTypeEnvironment getTypeEnv() {
+ ILogicalOperator op = dataSourceReference.getValue().getInputs().get(0).getValue();
+ return ctx.getOutputTypeEnvironment(op);
+ }
+ };
+ return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
+ ctx.getMetadataProvider(), TypePropagationPolicy.ALL, p);
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeInputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return computeOutputTypeEnvironment(ctx);
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
new file mode 100644
index 0000000..3a5cdde
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/OrderOperator.java
@@ -0,0 +1,159 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class OrderOperator extends AbstractLogicalOperator {
+
+ public interface IOrder {
+ public enum OrderKind {
+ FUNCTIONCALL,
+ ASC,
+ DESC
+ };
+
+ public Mutable<ILogicalExpression> getExpressionRef();
+
+ public OrderKind getKind();
+ }
+
+ public static IOrder ASC_ORDER = new IOrder() {
+
+ @Override
+ public Mutable<ILogicalExpression> getExpressionRef() {
+ return null;
+ }
+
+ @Override
+ public OrderKind getKind() {
+ return OrderKind.ASC;
+ }
+
+ };
+
+ public static IOrder DESC_ORDER = new IOrder() {
+
+ @Override
+ public Mutable<ILogicalExpression> getExpressionRef() {
+ return null;
+ }
+
+ @Override
+ public OrderKind getKind() {
+ return OrderKind.DESC;
+ }
+ };
+
+ public class FunOrder implements IOrder {
+ private final Mutable<ILogicalExpression> f;
+
+ public FunOrder(Mutable<ILogicalExpression> f) {
+ this.f = f;
+ }
+
+ @Override
+ public Mutable<ILogicalExpression> getExpressionRef() {
+ return f;
+ }
+
+ @Override
+ public OrderKind getKind() {
+ return OrderKind.FUNCTIONCALL;
+ }
+
+ };
+
+ private final List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExpressions;
+
+ // These are pairs of type (comparison, expr) where comparison is
+ // ASC or DESC or a boolean function of arity 2 that can take as
+ // arguments results of expr.
+
+ public OrderOperator() {
+ orderExpressions = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
+ }
+
+ public OrderOperator(List<Pair<IOrder, Mutable<ILogicalExpression>>> orderExpressions) {
+ this.orderExpressions = orderExpressions;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.ORDER;
+ }
+
+ public List<Pair<IOrder, Mutable<ILogicalExpression>>> getOrderExpressions() {
+ return orderExpressions;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean b = false;
+ for (Pair<IOrder, Mutable<ILogicalExpression>> p : orderExpressions) {
+ if (p.first.getKind() == OrderKind.FUNCTIONCALL) {
+ FunOrder fo = (FunOrder) p.first;
+ Mutable<ILogicalExpression> r1 = fo.getExpressionRef();
+ if (visitor.transform(r1)) {
+ b = true;
+ }
+ }
+ if (visitor.transform(p.second)) {
+ b = true;
+ }
+ }
+ return b;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitOrderOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
new file mode 100644
index 0000000..a76237b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/PartitioningSplitOperator.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class PartitioningSplitOperator extends AbstractLogicalOperator {
+
+ private Mutable<ILogicalExpression>[] expressions;
+ private boolean hasDefault;
+
+ public PartitioningSplitOperator(Mutable<ILogicalExpression>[] exprList, boolean defaultBranchDefined) {
+ expressions = exprList;
+ hasDefault = defaultBranchDefined;
+ }
+
+ public Mutable<ILogicalExpression>[] getExpressions() {
+ return expressions;
+ }
+
+ public boolean hasDefault() {
+ return hasDefault;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.PARTITIONINGSPLIT;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean b = false;
+ for (int i = 0; i < expressions.length; i++) {
+ if (visitor.transform(expressions[i])) {
+ b = true;
+ }
+ }
+ return b;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitPartitioningSplitOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
new file mode 100644
index 0000000..736ae1d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ProjectOperator.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FilteredVariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class ProjectOperator extends AbstractLogicalOperator {
+
+ private final List<LogicalVariable> variables;
+
+ public ProjectOperator(List<LogicalVariable> variables) {
+ this.variables = variables;
+ }
+
+ public ProjectOperator(LogicalVariable v) {
+ this.variables = new ArrayList<LogicalVariable>(1);
+ this.getVariables().add(v);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+ // do nothing
+ return false;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.PROJECT;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>(getVariables());
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new FilteredVariablePropagationPolicy(getVariables());
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitProjectOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return true;
+ }
+
+ public List<LogicalVariable> getVariables() {
+ return variables;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
new file mode 100644
index 0000000..f594373
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ReplicateOperator.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class ReplicateOperator extends AbstractLogicalOperator {
+
+ private int outputArity = 2;
+
+ public ReplicateOperator(int outputArity) {
+ this.outputArity = outputArity;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.REPLICATE;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitReplicateOperator(this, arg);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+ }
+
+ public void substituteVar(LogicalVariable v1, LogicalVariable v2) {
+ // do nothing
+ }
+
+ public int getOutputArity() {
+ return outputArity;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
new file mode 100644
index 0000000..798597e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/RunningAggregateOperator.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class RunningAggregateOperator extends AbstractAssignOperator {
+
+ public RunningAggregateOperator(List<LogicalVariable> variables, List<Mutable<ILogicalExpression>> expressions) {
+ super(variables, expressions);
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.RUNNINGAGGREGATE;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitRunningAggregateOperator(this, arg);
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new VariablePropagationPolicy() {
+
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ target.addAllVariables(sources[0]);
+ for (LogicalVariable v : variables) {
+ target.addVariable(v);
+ }
+ }
+ };
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
+ int n = variables.size();
+ for (int i = 0; i < n; i++) {
+ env.setVarType(
+ variables.get(i),
+ ctx.getExpressionTypeComputer().getType(expressions.get(i).getValue(), ctx.getMetadataProvider(),
+ env));
+ }
+ return env;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
new file mode 100644
index 0000000..dc2795c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/ScriptOperator.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class ScriptOperator extends AbstractLogicalOperator {
+
+ private ArrayList<LogicalVariable> inputVariables;
+ private ArrayList<LogicalVariable> outputVariables;
+ private IScriptDescription scriptDesc;
+
+ public ScriptOperator(IScriptDescription scriptDesc, ArrayList<LogicalVariable> inputVariables,
+ ArrayList<LogicalVariable> outputVariables) {
+ this.inputVariables = inputVariables;
+ this.outputVariables = outputVariables;
+ this.scriptDesc = scriptDesc;
+ }
+
+ public ArrayList<LogicalVariable> getInputVariables() {
+ return inputVariables;
+ }
+
+ public ArrayList<LogicalVariable> getOutputVariables() {
+ return outputVariables;
+ }
+
+ public IScriptDescription getScriptDescription() {
+ return scriptDesc;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.SCRIPT;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitScriptOperator(this, arg);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new VariablePropagationPolicy() {
+
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ for (LogicalVariable v : outputVariables) {
+ target.addVariable(v);
+ }
+ }
+ };
+
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ this.schema = outputVariables;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+ ctx.getMetadataProvider());
+ for (Pair<LogicalVariable, Object> p : scriptDesc.getVarTypePairs()) {
+ env.setVarType(p.first, p.second);
+ }
+ return env;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
new file mode 100644
index 0000000..c1e038e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SelectOperator.java
@@ -0,0 +1,105 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class SelectOperator extends AbstractLogicalOperator {
+ private final Mutable<ILogicalExpression> condition;
+
+ public SelectOperator(Mutable<ILogicalExpression> condition) {
+ this.condition = condition;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.SELECT;
+ }
+
+ public Mutable<ILogicalExpression> getCondition() {
+ return condition;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ return visitor.transform(condition);
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitSelectOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return true;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ ITypeEnvPointer[] envPointers = new ITypeEnvPointer[1];
+ envPointers[0] = new OpRefTypeEnvPointer(inputs.get(0), ctx);
+ PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+ ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
+ if (condition.getValue().getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) condition.getValue();
+ if (f1.getFunctionIdentifier() == AlgebricksBuiltinFunctions.NOT) {
+ ILogicalExpression a1 = f1.getArguments().get(0).getValue();
+ if (a1.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression f2 = (AbstractFunctionCallExpression) a1;
+ if (f2.getFunctionIdentifier() == AlgebricksBuiltinFunctions.IS_NULL) {
+ ILogicalExpression a2 = f2.getArguments().get(0).getValue();
+ if (a2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ LogicalVariable var = ((VariableReferenceExpression) a2).getVariableReference();
+ env.getNonNullVariables().add(var);
+ }
+ }
+ }
+ }
+ }
+ return env;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
new file mode 100644
index 0000000..a75a9a9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SinkOperator.java
@@ -0,0 +1,59 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class SinkOperator extends AbstractLogicalOperator {
+
+ @Override
+ public void recomputeSchema() throws AlgebricksException {
+ schema = new ArrayList<LogicalVariable>(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitSinkOperator(this, arg);
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ ITypeEnvPointer[] envPointers = new ITypeEnvPointer[1];
+ envPointers[0] = new OpRefTypeEnvPointer(inputs.get(0), ctx);
+ PropagatingTypeEnvironment env = new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+ ctx.getNullableTypeComputer(), ctx.getMetadataProvider(), TypePropagationPolicy.ALL, envPointers);
+ return env;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.SINK;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
new file mode 100644
index 0000000..0d1704b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/SubplanOperator.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.OpRefTypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.PropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class SubplanOperator extends AbstractOperatorWithNestedPlans {
+
+ public SubplanOperator() {
+ super();
+ }
+
+ public SubplanOperator(List<ILogicalPlan> plans) {
+ super(plans);
+ }
+
+ public SubplanOperator(ILogicalOperator planRoot) {
+ ArrayList<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>(1);
+ roots.add(new MutableObject<ILogicalOperator>(planRoot));
+ nestedPlans.add(new ALogicalPlanImpl(roots));
+ }
+
+ public void setRootOp(Mutable<ILogicalOperator> opRef) {
+ ILogicalPlan p = new ALogicalPlanImpl(opRef);
+ nestedPlans.add(p);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) {
+ // do nothing
+ return false;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.SUBPLAN;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ADDNEWVARIABLES;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitSubplanOperator(this, arg);
+ }
+
+ @Override
+ public void getProducedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
+ // do nothing
+ }
+
+ @Override
+ public void getUsedVariablesExceptNestedPlans(Collection<LogicalVariable> vars) {
+ // do nothing
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ int n = 0;
+ for (ILogicalPlan p : nestedPlans) {
+ n += p.getRoots().size();
+ }
+ ITypeEnvPointer[] envPointers = new ITypeEnvPointer[n + 1];
+ envPointers[0] = new OpRefTypeEnvPointer(inputs.get(0), ctx);
+ int i = 1;
+ for (ILogicalPlan p : nestedPlans) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ envPointers[i] = new OpRefTypeEnvPointer(r, ctx);
+ i++;
+ }
+ }
+ return new PropagatingTypeEnvironment(ctx.getExpressionTypeComputer(), ctx.getNullableTypeComputer(),
+ ctx.getMetadataProvider(), TypePropagationPolicy.LEFT_OUTER, envPointers);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
new file mode 100644
index 0000000..cec7d6f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnionAllOperator.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+
+public class UnionAllOperator extends AbstractLogicalOperator {
+
+ // (left-var, right-var, out-var)
+ private List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap;
+
+ public UnionAllOperator(List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap) {
+ this.varMap = varMap;
+ }
+
+ public List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> getVariableMappings() {
+ return varMap;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.UNIONALL;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new VariablePropagationPolicy() {
+
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+ target.addVariable(t.third);
+ }
+ }
+ };
+
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitUnionOperator(this, arg);
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ for (LogicalVariable v1 : inputs.get(0).getValue().getSchema()) {
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+ if (t.first.equals(v1)) {
+ schema.add(t.third);
+ } else {
+ schema.add(v1);
+ }
+ }
+ }
+ for (LogicalVariable v2 : inputs.get(1).getValue().getSchema()) {
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+ if (t.second.equals(v2)) {
+ schema.add(t.third);
+ } else {
+ schema.add(v2);
+ }
+ }
+ }
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+ ctx.getMetadataProvider());
+ IVariableTypeEnvironment envLeft = ctx.getOutputTypeEnvironment(inputs.get(0).getValue());
+ if (envLeft == null) {
+ throw new AlgebricksException("Left input types for union operator are not computed.");
+ }
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+ Object t1 = envLeft.getVarType(t.first);
+ if (t1 == null) {
+ throw new AlgebricksException("Failed typing union operator: no type for variable " + t.first);
+ }
+ env.setVarType(t.third, t1);
+ }
+ return env;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
new file mode 100644
index 0000000..c4f5cb0
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestMapOperator.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.NonPropagatingTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+/**
+ * This operator may go away after we add indexes to Algebricks.
+ *
+ */
+public class UnnestMapOperator extends AbstractUnnestOperator {
+
+ private final List<Object> variableTypes; // TODO: get rid of this and
+
+ // deprecate UnnestMap
+
+ public UnnestMapOperator(List<LogicalVariable> variables, Mutable<ILogicalExpression> expression,
+ List<Object> variableTypes) {
+ super(variables, expression);
+ this.variableTypes = variableTypes;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.UNNEST_MAP;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitUnnestMapOperator(this, arg);
+ }
+
+ /**
+ *
+ * UnnestMap doesn't propagate input variables, because currently it is only
+ * used to search indexes. In the future, it would be nice to have the
+ * choice to propagate input variables or not.
+ *
+ */
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return new VariablePropagationPolicy() {
+
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ for (LogicalVariable v : variables) {
+ target.addVariable(v);
+ }
+ }
+ };
+ }
+
+ public List<Object> getVariableTypes() {
+ return variableTypes;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ IVariableTypeEnvironment env = new NonPropagatingTypeEnvironment(ctx.getExpressionTypeComputer(),
+ ctx.getMetadataProvider());
+ int n = variables.size();
+ for (int i = 0; i < n; i++) {
+ env.setVarType(variables.get(i), variableTypes.get(i));
+ }
+ return env;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
new file mode 100644
index 0000000..f85cabb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UnnestOperator.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class UnnestOperator extends AbstractUnnestOperator {
+
+ private LogicalVariable positionalVariable;
+ private Object positionalVariableType;
+
+ public UnnestOperator(LogicalVariable variable, Mutable<ILogicalExpression> expression) {
+ super(makeSingletonList(variable), expression);
+ }
+
+ public UnnestOperator(LogicalVariable variable, Mutable<ILogicalExpression> expression,
+ LogicalVariable positionalVariable, Object positionalVariableType) {
+ this(variable, expression);
+ this.setPositionalVariable(positionalVariable);
+ this.setPositionalVariableType(positionalVariableType);
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.UNNEST;
+ }
+
+ public LogicalVariable getVariable() {
+ return variables.get(0);
+ }
+
+ public void setPositionalVariable(LogicalVariable positionalVariable) {
+ this.positionalVariable = positionalVariable;
+ }
+
+ public LogicalVariable getPositionalVariable() {
+ return positionalVariable;
+ }
+
+ public void setPositionalVariableType(Object positionalVariableType) {
+ this.positionalVariableType = positionalVariableType;
+ }
+
+ public Object getPositionalVariableType() {
+ return positionalVariableType;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitUnnestOperator(this, arg);
+ }
+
+ private static <E> ArrayList<E> makeSingletonList(E item) {
+ ArrayList<E> array = new ArrayList<E>(1);
+ array.add(item);
+ return array;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ IVariableTypeEnvironment env = createPropagatingAllInputsTypeEnvironment(ctx);
+ Object t = env.getType(expression.getValue());
+ env.setVarType(variables.get(0), t);
+ if (positionalVariable != null) {
+ env.setVarType(positionalVariable, positionalVariableType);
+ }
+ return env;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
new file mode 100644
index 0000000..4f94f34
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/UpdateOperator.java
@@ -0,0 +1,55 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class UpdateOperator extends AbstractLogicalOperator {
+
+ @Override
+ public void recomputeSchema() throws AlgebricksException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform transform) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public boolean isMap() {
+ // TODO Auto-generated method stub
+ return false;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
new file mode 100644
index 0000000..fdcdfd8
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteOperator.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class WriteOperator extends AbstractLogicalOperator {
+ private List<Mutable<ILogicalExpression>> expressions;
+ private IDataSink dataSink;
+
+ public WriteOperator(List<Mutable<ILogicalExpression>> expressions, IDataSink dataSink) {
+ this.expressions = expressions;
+ this.dataSink = dataSink;
+ }
+
+ public List<Mutable<ILogicalExpression>> getExpressions() {
+ return expressions;
+ }
+
+ public IDataSink getDataSink() {
+ return dataSink;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.WRITE;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitWriteOperator(this, arg);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean modif = false;
+ for (int i = 0; i < expressions.size(); i++) {
+ boolean b = visitor.transform(expressions.get(i));
+ if (b) {
+ modif = true;
+ }
+ }
+ return modif;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean isMap() {
+ return false; // actually depends on the physical op.
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
new file mode 100644
index 0000000..6d05c76
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/WriteResultOperator.java
@@ -0,0 +1,100 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.VariablePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class WriteResultOperator extends AbstractLogicalOperator {
+
+ private IDataSource<?> dataSource;
+ private Mutable<ILogicalExpression> payloadExpr;
+ private List<Mutable<ILogicalExpression>> keyExprs;
+
+ public WriteResultOperator(IDataSource<?> dataSource, Mutable<ILogicalExpression> payload,
+ List<Mutable<ILogicalExpression>> keyExprs) {
+ this.dataSource = dataSource;
+ this.payloadExpr = payload;
+ this.keyExprs = keyExprs;
+ }
+
+ public IDataSource<?> getDataSource() {
+ return dataSource;
+ }
+
+ public Mutable<ILogicalExpression> getPayloadExpression() {
+ return payloadExpr;
+ }
+
+ public List<Mutable<ILogicalExpression>> getKeyExpressions() {
+ return keyExprs;
+ }
+
+ @Override
+ public LogicalOperatorTag getOperatorTag() {
+ return LogicalOperatorTag.WRITE_RESULT;
+ }
+
+ @Override
+ public <R, T> R accept(ILogicalOperatorVisitor<R, T> visitor, T arg) throws AlgebricksException {
+ return visitor.visitWriteResultOperator(this, arg);
+ }
+
+ @Override
+ public boolean acceptExpressionTransform(ILogicalExpressionReferenceTransform visitor) throws AlgebricksException {
+ boolean b = visitor.transform(payloadExpr);
+ for (int i = 0; i < keyExprs.size(); i++) {
+ if (visitor.transform(keyExprs.get(i))) {
+ b = true;
+ }
+ }
+ return b;
+ }
+
+ @Override
+ public VariablePropagationPolicy getVariablePropagationPolicy() {
+ return VariablePropagationPolicy.ALL;
+ }
+
+ @Override
+ public boolean isMap() {
+ return false;
+ }
+
+ @Override
+ public void recomputeSchema() {
+ schema = new ArrayList<LogicalVariable>();
+ schema.addAll(inputs.get(0).getValue().getSchema());
+ }
+
+ @Override
+ public IVariableTypeEnvironment computeOutputTypeEnvironment(ITypingContext ctx) throws AlgebricksException {
+ return createPropagatingAllInputsTypeEnvironment(ctx);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
new file mode 100644
index 0000000..0c0f257
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
@@ -0,0 +1,640 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractUnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class FDsAndEquivClassesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> {
+
+ @Override
+ public Void visitAggregateOperator(AggregateOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ ctx.putEquivalenceClassMap(op, new HashMap<LogicalVariable, EquivalenceClass>());
+ ctx.putFDList(op, new ArrayList<FunctionalDependency>());
+ return null;
+ }
+
+ @Override
+ public Void visitAssignOperator(AssignOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ ILogicalOperator inp1 = op.getInputs().get(0).getValue();
+ Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
+ ctx.putEquivalenceClassMap(op, eqClasses);
+ List<LogicalVariable> used = new ArrayList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op, used);
+ List<FunctionalDependency> fds1 = getOrComputeFDs(inp1, ctx);
+ List<FunctionalDependency> eFds = new ArrayList<FunctionalDependency>(fds1.size());
+ for (FunctionalDependency fd : fds1) {
+ if (fd.getTail().containsAll(used)) {
+ List<LogicalVariable> hd = new ArrayList<LogicalVariable>(fd.getHead());
+ List<LogicalVariable> tl = new ArrayList<LogicalVariable>(fd.getTail());
+ tl.addAll(op.getVariables());
+ FunctionalDependency fd2 = new FunctionalDependency(hd, tl);
+ eFds.add(fd2);
+ } else {
+ eFds.add(fd);
+ }
+ }
+ ctx.putFDList(op, eFds);
+ return null;
+ }
+
+ @Override
+ public Void visitDataScanOperator(DataSourceScanOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ ILogicalOperator inp1 = op.getInputs().get(0).getValue();
+ Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
+ ctx.putEquivalenceClassMap(op, eqClasses);
+ List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(getOrComputeFDs(inp1, ctx));
+ ctx.putFDList(op, fds);
+ op.getDataSource().computeFDs(op.getVariables(), fds);
+ return null;
+ }
+
+ @Override
+ public Void visitDistinctOperator(DistinctOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ ILogicalOperator op0 = op.getInputs().get(0).getValue();
+ List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+ ctx.putFDList(op, functionalDependencies);
+ for (FunctionalDependency inherited : getOrComputeFDs(op0, ctx)) {
+ boolean isCoveredByDistinctByVars = true;
+ for (LogicalVariable v : inherited.getHead()) {
+ if (!op.isDistinctByVar(v)) {
+ isCoveredByDistinctByVars = false;
+ }
+ }
+ if (isCoveredByDistinctByVars) {
+ List<LogicalVariable> newTail = new ArrayList<LogicalVariable>();
+ for (LogicalVariable v2 : inherited.getTail()) {
+ if (op.isDistinctByVar(v2)) {
+ newTail.add(v2);
+ }
+ }
+ if (!newTail.isEmpty()) {
+ List<LogicalVariable> newHead = new ArrayList<LogicalVariable>(inherited.getHead());
+ FunctionalDependency newFd = new FunctionalDependency(newHead, newTail);
+ functionalDependencies.add(newFd);
+ }
+ }
+ }
+ Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
+ List<Mutable<ILogicalExpression>> expressions = op.getExpressions();
+ for (Mutable<ILogicalExpression> pRef : expressions) {
+ ILogicalExpression p = pRef.getValue();
+ if (p.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression v = (VariableReferenceExpression) p;
+ gbySet.add(v.getVariableReference());
+ }
+ }
+ LocalGroupingProperty lgp = new LocalGroupingProperty(gbySet);
+
+ Map<LogicalVariable, EquivalenceClass> equivalenceClasses = getOrComputeEqClasses(op0, ctx);
+ ctx.putEquivalenceClassMap(op, equivalenceClasses);
+
+ lgp.normalizeGroupingColumns(equivalenceClasses, functionalDependencies);
+ Set<LogicalVariable> normSet = lgp.getColumnSet();
+ List<Mutable<ILogicalExpression>> newDistinctByList = new ArrayList<Mutable<ILogicalExpression>>();
+ for (Mutable<ILogicalExpression> p2Ref : expressions) {
+ ILogicalExpression p2 = p2Ref.getValue();
+ if (p2.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression var2 = (VariableReferenceExpression) p2;
+ LogicalVariable v2 = var2.getVariableReference();
+ if (normSet.contains(v2)) {
+ newDistinctByList.add(p2Ref);
+ }
+ } else {
+ newDistinctByList.add(p2Ref);
+ }
+ }
+ expressions.clear();
+ expressions.addAll(newDistinctByList);
+ return null;
+ }
+
+ @Override
+ public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ ctx.putEquivalenceClassMap(op, new HashMap<LogicalVariable, EquivalenceClass>());
+ ctx.putFDList(op, new ArrayList<FunctionalDependency>());
+ return null;
+ }
+
+ @Override
+ public Void visitExchangeOperator(ExchangeOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ propagateFDsAndEquivClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitGroupByOperator(GroupByOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+ List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+ ctx.putEquivalenceClassMap(op, equivalenceClasses);
+ ctx.putFDList(op, functionalDependencies);
+
+ List<FunctionalDependency> inheritedFDs = new ArrayList<FunctionalDependency>();
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ ILogicalOperator op2 = r.getValue();
+ equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
+ inheritedFDs.addAll(getOrComputeFDs(op2, ctx));
+ }
+ }
+
+ ILogicalOperator op0 = op.getInputs().get(0).getValue();
+ inheritedFDs.addAll(getOrComputeFDs(op0, ctx));
+ Map<LogicalVariable, EquivalenceClass> inheritedEcs = getOrComputeEqClasses(op0, ctx);
+ for (FunctionalDependency inherited : inheritedFDs) {
+ boolean isCoveredByGbyOrDecorVars = true;
+ List<LogicalVariable> newHead = new ArrayList<LogicalVariable>(inherited.getHead().size());
+ for (LogicalVariable v : inherited.getHead()) {
+ LogicalVariable vnew = getNewGbyVar(op, v);
+ if (vnew == null) {
+ vnew = getNewDecorVar(op, v);
+ if (vnew == null) {
+ isCoveredByGbyOrDecorVars = false;
+ }
+ break;
+ }
+ newHead.add(vnew);
+ }
+
+ if (isCoveredByGbyOrDecorVars) {
+ List<LogicalVariable> newTail = new ArrayList<LogicalVariable>();
+ for (LogicalVariable v2 : inherited.getTail()) {
+ LogicalVariable v3 = getNewGbyVar(op, v2);
+ if (v3 != null) {
+ newTail.add(v3);
+ }
+ }
+ if (!newTail.isEmpty()) {
+ FunctionalDependency newFd = new FunctionalDependency(newHead, newTail);
+ functionalDependencies.add(newFd);
+ }
+ }
+ }
+
+ List<LogicalVariable> premiseGby = new LinkedList<LogicalVariable>();
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gByList = op.getGroupByList();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+ premiseGby.add(p.first);
+ }
+
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorList = op.getDecorList();
+
+ LinkedList<LogicalVariable> conclDecor = new LinkedList<LogicalVariable>();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorList) {
+ conclDecor.add(GroupByOperator.getDecorVariable(p));
+ }
+ if (!conclDecor.isEmpty()) {
+ functionalDependencies.add(new FunctionalDependency(premiseGby, conclDecor));
+ }
+
+ Set<LogicalVariable> gbySet = new HashSet<LogicalVariable>();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+ ILogicalExpression expr = p.second.getValue();
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression v = (VariableReferenceExpression) expr;
+ gbySet.add(v.getVariableReference());
+ }
+ }
+ LocalGroupingProperty lgp = new LocalGroupingProperty(gbySet);
+ lgp.normalizeGroupingColumns(inheritedEcs, inheritedFDs);
+ Set<LogicalVariable> normSet = lgp.getColumnSet();
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newGbyList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+ boolean changed = false;
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gByList) {
+ ILogicalExpression expr = p.second.getValue();
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+ LogicalVariable v2 = varRef.getVariableReference();
+ EquivalenceClass ec2 = inheritedEcs.get(v2);
+ LogicalVariable v3;
+ if (ec2 != null && !ec2.representativeIsConst()) {
+ v3 = ec2.getVariableRepresentative();
+ } else {
+ v3 = v2;
+ }
+ if (normSet.contains(v3)) {
+ newGbyList.add(p);
+ } else {
+ changed = true;
+ decorList.add(p);
+ }
+ } else {
+ newGbyList.add(p);
+ }
+ }
+ if (changed) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Group-by list changed from "
+ + GroupByOperator.veListToString(gByList) + " to " + GroupByOperator.veListToString(newGbyList)
+ + ".\n");
+ }
+ gByList.clear();
+ gByList.addAll(newGbyList);
+ return null;
+ }
+
+ @Override
+ public Void visitInnerJoinOperator(InnerJoinOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+ List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+ ctx.putEquivalenceClassMap(op, equivalenceClasses);
+ ctx.putFDList(op, functionalDependencies);
+ ILogicalOperator op0 = op.getInputs().get(0).getValue();
+ ILogicalOperator op1 = op.getInputs().get(1).getValue();
+ functionalDependencies.addAll(getOrComputeFDs(op0, ctx));
+ functionalDependencies.addAll(getOrComputeFDs(op1, ctx));
+ equivalenceClasses.putAll(getOrComputeEqClasses(op0, ctx));
+ equivalenceClasses.putAll(getOrComputeEqClasses(op1, ctx));
+ ILogicalExpression expr = op.getCondition().getValue();
+ expr.getConstraintsAndEquivClasses(functionalDependencies, equivalenceClasses);
+ return null;
+ }
+
+ @Override
+ public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+ List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+ ctx.putEquivalenceClassMap(op, equivalenceClasses);
+ ctx.putFDList(op, functionalDependencies);
+ ILogicalOperator opLeft = op.getInputs().get(0).getValue();
+ ILogicalOperator opRight = op.getInputs().get(1).getValue();
+ functionalDependencies.addAll(getOrComputeFDs(opLeft, ctx));
+ functionalDependencies.addAll(getOrComputeFDs(opRight, ctx));
+ equivalenceClasses.putAll(getOrComputeEqClasses(opLeft, ctx));
+ equivalenceClasses.putAll(getOrComputeEqClasses(opRight, ctx));
+
+ Collection<LogicalVariable> leftSideVars;
+ if (opLeft.getSchema() == null) {
+ leftSideVars = new LinkedList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(opLeft, leftSideVars);
+ // actually, not all produced vars. are visible (due to projection)
+ // so using cached schema is better and faster
+ } else {
+ leftSideVars = opLeft.getSchema();
+ }
+ ILogicalExpression expr = op.getCondition().getValue();
+ expr.getConstraintsForOuterJoin(functionalDependencies, leftSideVars);
+ return null;
+ }
+
+ @Override
+ public Void visitLimitOperator(LimitOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ propagateFDsAndEquivClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitDieOperator(DieOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ propagateFDsAndEquivClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getDataSourceReference().getValue();
+ ILogicalOperator inp1 = op1.getInputs().get(0).getValue();
+ Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
+ ctx.putEquivalenceClassMap(op, eqClasses);
+ List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>(getOrComputeFDs(inp1, ctx));
+ if (op1.getOperatorTag() == LogicalOperatorTag.GROUP) {
+ GroupByOperator gby = (GroupByOperator) op1;
+ LinkedList<LogicalVariable> tail = new LinkedList<LogicalVariable>();
+ for (LogicalVariable v : gby.getGbyVarList()) {
+ tail.add(v);
+ // all values for gby vars. are the same
+ }
+ FunctionalDependency gbyfd = new FunctionalDependency(new LinkedList<LogicalVariable>(), tail);
+ fds.add(gbyfd);
+ }
+ ctx.putFDList(op, fds);
+ return null;
+ }
+
+ @Override
+ public Void visitOrderOperator(OrderOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ propagateFDsAndEquivClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public Void visitProjectOperator(ProjectOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ propagateFDsAndEquivClassesForUsedVars(op, ctx, op.getVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitReplicateOperator(ReplicateOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ propagateFDsAndEquivClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitRunningAggregateOperator(RunningAggregateOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ propagateFDsAndEquivClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitScriptOperator(ScriptOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ propagateFDsAndEquivClassesForUsedVars(op, ctx, op.getInputVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitSelectOperator(SelectOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+ List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+ ctx.putEquivalenceClassMap(op, equivalenceClasses);
+ ctx.putFDList(op, functionalDependencies);
+ ILogicalOperator op0 = op.getInputs().get(0).getValue();
+ functionalDependencies.addAll(getOrComputeFDs(op0, ctx));
+ equivalenceClasses.putAll(getOrComputeEqClasses(op0, ctx));
+ ILogicalExpression expr = op.getCondition().getValue();
+ expr.getConstraintsAndEquivClasses(functionalDependencies, equivalenceClasses);
+ return null;
+ }
+
+ @Override
+ public Void visitSubplanOperator(SubplanOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ Map<LogicalVariable, EquivalenceClass> equivalenceClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+ List<FunctionalDependency> functionalDependencies = new ArrayList<FunctionalDependency>();
+ ctx.putEquivalenceClassMap(op, equivalenceClasses);
+ ctx.putFDList(op, functionalDependencies);
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ ILogicalOperator op2 = r.getValue();
+ equivalenceClasses.putAll(getOrComputeEqClasses(op2, ctx));
+ functionalDependencies.addAll(getOrComputeFDs(op2, ctx));
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitUnionOperator(UnionAllOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ setEmptyFDsEqClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestMapOperator(UnnestMapOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ fdsEqClassesForAbstractUnnestOperator(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestOperator(UnnestOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ fdsEqClassesForAbstractUnnestOperator(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitWriteOperator(WriteOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ // propagateFDsAndEquivClasses(op, ctx);
+ setEmptyFDsEqClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitWriteResultOperator(WriteResultOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ // propagateFDsAndEquivClasses(op, ctx);
+ setEmptyFDsEqClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitInsertDeleteOperator(InsertDeleteOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ setEmptyFDsEqClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ setEmptyFDsEqClasses(op, ctx);
+ return null;
+ }
+
+ @Override
+ public Void visitSinkOperator(SinkOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ setEmptyFDsEqClasses(op, ctx);
+ return null;
+ }
+
+ private void propagateFDsAndEquivClasses(ILogicalOperator op, IOptimizationContext ctx) throws AlgebricksException {
+ ILogicalOperator inp1 = op.getInputs().get(0).getValue();
+ Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
+ ctx.putEquivalenceClassMap(op, eqClasses);
+ List<FunctionalDependency> fds = getOrComputeFDs(inp1, ctx);
+ ctx.putFDList(op, fds);
+ }
+
+ private Map<LogicalVariable, EquivalenceClass> getOrComputeEqClasses(ILogicalOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ Map<LogicalVariable, EquivalenceClass> eqClasses = ctx.getEquivalenceClassMap(op);
+ if (eqClasses == null) {
+ op.accept(this, ctx);
+ eqClasses = ctx.getEquivalenceClassMap(op);
+ }
+ return eqClasses;
+ }
+
+ private List<FunctionalDependency> getOrComputeFDs(ILogicalOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ List<FunctionalDependency> fds = ctx.getFDList(op);
+ if (fds == null) {
+ op.accept(this, ctx);
+ fds = ctx.getFDList(op);
+ }
+ return fds;
+ }
+
+ private void propagateFDsAndEquivClassesForUsedVars(ILogicalOperator op, IOptimizationContext ctx,
+ List<LogicalVariable> usedVariables) throws AlgebricksException {
+ ILogicalOperator op2 = op.getInputs().get(0).getValue();
+ Map<LogicalVariable, EquivalenceClass> eqClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+ ctx.putEquivalenceClassMap(op, eqClasses);
+ List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>();
+ ctx.putFDList(op, fds);
+
+ Map<LogicalVariable, EquivalenceClass> chldClasses = getOrComputeEqClasses(op2, ctx);
+ for (LogicalVariable v : usedVariables) {
+ EquivalenceClass ec = eqClasses.get(v);
+ if (ec == null) {
+ EquivalenceClass oc = chldClasses.get(v);
+ if (oc == null) {
+ continue;
+ }
+ List<LogicalVariable> m = new LinkedList<LogicalVariable>();
+ for (LogicalVariable v2 : oc.getMembers()) {
+ if (usedVariables.contains(v2)) {
+ m.add(v2);
+ }
+ }
+ EquivalenceClass nc;
+ if (oc.representativeIsConst()) {
+ nc = new EquivalenceClass(m, oc.getConstRepresentative());
+ } else if (m.contains(oc.getVariableRepresentative())) {
+ nc = new EquivalenceClass(m, oc.getVariableRepresentative());
+ } else {
+ nc = new EquivalenceClass(m, v);
+ }
+ for (LogicalVariable v3 : m) {
+ eqClasses.put(v3, nc);
+ }
+ }
+ }
+
+ List<FunctionalDependency> chldFds = getOrComputeFDs(op2, ctx);
+ for (FunctionalDependency fd : chldFds) {
+ if (!usedVariables.containsAll(fd.getHead())) {
+ continue;
+ }
+ List<LogicalVariable> tl = new LinkedList<LogicalVariable>();
+ for (LogicalVariable v : fd.getTail()) {
+ if (usedVariables.contains(v)) {
+ tl.add(v);
+ }
+ }
+ if (!tl.isEmpty()) {
+ FunctionalDependency newFd = new FunctionalDependency(fd.getHead(), tl);
+ fds.add(newFd);
+ }
+ }
+ }
+
+ private void fdsEqClassesForAbstractUnnestOperator(AbstractUnnestOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ ILogicalOperator inp1 = op.getInputs().get(0).getValue();
+ Map<LogicalVariable, EquivalenceClass> eqClasses = getOrComputeEqClasses(inp1, ctx);
+ ctx.putEquivalenceClassMap(op, eqClasses);
+ List<FunctionalDependency> fds = getOrComputeFDs(inp1, ctx);
+ ctx.putFDList(op, fds);
+
+ ILogicalExpression expr = op.getExpressionRef().getValue();
+ if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression afe = (AbstractFunctionCallExpression) expr;
+ if (afe.getKind() == FunctionKind.UNNEST && ((UnnestingFunctionCallExpression) afe).returnsUniqueValues()) {
+ List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(op, vars);
+ ArrayList<LogicalVariable> h = new ArrayList<LogicalVariable>();
+ h.addAll(op.getVariables());
+ FunctionalDependency fd = new FunctionalDependency(h, vars);
+ fds.add(fd);
+ }
+ }
+ }
+
+ public static void setEmptyFDsEqClasses(ILogicalOperator op, IOptimizationContext ctx) {
+ Map<LogicalVariable, EquivalenceClass> eqClasses = new HashMap<LogicalVariable, EquivalenceClass>();
+ ctx.putEquivalenceClassMap(op, eqClasses);
+ List<FunctionalDependency> fds = new ArrayList<FunctionalDependency>();
+ ctx.putFDList(op, fds);
+ }
+
+ private LogicalVariable getNewGbyVar(GroupByOperator g, LogicalVariable v) {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getGroupByList()) {
+ ILogicalExpression e = p.second.getValue();
+ if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ LogicalVariable v2 = ((VariableReferenceExpression) e).getVariableReference();
+ if (v2 == v) {
+ return p.first;
+ }
+ }
+ }
+ return null;
+ }
+
+ private LogicalVariable getNewDecorVar(GroupByOperator g, LogicalVariable v) {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : g.getDecorList()) {
+ ILogicalExpression e = p.second.getValue();
+ if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ LogicalVariable v2 = ((VariableReferenceExpression) e).getVariableReference();
+ if (v2 == v) {
+ return (p.first != null) ? p.first : v2;
+ }
+ }
+ }
+ return null;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
new file mode 100644
index 0000000..5d9e6b9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@ -0,0 +1,812 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+
+public class IsomorphismOperatorVisitor implements ILogicalOperatorVisitor<Boolean, ILogicalOperator> {
+
+ private Map<LogicalVariable, LogicalVariable> variableMapping = new HashMap<LogicalVariable, LogicalVariable>();
+
+ public IsomorphismOperatorVisitor() {
+ }
+
+ @Override
+ public Boolean visitAggregateOperator(AggregateOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.AGGREGATE)
+ return Boolean.FALSE;
+ AggregateOperator aggOpArg = (AggregateOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(
+ getPairList(op.getVariables(), op.getExpressions()),
+ getPairList(aggOpArg.getVariables(), aggOpArg.getExpressions()));
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitRunningAggregateOperator(RunningAggregateOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.RUNNINGAGGREGATE)
+ return Boolean.FALSE;
+ RunningAggregateOperator aggOpArg = (RunningAggregateOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(
+ getPairList(op.getVariables(), op.getExpressions()),
+ getPairList(aggOpArg.getVariables(), aggOpArg.getExpressions()));
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) copyAndSubstituteVar(op, arg);
+ if (aop.getOperatorTag() != LogicalOperatorTag.EMPTYTUPLESOURCE)
+ return Boolean.FALSE;
+ return Boolean.TRUE;
+ }
+
+ @Override
+ public Boolean visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ // require the same physical operator, otherwise delivers different data
+ // properties
+ if (aop.getOperatorTag() != LogicalOperatorTag.GROUP
+ || aop.getPhysicalOperator().getOperatorTag() != op.getPhysicalOperator().getOperatorTag())
+ return Boolean.FALSE;
+
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyLists = op.getGroupByList();
+ GroupByOperator gbyOpArg = (GroupByOperator) copyAndSubstituteVar(op, arg);
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> keyListsArg = gbyOpArg.getGroupByList();
+
+ List<Pair<LogicalVariable, ILogicalExpression>> listLeft = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
+ List<Pair<LogicalVariable, ILogicalExpression>> listRight = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
+
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyLists)
+ listLeft.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : keyListsArg)
+ listRight.add(new Pair<LogicalVariable, ILogicalExpression>(pair.first, pair.second.getValue()));
+
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(listLeft, listRight);
+
+ if (!isomorphic)
+ return Boolean.FALSE;
+ int sizeOp = op.getNestedPlans().size();
+ int sizeArg = gbyOpArg.getNestedPlans().size();
+ if (sizeOp != sizeArg)
+ return Boolean.FALSE;
+
+ GroupByOperator argOp = (GroupByOperator) arg;
+ List<ILogicalPlan> plans = op.getNestedPlans();
+ List<ILogicalPlan> plansArg = argOp.getNestedPlans();
+ for (int i = 0; i < plans.size(); i++) {
+ List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
+ List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
+ if (roots.size() != rootsArg.size())
+ return Boolean.FALSE;
+ for (int j = 0; j < roots.size(); j++) {
+ ILogicalOperator topOp1 = roots.get(j).getValue();
+ ILogicalOperator topOp2 = rootsArg.get(j).getValue();
+ isomorphic = this.checkBottomUp(topOp1, topOp2);
+ if (!isomorphic)
+ return Boolean.FALSE;
+ }
+ }
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitLimitOperator(LimitOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.LIMIT)
+ return Boolean.FALSE;
+ LimitOperator limitOpArg = (LimitOperator) copyAndSubstituteVar(op, arg);
+ if (op.getOffset() != limitOpArg.getOffset())
+ return Boolean.FALSE;
+ boolean isomorphic = op.getMaxObjects().getValue().equals(limitOpArg.getMaxObjects().getValue());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitDieOperator(DieOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.DIE)
+ return Boolean.FALSE;
+ DieOperator dieOpArg = (DieOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = op.getAfterObjects().getValue().equals(dieOpArg.getAfterObjects().getValue());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitInnerJoinOperator(InnerJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.INNERJOIN)
+ return Boolean.FALSE;
+ InnerJoinOperator joinOpArg = (InnerJoinOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitLeftOuterJoinOperator(LeftOuterJoinOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN)
+ return Boolean.FALSE;
+ LeftOuterJoinOperator joinOpArg = (LeftOuterJoinOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = op.getCondition().getValue().equals(joinOpArg.getCondition().getValue());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitNestedTupleSourceOperator(NestedTupleSourceOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE)
+ return Boolean.FALSE;
+ return Boolean.TRUE;
+ }
+
+ @Override
+ public Boolean visitOrderOperator(OrderOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.ORDER)
+ return Boolean.FALSE;
+ OrderOperator orderOpArg = (OrderOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = compareIOrderAndExpressions(op.getOrderExpressions(), orderOpArg.getOrderExpressions());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitAssignOperator(AssignOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.ASSIGN)
+ return Boolean.FALSE;
+ AssignOperator assignOpArg = (AssignOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(
+ getPairList(op.getVariables(), op.getExpressions()),
+ getPairList(assignOpArg.getVariables(), assignOpArg.getExpressions()));
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitSelectOperator(SelectOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.SELECT)
+ return Boolean.FALSE;
+ SelectOperator selectOpArg = (SelectOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = op.getCondition().getValue().equals(selectOpArg.getCondition().getValue());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitProjectOperator(ProjectOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.PROJECT)
+ return Boolean.FALSE;
+ ProjectOperator projectOpArg = (ProjectOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), projectOpArg.getVariables());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitPartitioningSplitOperator(PartitioningSplitOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.PARTITIONINGSPLIT)
+ return Boolean.FALSE;
+ PartitioningSplitOperator partitionOpArg = (PartitioningSplitOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = compareExpressions(Arrays.asList(op.getExpressions()),
+ Arrays.asList(partitionOpArg.getExpressions()));
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitReplicateOperator(ReplicateOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.REPLICATE)
+ return Boolean.FALSE;
+ return Boolean.TRUE;
+ }
+
+ @Override
+ public Boolean visitScriptOperator(ScriptOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.SCRIPT)
+ return Boolean.FALSE;
+ ScriptOperator scriptOpArg = (ScriptOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = op.getScriptDescription().equals(scriptOpArg.getScriptDescription());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitSubplanOperator(SubplanOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.SUBPLAN)
+ return Boolean.FALSE;
+ SubplanOperator subplanOpArg = (SubplanOperator) copyAndSubstituteVar(op, arg);
+ List<ILogicalPlan> plans = op.getNestedPlans();
+ List<ILogicalPlan> plansArg = subplanOpArg.getNestedPlans();
+ for (int i = 0; i < plans.size(); i++) {
+ List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
+ List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
+ if (roots.size() == rootsArg.size())
+ return Boolean.FALSE;
+ for (int j = 0; j < roots.size(); j++) {
+ ILogicalOperator topOp1 = roots.get(j).getValue();
+ ILogicalOperator topOp2 = rootsArg.get(j).getValue();
+ boolean isomorphic = this.checkBottomUp(topOp1, topOp2);
+ if (!isomorphic)
+ return Boolean.FALSE;
+ }
+ }
+ return Boolean.TRUE;
+ }
+
+ @Override
+ public Boolean visitUnionOperator(UnionAllOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.UNIONALL)
+ return Boolean.FALSE;
+ UnionAllOperator unionOpArg = (UnionAllOperator) copyAndSubstituteVar(op, arg);
+ List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> mapping = op.getVariableMappings();
+ List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> mappingArg = unionOpArg.getVariableMappings();
+ if (mapping.size() != mappingArg.size())
+ return Boolean.FALSE;
+ return VariableUtilities.varListEqualUnordered(mapping, mappingArg);
+ }
+
+ @Override
+ public Boolean visitUnnestOperator(UnnestOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST)
+ return Boolean.FALSE;
+ UnnestOperator unnestOpArg = (UnnestOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables())
+ && variableEqual(op.getPositionalVariable(), unnestOpArg.getPositionalVariable());
+ if (!isomorphic)
+ return Boolean.FALSE;
+ isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitUnnestMapOperator(UnnestMapOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.UNNEST_MAP)
+ return Boolean.FALSE;
+ UnnestOperator unnestOpArg = (UnnestOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), unnestOpArg.getVariables());
+ if (!isomorphic)
+ return Boolean.FALSE;
+ isomorphic = op.getExpressionRef().getValue().equals(unnestOpArg.getExpressionRef().getValue());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN)
+ return Boolean.FALSE;
+ DataSourceScanOperator argScan = (DataSourceScanOperator) arg;
+ if (!argScan.getDataSource().toString().equals(op.getDataSource().toString()))
+ return Boolean.FALSE;
+ DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getVariables(), scanOpArg.getVariables())
+ && op.getDataSource().toString().equals(scanOpArg.getDataSource().toString());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitDistinctOperator(DistinctOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.DISTINCT)
+ return Boolean.FALSE;
+ DistinctOperator distinctOpArg = (DistinctOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = compareExpressions(op.getExpressions(), distinctOpArg.getExpressions());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitExchangeOperator(ExchangeOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.EXCHANGE)
+ return Boolean.FALSE;
+ // require the same partition property
+ if (!(op.getPhysicalOperator().getOperatorTag() == aop.getPhysicalOperator().getOperatorTag()))
+ return Boolean.FALSE;
+ variableMapping.clear();
+ IsomorphismUtilities.mapVariablesTopDown(op, arg, variableMapping);
+ IPhysicalPropertiesVector properties = op.getPhysicalOperator().getDeliveredProperties();
+ IPhysicalPropertiesVector propertiesArg = aop.getPhysicalOperator().getDeliveredProperties();
+ if (properties == null && propertiesArg == null)
+ return Boolean.TRUE;
+ if (properties == null || propertiesArg == null)
+ return Boolean.FALSE;
+ IPartitioningProperty partProp = properties.getPartitioningProperty();
+ IPartitioningProperty partPropArg = propertiesArg.getPartitioningProperty();
+ if (!partProp.getPartitioningType().equals(partPropArg.getPartitioningType()))
+ return Boolean.FALSE;
+ List<LogicalVariable> columns = new ArrayList<LogicalVariable>();
+ partProp.getColumns(columns);
+ List<LogicalVariable> columnsArg = new ArrayList<LogicalVariable>();
+ partPropArg.getColumns(columnsArg);
+ if (columns.size() != columnsArg.size())
+ return Boolean.FALSE;
+ if (columns.size() == 0)
+ return Boolean.TRUE;
+ for (int i = 0; i < columnsArg.size(); i++) {
+ LogicalVariable rightVar = columnsArg.get(i);
+ LogicalVariable leftVar = variableMapping.get(rightVar);
+ if (leftVar != null)
+ columnsArg.set(i, leftVar);
+ }
+ return VariableUtilities.varListEqualUnordered(columns, columnsArg);
+ }
+
+ @Override
+ public Boolean visitWriteOperator(WriteOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.WRITE)
+ return Boolean.FALSE;
+ WriteOperator writeOpArg = (WriteOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema());
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitWriteResultOperator(WriteResultOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT)
+ return Boolean.FALSE;
+ WriteResultOperator writeOpArg = (WriteResultOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), writeOpArg.getSchema());
+ if (!op.getDataSource().equals(writeOpArg.getDataSource()))
+ isomorphic = false;
+ if (!op.getPayloadExpression().equals(writeOpArg.getPayloadExpression()))
+ isomorphic = false;
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitInsertDeleteOperator(InsertDeleteOperator op, ILogicalOperator arg) throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.INSERT_DELETE)
+ return Boolean.FALSE;
+ InsertDeleteOperator insertOpArg = (InsertDeleteOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), insertOpArg.getSchema());
+ if (!op.getDataSource().equals(insertOpArg.getDataSource()))
+ isomorphic = false;
+ if (!op.getPayloadExpression().equals(insertOpArg.getPayloadExpression()))
+ isomorphic = false;
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) arg;
+ if (aop.getOperatorTag() != LogicalOperatorTag.INDEX_INSERT_DELETE)
+ return Boolean.FALSE;
+ IndexInsertDeleteOperator insertOpArg = (IndexInsertDeleteOperator) copyAndSubstituteVar(op, arg);
+ boolean isomorphic = VariableUtilities.varListEqualUnordered(op.getSchema(), insertOpArg.getSchema());
+ if (!op.getDataSourceIndex().equals(insertOpArg.getDataSourceIndex()))
+ isomorphic = false;
+ return isomorphic;
+ }
+
+ @Override
+ public Boolean visitSinkOperator(SinkOperator op, ILogicalOperator arg) throws AlgebricksException {
+ return true;
+ }
+
+ private Boolean compareExpressions(List<Mutable<ILogicalExpression>> opExprs,
+ List<Mutable<ILogicalExpression>> argExprs) {
+ if (opExprs.size() != argExprs.size())
+ return Boolean.FALSE;
+ for (int i = 0; i < opExprs.size(); i++) {
+ boolean isomorphic = opExprs.get(i).getValue().equals(argExprs.get(i).getValue());
+ if (!isomorphic)
+ return Boolean.FALSE;
+ }
+ return Boolean.TRUE;
+ }
+
+ private Boolean compareIOrderAndExpressions(List<Pair<IOrder, Mutable<ILogicalExpression>>> opOrderExprs,
+ List<Pair<IOrder, Mutable<ILogicalExpression>>> argOrderExprs) {
+ if (opOrderExprs.size() != argOrderExprs.size())
+ return Boolean.FALSE;
+ for (int i = 0; i < opOrderExprs.size(); i++) {
+ boolean isomorphic = opOrderExprs.get(i).first.equals(argOrderExprs.get(i).first);
+ if (!isomorphic)
+ return Boolean.FALSE;
+ isomorphic = opOrderExprs.get(i).second.getValue().equals(argOrderExprs.get(i).second.getValue());
+ if (!isomorphic)
+ return Boolean.FALSE;
+ }
+ return Boolean.TRUE;
+ }
+
+ private Boolean checkBottomUp(ILogicalOperator op1, ILogicalOperator op2) throws AlgebricksException {
+ List<Mutable<ILogicalOperator>> inputs1 = op1.getInputs();
+ List<Mutable<ILogicalOperator>> inputs2 = op2.getInputs();
+ if (inputs1.size() != inputs2.size())
+ return Boolean.FALSE;
+ for (int i = 0; i < inputs1.size(); i++) {
+ ILogicalOperator input1 = inputs1.get(i).getValue();
+ ILogicalOperator input2 = inputs2.get(i).getValue();
+ boolean isomorphic = checkBottomUp(input1, input2);
+ if (!isomorphic)
+ return Boolean.FALSE;
+ }
+ return IsomorphismUtilities.isOperatorIsomorphic(op1, op2);
+ }
+
+ private ILogicalOperator copyAndSubstituteVar(ILogicalOperator op, ILogicalOperator argOp)
+ throws AlgebricksException {
+ ILogicalOperator newOp = IsomorphismOperatorVisitor.deepCopy(argOp);
+ variableMapping.clear();
+ IsomorphismUtilities.mapVariablesTopDown(op, argOp, variableMapping);
+
+ List<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
+ if (argOp.getInputs().size() > 0)
+ for (int i = 0; i < argOp.getInputs().size(); i++)
+ VariableUtilities.getLiveVariables(argOp.getInputs().get(i).getValue(), liveVars);
+ List<LogicalVariable> producedVars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(argOp, producedVars);
+ List<LogicalVariable> producedVarsNew = new ArrayList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(op, producedVarsNew);
+
+ if (producedVars.size() != producedVarsNew.size())
+ return newOp;
+ for (Entry<LogicalVariable, LogicalVariable> map : variableMapping.entrySet()) {
+ if (liveVars.contains(map.getKey())) {
+ VariableUtilities.substituteVariables(newOp, map.getKey(), map.getValue(), null);
+ }
+ }
+ for (int i = 0; i < producedVars.size(); i++)
+ VariableUtilities.substituteVariables(newOp, producedVars.get(i), producedVarsNew.get(i), null);
+ return newOp;
+ }
+
+ public List<Pair<LogicalVariable, ILogicalExpression>> getPairList(List<LogicalVariable> vars,
+ List<Mutable<ILogicalExpression>> exprs) throws AlgebricksException {
+ List<Pair<LogicalVariable, ILogicalExpression>> list = new ArrayList<Pair<LogicalVariable, ILogicalExpression>>();
+ if (vars.size() != exprs.size())
+ throw new AlgebricksException("variable list size does not equal to expression list size ");
+ for (int i = 0; i < vars.size(); i++) {
+ list.add(new Pair<LogicalVariable, ILogicalExpression>(vars.get(i), exprs.get(i).getValue()));
+ }
+ return list;
+ }
+
+ private static ILogicalOperator deepCopy(ILogicalOperator op) throws AlgebricksException {
+ OperatorDeepCopyVisitor visitor = new OperatorDeepCopyVisitor();
+ return op.accept(visitor, null);
+ }
+
+ private static ILogicalPlan deepCopy(ILogicalPlan plan) throws AlgebricksException {
+ List<Mutable<ILogicalOperator>> roots = plan.getRoots();
+ List<Mutable<ILogicalOperator>> newRoots = new ArrayList<Mutable<ILogicalOperator>>();
+ for (Mutable<ILogicalOperator> opRef : roots)
+ newRoots.add(new MutableObject<ILogicalOperator>(bottomUpCopyOperators(opRef.getValue())));
+ return new ALogicalPlanImpl(newRoots);
+ }
+
+ private static ILogicalOperator bottomUpCopyOperators(ILogicalOperator op) throws AlgebricksException {
+ ILogicalOperator newOp = deepCopy(op);
+ newOp.getInputs().clear();
+ for (Mutable<ILogicalOperator> child : op.getInputs())
+ newOp.getInputs().add(new MutableObject<ILogicalOperator>(bottomUpCopyOperators(child.getValue())));
+ return newOp;
+ }
+
+ private static boolean variableEqual(LogicalVariable var, LogicalVariable varArg) {
+ if (var == null && varArg == null)
+ return true;
+ if (var.equals(varArg))
+ return true;
+ else
+ return false;
+ }
+
+ private static class OperatorDeepCopyVisitor implements ILogicalOperatorVisitor<ILogicalOperator, Void> {
+
+ @Override
+ public ILogicalOperator visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
+ ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
+ ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ newList.addAll(op.getVariables());
+ deepCopyExpressionRefs(newExpressions, op.getExpressions());
+ return new AggregateOperator(newList, newExpressions);
+ }
+
+ @Override
+ public ILogicalOperator visitRunningAggregateOperator(RunningAggregateOperator op, Void arg)
+ throws AlgebricksException {
+ ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
+ ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ newList.addAll(op.getVariables());
+ deepCopyExpressionRefs(newExpressions, op.getExpressions());
+ return new RunningAggregateOperator(newList, newExpressions);
+ }
+
+ @Override
+ public ILogicalOperator visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg)
+ throws AlgebricksException {
+ return new EmptyTupleSourceOperator();
+ }
+
+ @Override
+ public ILogicalOperator visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decoList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+ ArrayList<ILogicalPlan> newSubplans = new ArrayList<ILogicalPlan>();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getGroupByList())
+ groupByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(pair.first,
+ deepCopyExpressionRef(pair.second)));
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> pair : op.getDecorList())
+ decoList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(pair.first,
+ deepCopyExpressionRef(pair.second)));
+ for (ILogicalPlan plan : op.getNestedPlans()) {
+ newSubplans.add(IsomorphismOperatorVisitor.deepCopy(plan));
+ }
+ return new GroupByOperator(groupByList, decoList, newSubplans);
+ }
+
+ @Override
+ public ILogicalOperator visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+ return new LimitOperator(deepCopyExpressionRef(op.getMaxObjects()).getValue(), deepCopyExpressionRef(
+ op.getOffset()).getValue(), op.isTopmostLimitOp());
+ }
+
+ @Override
+ public ILogicalOperator visitDieOperator(DieOperator op, Void arg) throws AlgebricksException {
+ return new DieOperator(deepCopyExpressionRef(op.getAfterObjects()).getValue());
+ }
+
+ @Override
+ public ILogicalOperator visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
+ return new InnerJoinOperator(deepCopyExpressionRef(op.getCondition()), op.getInputs().get(0), op
+ .getInputs().get(1));
+ }
+
+ @Override
+ public ILogicalOperator visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg)
+ throws AlgebricksException {
+ return new LeftOuterJoinOperator(deepCopyExpressionRef(op.getCondition()), op.getInputs().get(0), op
+ .getInputs().get(1));
+ }
+
+ @Override
+ public ILogicalOperator visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg)
+ throws AlgebricksException {
+ return new NestedTupleSourceOperator(null);
+ }
+
+ @Override
+ public ILogicalOperator visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
+ return new OrderOperator(this.deepCopyOrderAndExpression(op.getOrderExpressions()));
+ }
+
+ @Override
+ public ILogicalOperator visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
+ ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
+ ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ newList.addAll(op.getVariables());
+ deepCopyExpressionRefs(newExpressions, op.getExpressions());
+ return new AssignOperator(newList, newExpressions);
+ }
+
+ @Override
+ public ILogicalOperator visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
+ return new SelectOperator(deepCopyExpressionRef(op.getCondition()));
+ }
+
+ @Override
+ public ILogicalOperator visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
+ ArrayList<LogicalVariable> newList = new ArrayList<LogicalVariable>();
+ newList.addAll(op.getVariables());
+ return new ProjectOperator(newList);
+ }
+
+ @Override
+ public ILogicalOperator visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg)
+ throws AlgebricksException {
+ ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newExpressions, Arrays.asList(op.getExpressions()));
+ return new PartitioningSplitOperator(newExpressions.toArray(new Mutable[0]),
+ op.hasDefault());
+ }
+
+ @Override
+ public ILogicalOperator visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+ return new ReplicateOperator(op.getOutputArity());
+ }
+
+ @Override
+ public ILogicalOperator visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
+ ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
+ ArrayList<LogicalVariable> newOutputList = new ArrayList<LogicalVariable>();
+ newInputList.addAll(op.getInputVariables());
+ newOutputList.addAll(op.getOutputVariables());
+ return new ScriptOperator(op.getScriptDescription(), newInputList, newOutputList);
+ }
+
+ @Override
+ public ILogicalOperator visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+ ArrayList<ILogicalPlan> newSubplans = new ArrayList<ILogicalPlan>();
+ for (ILogicalPlan plan : op.getNestedPlans()) {
+ newSubplans.add(IsomorphismOperatorVisitor.deepCopy(plan));
+ }
+ return new SubplanOperator(newSubplans);
+ }
+
+ @Override
+ public ILogicalOperator visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
+ List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> newVarMap = new ArrayList<Triple<LogicalVariable, LogicalVariable, LogicalVariable>>();
+ List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = op.getVariableMappings();
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple : varMap)
+ newVarMap.add(new Triple<LogicalVariable, LogicalVariable, LogicalVariable>(triple.first,
+ triple.second, triple.third));
+ return new UnionAllOperator(newVarMap);
+ }
+
+ @Override
+ public ILogicalOperator visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
+ return new UnnestOperator(op.getVariable(), deepCopyExpressionRef(op.getExpressionRef()),
+ op.getPositionalVariable(), op.getPositionalVariableType());
+ }
+
+ @Override
+ public ILogicalOperator visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
+ ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
+ newInputList.addAll(op.getVariables());
+ return new UnnestMapOperator(newInputList, deepCopyExpressionRef(op.getExpressionRef()),
+ new ArrayList<Object>(op.getVariableTypes()));
+ }
+
+ @Override
+ public ILogicalOperator visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
+ ArrayList<LogicalVariable> newInputList = new ArrayList<LogicalVariable>();
+ newInputList.addAll(op.getVariables());
+ return new DataSourceScanOperator(newInputList, op.getDataSource());
+ }
+
+ @Override
+ public ILogicalOperator visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
+ ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newExpressions, op.getExpressions());
+ return new DistinctOperator(newExpressions);
+ }
+
+ @Override
+ public ILogicalOperator visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+ return new ExchangeOperator();
+ }
+
+ @Override
+ public ILogicalOperator visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
+ ArrayList<Mutable<ILogicalExpression>> newExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newExpressions, op.getExpressions());
+ return new WriteOperator(newExpressions, op.getDataSink());
+ }
+
+ @Override
+ public ILogicalOperator visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
+ ArrayList<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newKeyExpressions, op.getKeyExpressions());
+ return new WriteResultOperator(op.getDataSource(), deepCopyExpressionRef(op.getPayloadExpression()),
+ newKeyExpressions);
+ }
+
+ @Override
+ public ILogicalOperator visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
+ List<Mutable<ILogicalExpression>> newKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newKeyExpressions, op.getPrimaryKeyExpressions());
+ return new InsertDeleteOperator(op.getDataSource(), deepCopyExpressionRef(op.getPayloadExpression()),
+ newKeyExpressions, op.getOperation());
+ }
+
+ @Override
+ public ILogicalOperator visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg)
+ throws AlgebricksException {
+ List<Mutable<ILogicalExpression>> newPrimaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newPrimaryKeyExpressions, op.getPrimaryKeyExpressions());
+ List<Mutable<ILogicalExpression>> newSecondaryKeyExpressions = new ArrayList<Mutable<ILogicalExpression>>();
+ deepCopyExpressionRefs(newSecondaryKeyExpressions, op.getSecondaryKeyExpressions());
+ return new IndexInsertDeleteOperator(op.getDataSourceIndex(), newPrimaryKeyExpressions,
+ newSecondaryKeyExpressions, op.getOperation());
+ }
+
+ @Override
+ public ILogicalOperator visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
+ return new SinkOperator();
+ }
+
+ private void deepCopyExpressionRefs(List<Mutable<ILogicalExpression>> newExprs,
+ List<Mutable<ILogicalExpression>> oldExprs) {
+ for (Mutable<ILogicalExpression> oldExpr : oldExprs)
+ newExprs.add(new MutableObject<ILogicalExpression>(((AbstractLogicalExpression) oldExpr.getValue())
+ .cloneExpression()));
+ }
+
+ private Mutable<ILogicalExpression> deepCopyExpressionRef(Mutable<ILogicalExpression> oldExpr) {
+ return new MutableObject<ILogicalExpression>(
+ ((AbstractLogicalExpression) oldExpr.getValue()).cloneExpression());
+ }
+
+ private List<Pair<IOrder, Mutable<ILogicalExpression>>> deepCopyOrderAndExpression(
+ List<Pair<IOrder, Mutable<ILogicalExpression>>> ordersAndExprs) {
+ List<Pair<IOrder, Mutable<ILogicalExpression>>> newOrdersAndExprs = new ArrayList<Pair<IOrder, Mutable<ILogicalExpression>>>();
+ for (Pair<IOrder, Mutable<ILogicalExpression>> pair : ordersAndExprs)
+ newOrdersAndExprs.add(new Pair<IOrder, Mutable<ILogicalExpression>>(pair.first,
+ deepCopyExpressionRef(pair.second)));
+ return newOrdersAndExprs;
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
new file mode 100644
index 0000000..21271fd
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismUtilities.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class IsomorphismUtilities {
+
+ public static void mapVariablesTopDown(ILogicalOperator op, ILogicalOperator arg,
+ Map<LogicalVariable, LogicalVariable> variableMapping) throws AlgebricksException {
+ IsomorphismVariableMappingVisitor visitor = new IsomorphismVariableMappingVisitor(variableMapping);
+ op.accept(visitor, arg);
+ }
+
+ public static boolean isOperatorIsomorphic(ILogicalOperator op, ILogicalOperator arg) throws AlgebricksException {
+ IsomorphismOperatorVisitor visitor = new IsomorphismOperatorVisitor();
+ return op.accept(visitor, arg).booleanValue();
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
new file mode 100644
index 0000000..340b855
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismVariableMappingVisitor.java
@@ -0,0 +1,417 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractAssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+
+public class IsomorphismVariableMappingVisitor implements ILogicalOperatorVisitor<Void, ILogicalOperator> {
+
+ private Map<LogicalVariable, LogicalVariable> variableMapping;
+
+ public IsomorphismVariableMappingVisitor(Map<LogicalVariable, LogicalVariable> variableMapping) {
+ this.variableMapping = variableMapping;
+ }
+
+ @Override
+ public Void visitAggregateOperator(AggregateOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapChildren(op, arg);
+ mapVariablesForAbstractAssign(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitRunningAggregateOperator(RunningAggregateOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ mapChildren(op, arg);
+ mapVariablesForAbstractAssign(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitGroupByOperator(GroupByOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapChildren(op, arg);
+ mapVariablesForGroupBy(op, arg);
+ mapVariablesInNestedPlans(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitLimitOperator(LimitOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitDieOperator(DieOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitInnerJoinOperator(InnerJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ ILogicalOperator inputToCreator1 = op.getSourceOperator();
+ NestedTupleSourceOperator nts = (NestedTupleSourceOperator) arg;
+ ILogicalOperator inputToCreator2 = nts.getSourceOperator();
+ inputToCreator1.accept(this, inputToCreator2);
+ return null;
+ }
+
+ @Override
+ public Void visitOrderOperator(OrderOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitAssignOperator(AssignOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapChildren(op, arg);
+ mapVariablesForAbstractAssign(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitSelectOperator(SelectOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitProjectOperator(ProjectOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitReplicateOperator(ReplicateOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitScriptOperator(ScriptOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitSubplanOperator(SubplanOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapChildren(op, arg);
+ mapVariablesInNestedPlans(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitUnionOperator(UnionAllOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapChildren(op, arg);
+ mapVariablesForUnion(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestOperator(UnnestOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestMapOperator(UnnestMapOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitDataScanOperator(DataSourceScanOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitDistinctOperator(DistinctOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitExchangeOperator(ExchangeOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitWriteOperator(WriteOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitWriteResultOperator(WriteResultOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitInsertDeleteOperator(InsertDeleteOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, ILogicalOperator arg)
+ throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitSinkOperator(SinkOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapVariablesStandard(op, arg);
+ return null;
+ }
+
+ private void mapChildren(ILogicalOperator op, ILogicalOperator opArg) throws AlgebricksException {
+ List<Mutable<ILogicalOperator>> inputs = op.getInputs();
+ List<Mutable<ILogicalOperator>> inputsArg = opArg.getInputs();
+ if (inputs.size() != inputsArg.size())
+ throw new AlgebricksException("children are not isomoprhic");
+ for (int i = 0; i < inputs.size(); i++) {
+ ILogicalOperator input = inputs.get(i).getValue();
+ ILogicalOperator inputArg = inputsArg.get(i).getValue();
+ input.accept(this, inputArg);
+ }
+ }
+
+ private void mapVariables(ILogicalOperator left, ILogicalOperator right) throws AlgebricksException {
+ List<LogicalVariable> producedVarLeft = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> producedVarRight = new ArrayList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(left, producedVarLeft);
+ VariableUtilities.getProducedVariables(right, producedVarRight);
+ mapVariables(producedVarLeft, producedVarRight);
+ }
+
+ private void mapVariables(List<LogicalVariable> variablesLeft, List<LogicalVariable> variablesRight) {
+ if (variablesLeft.size() != variablesRight.size())
+ return;
+ int size = variablesLeft.size();
+ for (int i = 0; i < size; i++) {
+ LogicalVariable left = variablesLeft.get(i);
+ LogicalVariable right = variablesRight.get(i);
+ variableMapping.put(right, left);
+ }
+ }
+
+ private void mapVariablesForAbstractAssign(ILogicalOperator left, ILogicalOperator right)
+ throws AlgebricksException {
+ AbstractAssignOperator leftOp = (AbstractAssignOperator) left;
+ AbstractAssignOperator rightOp = (AbstractAssignOperator) right;
+ List<LogicalVariable> producedVarLeft = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> producedVarRight = new ArrayList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(left, producedVarLeft);
+ VariableUtilities.getProducedVariables(right, producedVarRight);
+ mapVariablesForAbstractAssign(producedVarLeft, leftOp.getExpressions(), producedVarRight,
+ rightOp.getExpressions());
+ }
+
+ private void mapVariablesForGroupBy(ILogicalOperator left, ILogicalOperator right) throws AlgebricksException {
+ GroupByOperator leftOp = (GroupByOperator) left;
+ GroupByOperator rightOp = (GroupByOperator) right;
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs = leftOp.getGroupByList();
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs = rightOp.getGroupByList();
+ mapVarExprPairList(leftPairs, rightPairs);
+ leftPairs = leftOp.getDecorList();
+ rightPairs = rightOp.getDecorList();
+ mapVarExprPairList(leftPairs, rightPairs);
+ }
+
+ private void mapVarExprPairList(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> leftPairs,
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> rightPairs) {
+ if (leftPairs.size() != rightPairs.size())
+ return;
+ for (int i = 0; i < leftPairs.size(); i++) {
+ ILogicalExpression exprLeft = leftPairs.get(i).second.getValue();
+ LogicalVariable leftVar = leftPairs.get(i).first;
+ for (int j = 0; j < leftPairs.size(); j++) {
+ ILogicalExpression exprRight = copyExpressionAndSubtituteVars(rightPairs.get(j).second).getValue();
+ if (exprLeft.equals(exprRight)) {
+ LogicalVariable rightVar = rightPairs.get(j).first;
+ if (rightVar != null && leftVar != null) {
+ variableMapping.put(rightVar, leftVar);
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ private void mapVariablesForAbstractAssign(List<LogicalVariable> variablesLeft,
+ List<Mutable<ILogicalExpression>> exprsLeft, List<LogicalVariable> variablesRight,
+ List<Mutable<ILogicalExpression>> exprsRight) {
+ if (variablesLeft.size() != variablesRight.size())
+ return;
+ int size = variablesLeft.size();
+ for (int i = 0; i < size; i++) {
+ ILogicalExpression exprLeft = exprsLeft.get(i).getValue();
+ LogicalVariable left = variablesLeft.get(i);
+ for (int j = 0; j < size; j++) {
+ ILogicalExpression exprRight = copyExpressionAndSubtituteVars(exprsRight.get(j)).getValue();
+ if (exprLeft.equals(exprRight)) {
+ LogicalVariable right = variablesRight.get(j);
+ variableMapping.put(right, left);
+ break;
+ }
+ }
+ }
+ }
+
+ private void mapVariablesInNestedPlans(ILogicalOperator opOrigin, ILogicalOperator arg) throws AlgebricksException {
+ AbstractOperatorWithNestedPlans op = (AbstractOperatorWithNestedPlans) opOrigin;
+ AbstractOperatorWithNestedPlans argOp = (AbstractOperatorWithNestedPlans) arg;
+ List<ILogicalPlan> plans = op.getNestedPlans();
+ List<ILogicalPlan> plansArg = argOp.getNestedPlans();
+ if (plans.size() != plansArg.size())
+ return;
+ for (int i = 0; i < plans.size(); i++) {
+ List<Mutable<ILogicalOperator>> roots = plans.get(i).getRoots();
+ List<Mutable<ILogicalOperator>> rootsArg = plansArg.get(i).getRoots();
+ if (roots.size() != rootsArg.size())
+ return;
+ for (int j = 0; j < roots.size(); j++) {
+ ILogicalOperator topOp1 = roots.get(j).getValue();
+ ILogicalOperator topOp2 = rootsArg.get(j).getValue();
+ topOp1.accept(this, topOp2);
+ }
+ }
+ }
+
+ private void mapVariablesStandard(ILogicalOperator op, ILogicalOperator arg) throws AlgebricksException {
+ mapChildren(op, arg);
+ mapVariables(op, arg);
+ }
+
+ private Mutable<ILogicalExpression> copyExpressionAndSubtituteVars(Mutable<ILogicalExpression> expr) {
+ ILogicalExpression copy = ((AbstractLogicalExpression) expr.getValue()).cloneExpression();
+ for (Entry<LogicalVariable, LogicalVariable> entry : variableMapping.entrySet())
+ copy.substituteVar(entry.getKey(), entry.getValue());
+ return new MutableObject<ILogicalExpression>(copy);
+ }
+
+ private void mapVariablesForUnion(ILogicalOperator op, ILogicalOperator arg) {
+ UnionAllOperator union = (UnionAllOperator) op;
+ UnionAllOperator unionArg = (UnionAllOperator) arg;
+ mapVarTripleList(union.getVariableMappings(), unionArg.getVariableMappings());
+ }
+
+ private void mapVarTripleList(List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> leftTriples,
+ List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> rightTriples) {
+ if (leftTriples.size() != rightTriples.size())
+ return;
+ for (int i = 0; i < leftTriples.size(); i++) {
+ LogicalVariable leftFirstVar = leftTriples.get(i).first;
+ LogicalVariable leftSecondVar = leftTriples.get(i).second;
+ LogicalVariable leftThirdVar = leftTriples.get(i).third;
+ for (int j = 0; j < rightTriples.size(); j++) {
+ LogicalVariable rightFirstVar = rightTriples.get(j).first;
+ LogicalVariable rightSecondVar = rightTriples.get(j).second;
+ LogicalVariable rightThirdVar = rightTriples.get(j).third;
+ if (varEquivalent(leftFirstVar, rightFirstVar) && varEquivalent(leftSecondVar, rightSecondVar)) {
+ variableMapping.put(rightThirdVar, leftThirdVar);
+ break;
+ }
+ }
+ }
+ }
+
+ private boolean varEquivalent(LogicalVariable left, LogicalVariable right) {
+ if (variableMapping.get(right) == null)
+ return false;
+ if (variableMapping.get(right).equals(left))
+ return true;
+ else
+ return false;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
new file mode 100644
index 0000000..30ad3ab
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/LogicalPropertiesVisitor.java
@@ -0,0 +1,321 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractAssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LogicalPropertiesVectorImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public class LogicalPropertiesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> {
+
+ public static void computeLogicalPropertiesDFS(ILogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ LogicalPropertiesVisitor visitor = new LogicalPropertiesVisitor();
+ computeLogicalPropertiesRec(op, visitor, context);
+ }
+
+ private static void computeLogicalPropertiesRec(ILogicalOperator op, LogicalPropertiesVisitor visitor,
+ IOptimizationContext context) throws AlgebricksException {
+ for (Mutable<ILogicalOperator> ref : op.getInputs()) {
+ computeLogicalPropertiesRec(ref.getValue(), visitor, context);
+ }
+ op.accept(visitor, context);
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest("Logical properties visitor for " + op + ": "
+ + context.getLogicalPropertiesVector(op) + "\n");
+ }
+ }
+
+ @Override
+ public Void visitAggregateOperator(AggregateOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitAssignOperator(AssignOperator op, IOptimizationContext context) throws AlgebricksException {
+ visitAssignment(op, context);
+ return null;
+ }
+
+ @Override
+ public Void visitDataScanOperator(DataSourceScanOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitDistinctOperator(DistinctOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, IOptimizationContext arg)
+ throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitExchangeOperator(ExchangeOperator op, IOptimizationContext arg) throws AlgebricksException {
+ propagateCardinalityAndFrameNumber(op, arg);
+ return null;
+ }
+
+ @Override
+ public Void visitGroupByOperator(GroupByOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitInnerJoinOperator(InnerJoinOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, IOptimizationContext arg)
+ throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitLimitOperator(LimitOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitDieOperator(DieOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, IOptimizationContext arg)
+ throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitOrderOperator(OrderOperator op, IOptimizationContext arg) throws AlgebricksException {
+ Object annot1 = op.getAnnotations().get(OperatorAnnotations.CARDINALITY);
+ if (annot1 == null) {
+ return null;
+ }
+ Integer m = (Integer) annot1;
+ LogicalPropertiesVectorImpl v = new LogicalPropertiesVectorImpl();
+ v.setNumberOfTuples(m);
+ Object annot2 = op.getAnnotations().get(OperatorAnnotations.MAX_NUMBER_FRAMES);
+ if (annot2 != null) {
+ Integer f = (Integer) annot2;
+ v.setMaxOutputFrames(f);
+ }
+ arg.putLogicalPropertiesVector(op, v);
+ return null;
+ }
+
+ @Override
+ public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, IOptimizationContext arg)
+ throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitProjectOperator(ProjectOperator op, IOptimizationContext context) throws AlgebricksException {
+ propagateCardinalityAndFrameNumber(op, context);
+ return null;
+ }
+
+ @Override
+ public Void visitReplicateOperator(ReplicateOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitRunningAggregateOperator(RunningAggregateOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ visitAssignment(op, context);
+ return null;
+ }
+
+ @Override
+ public Void visitScriptOperator(ScriptOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitSelectOperator(SelectOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitSubplanOperator(SubplanOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitUnionOperator(UnionAllOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestMapOperator(UnnestMapOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestOperator(UnnestOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitWriteOperator(WriteOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitWriteResultOperator(WriteResultOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitInsertDeleteOperator(InsertDeleteOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, IOptimizationContext arg)
+ throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Void visitSinkOperator(SinkOperator op, IOptimizationContext arg) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ private LogicalPropertiesVectorImpl propagateCardinality(ILogicalOperator op, IOptimizationContext context) {
+ ILogicalOperator op0 = op.getInputs().get(0).getValue();
+ ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
+ if (v0 == null) {
+ return null;
+ }
+ LogicalPropertiesVectorImpl v = new LogicalPropertiesVectorImpl();
+ v.setNumberOfTuples(v0.getNumberOfTuples());
+ context.putLogicalPropertiesVector(op, v);
+ return v;
+ }
+
+ private void visitAssignment(AbstractAssignOperator op, IOptimizationContext context) throws AlgebricksException {
+ LogicalPropertiesVectorImpl v = propagateCardinality(op, context);
+ if (v != null && v.getNumberOfTuples() != null) {
+ IVariableEvalSizeEnvironment varSizeEnv = context.getVariableEvalSizeEnvironment();
+ IExpressionEvalSizeComputer evalSize = context.getExpressionEvalSizeComputer();
+ if (evalSize != null) {
+ ILogicalOperator op0 = op.getInputs().get(0).getValue();
+ ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
+ if (v0 != null) {
+ long frames0 = v0.getMaxOutputFrames();
+ long overhead = 0; // added per tuple
+ for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+ int sz = evalSize.getEvalSize(exprRef.getValue(), varSizeEnv);
+ if (sz == -1) {
+ return;
+ }
+ overhead += sz;
+ }
+ int frameSize = context.getFrameSize();
+ if (frameSize > 0) {
+ long sz = frames0 * frameSize + overhead * v.getNumberOfTuples();
+ int frames1 = (int) (sz / frameSize);
+ if (sz % frameSize > 0) {
+ frames1++;
+ }
+ v.setMaxOutputFrames(frames1);
+ }
+ }
+ }
+ }
+ }
+
+ public void propagateCardinalityAndFrameNumber(ILogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ LogicalPropertiesVectorImpl v = propagateCardinality(op, context);
+ // propagate also max number of frames (conservatively)
+ ILogicalOperator op0 = op.getInputs().get(0).getValue();
+ ILogicalPropertiesVector v0 = context.getLogicalPropertiesVector(op0);
+ if (v0 != null) {
+ v.setMaxOutputFrames(v0.getMaxOutputFrames());
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
new file mode 100644
index 0000000..cd05624
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/ProducedVariableVisitor.java
@@ -0,0 +1,248 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+
+public class ProducedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
+
+ private Collection<LogicalVariable> producedVariables;
+
+ public ProducedVariableVisitor(Collection<LogicalVariable> producedVariables) throws AlgebricksException {
+ this.producedVariables = producedVariables;
+ }
+
+ @Override
+ public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
+ producedVariables.addAll(op.getVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
+ producedVariables.addAll(op.getVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
+ producedVariables.addAll(op.getVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitDistinctOperator(DistinctOperator op, Void arg) {
+ return null;
+ }
+
+ @Override
+ public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
+ }
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
+ if (p.first != null) {
+ producedVariables.add(p.first);
+ }
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
+ if (p.first != null) {
+ producedVariables.add(p.first);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitDieOperator(DieOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
+ producedVariables.addAll(op.getVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
+ List<LogicalVariable> usedVariables = new ArrayList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op, usedVariables);
+ for (LogicalVariable v : op.getOutputVariables()) {
+ if (!usedVariables.contains(v)) {
+ producedVariables.add(v);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ VariableUtilities.getLiveVariables(r.getValue(), producedVariables);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : op.getVariableMappings()) {
+ producedVariables.add(t.third);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
+ producedVariables.addAll(op.getVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
+ producedVariables.addAll(op.getVariables());
+ LogicalVariable positionalVariable = op.getPositionalVariable();
+ if (positionalVariable != null) {
+ if (!producedVariables.contains(positionalVariable))
+ producedVariables.add(positionalVariable);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
new file mode 100644
index 0000000..6aa3796
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SchemaVariableVisitor.java
@@ -0,0 +1,270 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.Collection;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class SchemaVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
+
+ private Collection<LogicalVariable> schemaVariables;
+
+ public SchemaVariableVisitor(Collection<LogicalVariable> schemaVariables) {
+ this.schemaVariables = schemaVariables;
+ }
+
+ @Override
+ public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
+ schemaVariables.addAll(op.getVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitAssignOperator(AssignOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitExchangeOperator(ExchangeOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
+ }
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
+ if (p.first != null) {
+ schemaVariables.add(p.first);
+ }
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
+ if (p.first != null) {
+ schemaVariables.add(p.first);
+ } else {
+ ILogicalExpression e = p.second.getValue();
+ if (e.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ schemaVariables.add(((VariableReferenceExpression) e).getVariableReference());
+ }
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitLimitOperator(LimitOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitDieOperator(DieOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) throws AlgebricksException {
+ VariableUtilities.getLiveVariables(op.getSourceOperator(), schemaVariables);
+ return null;
+ }
+
+ @Override
+ public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitProjectOperator(ProjectOperator op, Void arg) throws AlgebricksException {
+ schemaVariables.addAll(op.getVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
+ // VariableUtilities.getProducedVariables(op, schemaVariables);
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitScriptOperator(ScriptOperator op, Void arg) throws AlgebricksException {
+ schemaVariables.addAll(op.getOutputVariables());
+ return null;
+ }
+
+ @Override
+ public Void visitSelectOperator(SelectOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+ for (Mutable<ILogicalOperator> c : op.getInputs()) {
+ VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
+ }
+ VariableUtilities.getProducedVariables(op, schemaVariables);
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ VariableUtilities.getLiveVariables(r.getValue(), schemaVariables);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitUnionOperator(UnionAllOperator op, Void arg) throws AlgebricksException {
+ VariableUtilities.getProducedVariables(op, schemaVariables);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestOperator(UnnestOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitWriteOperator(WriteOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitWriteResultOperator(WriteResultOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ private void standardLayout(ILogicalOperator op) throws AlgebricksException {
+ for (Mutable<ILogicalOperator> c : op.getInputs()) {
+ VariableUtilities.getLiveVariables(c.getValue(), schemaVariables);
+ }
+ VariableUtilities.getProducedVariables(op, schemaVariables);
+ }
+
+ @Override
+ public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+ @Override
+ public Void visitSinkOperator(SinkOperator op, Void arg) throws AlgebricksException {
+ standardLayout(op);
+ return null;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
new file mode 100644
index 0000000..b85c798
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/SubstituteVariableVisitor.java
@@ -0,0 +1,412 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+
+public class SubstituteVariableVisitor implements ILogicalOperatorVisitor<Void, Pair<LogicalVariable, LogicalVariable>> {
+
+ private final boolean goThroughNts;
+ private final ITypingContext ctx;
+
+ public SubstituteVariableVisitor(boolean goThroughNts, ITypingContext ctx) {
+ this.goThroughNts = goThroughNts;
+ this.ctx = ctx;
+ }
+
+ @Override
+ public Void visitAggregateOperator(AggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ List<LogicalVariable> variables = op.getVariables();
+ int n = variables.size();
+ for (int i = 0; i < n; i++) {
+ if (variables.get(i).equals(pair.first)) {
+ variables.set(i, pair.second);
+ } else {
+ op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
+ }
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitAssignOperator(AssignOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ List<LogicalVariable> variables = op.getVariables();
+ int n = variables.size();
+ for (int i = 0; i < n; i++) {
+ if (variables.get(i).equals(pair.first)) {
+ variables.set(i, pair.second);
+ } else {
+ op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
+ }
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitDataScanOperator(DataSourceScanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ List<LogicalVariable> variables = op.getVariables();
+ for (int i = 0; i < variables.size(); i++) {
+ if (variables.get(i) == pair.first) {
+ variables.set(i, pair.second);
+ return null;
+ }
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitDistinctOperator(DistinctOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
+ eRef.getValue().substituteVar(pair.first, pair.second);
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
+ // does not use any variable
+ return null;
+ }
+
+ @Override
+ public Void visitExchangeOperator(ExchangeOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
+ // does not use any variable
+ return null;
+ }
+
+ @Override
+ public Void visitGroupByOperator(GroupByOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ subst(pair.first, pair.second, op.getGroupByList());
+ subst(pair.first, pair.second, op.getDecorList());
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
+ pair.second, goThroughNts, ctx);
+ }
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitInnerJoinOperator(InnerJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ op.getCondition().getValue().substituteVar(pair.first, pair.second);
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ op.getCondition().getValue().substituteVar(pair.first, pair.second);
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitLimitOperator(LimitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ op.getMaxObjects().getValue().substituteVar(pair.first, pair.second);
+ ILogicalExpression offset = op.getOffset().getValue();
+ if (offset != null) {
+ offset.substituteVar(pair.first, pair.second);
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitDieOperator(DieOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ op.getAfterObjects().getValue().substituteVar(pair.first, pair.second);
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Void visitOrderOperator(OrderOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
+ oe.second.getValue().substituteVar(pair.first, pair.second);
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+ e.getValue().substituteVar(pair.first, pair.second);
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitProjectOperator(ProjectOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ List<LogicalVariable> usedVariables = op.getVariables();
+ int n = usedVariables.size();
+ for (int i = 0; i < n; i++) {
+ LogicalVariable v = usedVariables.get(i);
+ if (v.equals(pair.first)) {
+ usedVariables.set(i, pair.second);
+ }
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitRunningAggregateOperator(RunningAggregateOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ List<LogicalVariable> variables = op.getVariables();
+ int n = variables.size();
+ for (int i = 0; i < n; i++) {
+ if (variables.get(i).equals(pair.first)) {
+ variables.set(i, pair.second);
+ } else {
+ op.getExpressions().get(i).getValue().substituteVar(pair.first, pair.second);
+ }
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitScriptOperator(ScriptOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ substInArray(op.getInputVariables(), pair.first, pair.second);
+ substInArray(op.getOutputVariables(), pair.first, pair.second);
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitSelectOperator(SelectOperator op, Pair<LogicalVariable, LogicalVariable> pair) {
+ op.getCondition().getValue().substituteVar(pair.first, pair.second);
+ return null;
+ }
+
+ @Override
+ public Void visitSubplanOperator(SubplanOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), pair.first,
+ pair.second, goThroughNts, ctx);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitUnionOperator(UnionAllOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = op.getVariableMappings();
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : varMap) {
+ if (t.first.equals(pair.first)) {
+ t.first = pair.second;
+ }
+ if (t.second.equals(pair.first)) {
+ t.second = pair.second;
+ }
+ if (t.third.equals(pair.first)) {
+ t.third = pair.second;
+ }
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestMapOperator(UnnestMapOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ List<LogicalVariable> variables = op.getVariables();
+ for (int i = 0; i < variables.size(); i++) {
+ if (variables.get(i) == pair.first) {
+ variables.set(i, pair.second);
+ return null;
+ }
+ }
+ op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestOperator(UnnestOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ List<LogicalVariable> variables = op.getVariables();
+ for (int i = 0; i < variables.size(); i++) {
+ if (variables.get(i) == pair.first) {
+ variables.set(i, pair.second);
+ return null;
+ }
+ }
+ op.getExpressionRef().getValue().substituteVar(pair.first, pair.second);
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitWriteOperator(WriteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+ e.getValue().substituteVar(pair.first, pair.second);
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitWriteResultOperator(WriteResultOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
+ for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
+ e.getValue().substituteVar(pair.first, pair.second);
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ private void subst(LogicalVariable v1, LogicalVariable v2,
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> varExprPairList) {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : varExprPairList) {
+ if (ve.first != null && ve.first.equals(v1)) {
+ ve.first = v2;
+ return;
+ }
+ ve.second.getValue().substituteVar(v1, v2);
+ }
+ }
+
+ private void substInArray(ArrayList<LogicalVariable> varArray, LogicalVariable v1, LogicalVariable v2) {
+ for (int i = 0; i < varArray.size(); i++) {
+ LogicalVariable v = varArray.get(i);
+ if (v == v1) {
+ varArray.set(i, v2);
+ }
+ }
+ }
+
+ @Override
+ public Void visitReplicateOperator(ReplicateOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+ throws AlgebricksException {
+ op.substituteVar(arg.first, arg.second);
+ return null;
+ }
+
+ @Override
+ public Void visitInsertDeleteOperator(InsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ op.getPayloadExpression().getValue().substituteVar(pair.first, pair.second);
+ for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+ e.getValue().substituteVar(pair.first, pair.second);
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+ e.getValue().substituteVar(pair.first, pair.second);
+ }
+ for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+ e.getValue().substituteVar(pair.first, pair.second);
+ }
+ substVarTypes(op, pair);
+ return null;
+ }
+
+ @Override
+ public Void visitSinkOperator(SinkOperator op, Pair<LogicalVariable, LogicalVariable> pair)
+ throws AlgebricksException {
+ return null;
+ }
+
+ private void substVarTypes(ILogicalOperator op, Pair<LogicalVariable, LogicalVariable> arg)
+ throws AlgebricksException {
+ if (ctx == null) {
+ return;
+ }
+ IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(op);
+ env.substituteProducedVariable(arg.first, arg.second);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
new file mode 100644
index 0000000..d9d6a6a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/UsedVariableVisitor.java
@@ -0,0 +1,294 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+
+public class UsedVariableVisitor implements ILogicalOperatorVisitor<Void, Void> {
+
+ private Collection<LogicalVariable> usedVariables;
+
+ public UsedVariableVisitor(Collection<LogicalVariable> usedVariables) {
+ this.usedVariables = usedVariables;
+ }
+
+ @Override
+ public Void visitAggregateOperator(AggregateOperator op, Void arg) {
+ for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+ exprRef.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitAssignOperator(AssignOperator op, Void arg) {
+ for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+ exprRef.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitDataScanOperator(DataSourceScanOperator op, Void arg) {
+ // does not use any variable
+ return null;
+ }
+
+ @Override
+ public Void visitDistinctOperator(DistinctOperator op, Void arg) {
+ for (Mutable<ILogicalExpression> eRef : op.getExpressions()) {
+ eRef.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Void arg) {
+ // does not use any variable
+ return null;
+ }
+
+ @Override
+ public Void visitExchangeOperator(ExchangeOperator op, Void arg) {
+ // does not use any variable
+ return null;
+ }
+
+ @Override
+ public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
+ }
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getGroupByList()) {
+ g.second.getValue().getUsedVariables(usedVariables);
+ }
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> g : op.getDecorList()) {
+ g.second.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) {
+ op.getCondition().getValue().getUsedVariables(usedVariables);
+ return null;
+ }
+
+ @Override
+ public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) {
+ op.getCondition().getValue().getUsedVariables(usedVariables);
+ return null;
+ }
+
+ @Override
+ public Void visitLimitOperator(LimitOperator op, Void arg) {
+ op.getMaxObjects().getValue().getUsedVariables(usedVariables);
+ ILogicalExpression offsetExpr = op.getOffset().getValue();
+ if (offsetExpr != null) {
+ offsetExpr.getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitDieOperator(DieOperator op, Void arg) {
+ op.getAfterObjects().getValue().getUsedVariables(usedVariables);
+ return null;
+ }
+
+ @Override
+ public Void visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Void arg) {
+ // does not use any variable
+ return null;
+ }
+
+ @Override
+ public Void visitOrderOperator(OrderOperator op, Void arg) {
+ for (Pair<IOrder, Mutable<ILogicalExpression>> oe : op.getOrderExpressions()) {
+ oe.second.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitPartitioningSplitOperator(PartitioningSplitOperator op, Void arg) {
+ for (Mutable<ILogicalExpression> e : op.getExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitProjectOperator(ProjectOperator op, Void arg) {
+ List<LogicalVariable> parameterVariables = op.getVariables();
+ for (LogicalVariable v : parameterVariables) {
+ if (!usedVariables.contains(v)) {
+ usedVariables.add(v);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) {
+ for (Mutable<ILogicalExpression> exprRef : op.getExpressions()) {
+ exprRef.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitScriptOperator(ScriptOperator op, Void arg) {
+ List<LogicalVariable> parameterVariables = op.getInputVariables();
+ for (LogicalVariable v : parameterVariables) {
+ if (!usedVariables.contains(v)) {
+ usedVariables.add(v);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitSelectOperator(SelectOperator op, Void arg) {
+ op.getCondition().getValue().getUsedVariables(usedVariables);
+ return null;
+ }
+
+ @Override
+ public Void visitSubplanOperator(SubplanOperator op, Void arg) throws AlgebricksException {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ VariableUtilities.getUsedVariablesInDescendantsAndSelf(r.getValue(), usedVariables);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitUnionOperator(UnionAllOperator op, Void arg) {
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> m : op.getVariableMappings()) {
+ if (!usedVariables.contains(m.first)) {
+ usedVariables.add(m.first);
+ }
+ if (!usedVariables.contains(m.second)) {
+ usedVariables.add(m.second);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestMapOperator(UnnestMapOperator op, Void arg) {
+ op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+ return null;
+ }
+
+ @Override
+ public Void visitUnnestOperator(UnnestOperator op, Void arg) {
+ op.getExpressionRef().getValue().getUsedVariables(usedVariables);
+ return null;
+ }
+
+ @Override
+ public Void visitWriteOperator(WriteOperator op, Void arg) {
+ for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
+ expr.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitWriteResultOperator(WriteResultOperator op, Void arg) {
+ op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+ for (Mutable<ILogicalExpression> e : op.getKeyExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitInsertDeleteOperator(InsertDeleteOperator op, Void arg) {
+ op.getPayloadExpression().getValue().getUsedVariables(usedVariables);
+ for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Void arg) {
+ for (Mutable<ILogicalExpression> e : op.getPrimaryKeyExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ for (Mutable<ILogicalExpression> e : op.getSecondaryKeyExpressions()) {
+ e.getValue().getUsedVariables(usedVariables);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visitSinkOperator(SinkOperator op, Void arg) {
+ return null;
+ }
+
+ @Override
+ public Void visitReplicateOperator(ReplicateOperator op, Void arg) throws AlgebricksException {
+ return null;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
new file mode 100644
index 0000000..8e9c5b8
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/logical/visitors/VariableUtilities.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class VariableUtilities {
+
+ public static void getUsedVariables(ILogicalOperator op, Collection<LogicalVariable> usedVariables)
+ throws AlgebricksException {
+ ILogicalOperatorVisitor<Void, Void> visitor = new UsedVariableVisitor(usedVariables);
+ op.accept(visitor, null);
+ }
+
+ public static void getProducedVariables(ILogicalOperator op, Collection<LogicalVariable> producedVariables)
+ throws AlgebricksException {
+ ILogicalOperatorVisitor<Void, Void> visitor = new ProducedVariableVisitor(producedVariables);
+ op.accept(visitor, null);
+ }
+
+ public static void getLiveVariables(ILogicalOperator op, Collection<LogicalVariable> schemaVariables)
+ throws AlgebricksException {
+ ILogicalOperatorVisitor<Void, Void> visitor = new SchemaVariableVisitor(schemaVariables);
+ op.accept(visitor, null);
+ }
+
+ public static void getUsedVariablesInDescendantsAndSelf(ILogicalOperator op, Collection<LogicalVariable> vars)
+ throws AlgebricksException {
+ // DFS traversal
+ VariableUtilities.getUsedVariables(op, vars);
+ for (Mutable<ILogicalOperator> c : op.getInputs()) {
+ getUsedVariablesInDescendantsAndSelf(c.getValue(), vars);
+ }
+ }
+
+ public static void substituteVariables(ILogicalOperator op, LogicalVariable v1, LogicalVariable v2,
+ ITypingContext ctx) throws AlgebricksException {
+ substituteVariables(op, v1, v2, true, ctx);
+ }
+
+ public static void substituteVariables(ILogicalOperator op, LogicalVariable v1, LogicalVariable v2,
+ boolean goThroughNts, ITypingContext ctx) throws AlgebricksException {
+ ILogicalOperatorVisitor<Void, Pair<LogicalVariable, LogicalVariable>> visitor = new SubstituteVariableVisitor(
+ goThroughNts, ctx);
+ op.accept(visitor, new Pair<LogicalVariable, LogicalVariable>(v1, v2));
+ }
+
+ public static <T> boolean varListEqualUnordered(List<T> var, List<T> varArg) {
+ Set<T> varSet = new HashSet<T>();
+ Set<T> varArgSet = new HashSet<T>();
+ varSet.addAll(var);
+ varArgSet.addAll(varArg);
+ return varSet.equals(varArgSet);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
new file mode 100644
index 0000000..761c7f2
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractExchangePOperator.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public abstract class AbstractExchangePOperator extends AbstractPhysicalOperator {
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ Pair<IConnectorDescriptor, TargetConstraint> connPair = createConnectorDescriptor(builder.getJobSpec(), op,
+ opSchema, context);
+ builder.contributeConnectorWithTargetConstraint(op, connPair.first, connPair.second);
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ public abstract Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
new file mode 100644
index 0000000..b9352f3
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractHashJoinPOperator.java
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public abstract class AbstractHashJoinPOperator extends AbstractJoinPOperator {
+
+ protected List<LogicalVariable> keysLeftBranch;
+ protected List<LogicalVariable> keysRightBranch;
+
+ public AbstractHashJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType,
+ List<LogicalVariable> sideLeftOfEqualities, List<LogicalVariable> sideRightOfEqualities) {
+ super(kind, partitioningType);
+ this.keysLeftBranch = sideLeftOfEqualities;
+ this.keysRightBranch = sideRightOfEqualities;
+ }
+
+ public List<LogicalVariable> getKeysLeftBranch() {
+ return keysLeftBranch;
+ }
+
+ public List<LogicalVariable> getKeysRightBranch() {
+ return keysRightBranch;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator iop, IOptimizationContext context)
+ throws AlgebricksException {
+ IPartitioningProperty pp;
+ AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
+
+ if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
+ AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties();
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getInputs().get(1).getValue();
+ IPhysicalPropertiesVector pv1 = op1.getPhysicalOperator().getDeliveredProperties();
+
+ if (pv0 == null || pv1 == null) {
+ pp = null;
+ } else {
+ pp = pv0.getPartitioningProperty();
+ }
+ } else {
+ pp = null;
+ }
+ this.deliveredProperties = new StructuralPropertiesVector(pp, deliveredLocalProperties(iop, context));
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator iop,
+ IPhysicalPropertiesVector reqdByParent) {
+ StructuralPropertiesVector[] pv = new StructuralPropertiesVector[2];
+ // In a cost-based optimizer, we would also try to propagate the
+ // parent's partitioning requirements.
+ AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
+
+ IPartitioningProperty pp1 = null;
+ IPartitioningProperty pp2 = null;
+ if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
+ switch (partitioningType) {
+ case PAIRWISE: {
+ pp1 = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(keysLeftBranch), null);
+ pp2 = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(keysRightBranch), null);
+ break;
+ }
+ case BROADCAST: {
+ pp2 = new BroadcastPartitioningProperty(null);
+ break;
+ }
+ default: {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ pv[0] = new StructuralPropertiesVector(pp1, null);
+ pv[1] = new StructuralPropertiesVector(pp2, null);
+
+ IPartitioningRequirementsCoordinator prc;
+ switch (kind) {
+ case INNER: {
+ prc = IPartitioningRequirementsCoordinator.EQCLASS_PARTITIONING_COORDINATOR;
+ break;
+ }
+ case LEFT_OUTER: {
+ prc = new IPartitioningRequirementsCoordinator() {
+
+ @Override
+ public Pair<Boolean, IPartitioningProperty> coordinateRequirements(
+ IPartitioningProperty requirements, IPartitioningProperty firstDeliveredPartitioning,
+ ILogicalOperator op, IOptimizationContext context) throws AlgebricksException {
+ if (firstDeliveredPartitioning != null
+ && firstDeliveredPartitioning.getPartitioningType() == requirements
+ .getPartitioningType()) {
+ switch (requirements.getPartitioningType()) {
+ case UNORDERED_PARTITIONED: {
+ UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
+ Set<LogicalVariable> set1 = upp1.getColumnSet();
+ UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) requirements;
+ Set<LogicalVariable> modifuppreq = new HashSet<LogicalVariable>();
+ Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
+ Set<LogicalVariable> covered = new HashSet<LogicalVariable>();
+ for (LogicalVariable r : uppreq.getColumnSet()) {
+ EquivalenceClass ecSnd = eqmap.get(r);
+ boolean found = false;
+ int j = 0;
+ for (LogicalVariable rvar : keysRightBranch) {
+ if (rvar == r || ecSnd != null && eqmap.get(rvar) == ecSnd) {
+ found = true;
+ break;
+ }
+ j++;
+ }
+ if (!found) {
+ throw new IllegalStateException("Did not find a variable equivalent to "
+ + r + " among " + keysRightBranch);
+ }
+ LogicalVariable v2 = keysLeftBranch.get(j);
+ EquivalenceClass ecFst = eqmap.get(v2);
+ for (LogicalVariable vset1 : set1) {
+ if (vset1 == v2 || ecFst != null && eqmap.get(vset1) == ecFst) {
+ covered.add(vset1);
+ modifuppreq.add(r);
+ break;
+ }
+ }
+ }
+ if (!covered.equals(set1)) {
+ throw new AlgebricksException("Could not modify " + requirements
+ + " to agree with partitioning property " + firstDeliveredPartitioning
+ + " delivered by previous input operator.");
+ }
+ UnorderedPartitionedProperty upp2 = new UnorderedPartitionedProperty(modifuppreq,
+ requirements.getNodeDomain());
+ return new Pair<Boolean, IPartitioningProperty>(false, upp2);
+ }
+ case ORDERED_PARTITIONED: {
+ throw new NotImplementedException();
+ }
+ }
+ }
+ return new Pair<Boolean, IPartitioningProperty>(true, requirements);
+ }
+ };
+ break;
+ }
+ default: {
+ throw new IllegalStateException();
+ }
+ }
+
+ return new PhysicalRequirements(pv, prc);
+ }
+
+ protected abstract List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op,
+ IOptimizationContext context) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java
new file mode 100644
index 0000000..68e3a51
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractJoinPOperator.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+
+public abstract class AbstractJoinPOperator extends AbstractPhysicalOperator {
+
+ public enum JoinPartitioningType {
+ PAIRWISE, BROADCAST
+ }
+
+ protected final JoinKind kind;
+ protected final JoinPartitioningType partitioningType;
+
+ public AbstractJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType) {
+ this.kind = kind;
+ this.partitioningType = partitioningType;
+ }
+
+ public JoinKind getKind() {
+ return kind;
+ }
+
+ public JoinPartitioningType getPartitioningType() {
+ return partitioningType;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
new file mode 100644
index 0000000..620c161
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPhysicalOperator.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.PlanCompiler;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksCountPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public abstract class AbstractPhysicalOperator implements IPhysicalOperator {
+
+ protected IPhysicalPropertiesVector deliveredProperties;
+ private boolean disableJobGenBelow = false;
+ private Object hostQueryContext;
+
+ @Override
+ public final IPhysicalPropertiesVector getDeliveredProperties() {
+ return deliveredProperties;
+ }
+
+ @Override
+ public String toString() {
+ return getOperatorTag().toString();
+ }
+
+ public void setHostQueryContext(Object context) {
+ this.hostQueryContext = context;
+ }
+
+ public Object getHostQueryContext() {
+ return hostQueryContext;
+ }
+
+ protected PhysicalRequirements emptyUnaryRequirements() {
+ StructuralPropertiesVector[] req = new StructuralPropertiesVector[] { StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR };
+ return new PhysicalRequirements(req, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @Override
+ public void disableJobGenBelowMe() {
+ this.disableJobGenBelow = true;
+ }
+
+ @Override
+ public boolean isJobGenDisabledBelowMe() {
+ return disableJobGenBelow;
+ }
+
+ protected void contributeOpDesc(IHyracksJobBuilder builder, AbstractLogicalOperator op, IOperatorDescriptor opDesc) {
+ if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) {
+ AlgebricksPartitionConstraint apc = new AlgebricksCountPartitionConstraint(1);
+ builder.contributeAlgebricksPartitionConstraint(opDesc, apc);
+ }
+ builder.contributeHyracksOperator(op, opDesc);
+ }
+
+ protected AlgebricksPipeline[] compileSubplans(IOperatorSchema outerPlanSchema,
+ AbstractOperatorWithNestedPlans npOp, IOperatorSchema opSchema, JobGenContext context)
+ throws AlgebricksException {
+ AlgebricksPipeline[] subplans = new AlgebricksPipeline[npOp.getNestedPlans().size()];
+ PlanCompiler pc = new PlanCompiler(context);
+ int i = 0;
+ for (ILogicalPlan p : npOp.getNestedPlans()) {
+ subplans[i++] = buildPipelineWithProjection(p, outerPlanSchema, npOp, opSchema, pc);
+ }
+ return subplans;
+ }
+
+ private AlgebricksPipeline buildPipelineWithProjection(ILogicalPlan p, IOperatorSchema outerPlanSchema,
+ AbstractOperatorWithNestedPlans npOp, IOperatorSchema opSchema, PlanCompiler pc) throws AlgebricksException {
+ if (p.getRoots().size() > 1) {
+ throw new NotImplementedException("Nested plans with several roots are not supported.");
+ }
+ JobSpecification nestedJob = pc.compilePlan(p, outerPlanSchema);
+ ILogicalOperator topOpInSubplan = p.getRoots().get(0).getValue();
+ JobGenContext context = pc.getContext();
+ IOperatorSchema topOpInSubplanScm = context.getSchema(topOpInSubplan);
+ opSchema.addAllVariables(topOpInSubplanScm);
+
+ Map<OperatorDescriptorId, IOperatorDescriptor> opMap = nestedJob.getOperatorMap();
+ if (opMap.size() != 1) {
+ throw new AlgebricksException(
+ "Attempting to construct a nested plan with "
+ + opMap.size()
+ + " operator descriptors. Currently, nested plans can only consist in linear pipelines of Asterix micro operators.");
+ }
+
+ for (OperatorDescriptorId oid : opMap.keySet()) {
+ IOperatorDescriptor opd = opMap.get(oid);
+ if (!(opd instanceof AlgebricksMetaOperatorDescriptor)) {
+ throw new AlgebricksException(
+ "Can only generate Hyracks jobs for pipelinable Asterix nested plans, not for "
+ + opd.getClass().getName());
+ }
+ AlgebricksMetaOperatorDescriptor amod = (AlgebricksMetaOperatorDescriptor) opd;
+
+ return amod.getPipeline();
+ // we suppose that the top operator in the subplan already does the
+ // projection for us
+ }
+
+ throw new IllegalStateException();
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
new file mode 100644
index 0000000..84982aa
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPreclusteredGroupByPOperator.java
@@ -0,0 +1,238 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public abstract class AbstractPreclusteredGroupByPOperator extends AbstractPhysicalOperator {
+
+ protected List<LogicalVariable> columnList;
+
+ public AbstractPreclusteredGroupByPOperator(List<LogicalVariable> columnList) {
+ this.columnList = columnList;
+ }
+
+ @Override
+ public String toString() {
+ return getOperatorTag().toString() + columnList;
+ }
+
+ public List<LogicalVariable> getGbyColumns() {
+ return columnList;
+ }
+
+ public void setGbyColumns(List<LogicalVariable> gByColumns) {
+ this.columnList = gByColumns;
+ }
+
+ // Obs: We don't propagate properties corresponding to decors, since they
+ // are func. dep. on the group-by variables.
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
+ GroupByOperator gby = (GroupByOperator) op;
+ ILogicalOperator op2 = gby.getInputs().get(0).getValue();
+ IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
+ IPartitioningProperty pp = childProp.getPartitioningProperty();
+ List<ILocalStructuralProperty> childLocals = childProp.getLocalProperties();
+ if (childLocals != null) {
+ for (ILocalStructuralProperty lsp : childLocals) {
+ boolean failed = false;
+ switch (lsp.getPropertyType()) {
+ case LOCAL_GROUPING_PROPERTY: {
+ LocalGroupingProperty lgp = (LocalGroupingProperty) lsp;
+ Set<LogicalVariable> colSet = new HashSet<LogicalVariable>();
+ for (LogicalVariable v : lgp.getColumnSet()) {
+ LogicalVariable v2 = getLhsGbyVar(gby, v);
+ if (v2 != null) {
+ colSet.add(v2);
+ } else {
+ failed = true;
+ }
+ }
+ if (!failed) {
+ propsLocal.add(new LocalGroupingProperty(colSet));
+ }
+ break;
+ }
+ case LOCAL_ORDER_PROPERTY: {
+ LocalOrderProperty lop = (LocalOrderProperty) lsp;
+ OrderColumn oc = lop.getOrderColumn();
+ LogicalVariable v2 = getLhsGbyVar(gby, oc.getColumn());
+ if (v2 != null) {
+ propsLocal.add(new LocalOrderProperty(new OrderColumn(v2, oc.getOrder())));
+ } else {
+ failed = true;
+ }
+ break;
+ }
+ default: {
+ throw new IllegalStateException();
+ }
+ }
+ if (failed) {
+ break;
+ }
+ }
+ }
+ deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+ List<ILocalStructuralProperty> localProps = null;
+
+ localProps = new ArrayList<ILocalStructuralProperty>(1);
+ Set<LogicalVariable> gbvars = new HashSet<LogicalVariable>(columnList);
+ LocalGroupingProperty groupProp = new LocalGroupingProperty(gbvars, new ArrayList<LogicalVariable>(columnList));
+
+ GroupByOperator gby = (GroupByOperator) op;
+ boolean goon = true;
+ for (ILogicalPlan p : gby.getNestedPlans()) {
+ // try to propagate secondary order requirements from nested
+ // groupings
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
+ if (op1.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+ IPhysicalOperator pop2 = op2.getPhysicalOperator();
+ if (pop2 instanceof AbstractPreclusteredGroupByPOperator) {
+ List<LogicalVariable> sndOrder = ((AbstractPreclusteredGroupByPOperator) pop2).getGbyColumns();
+ groupProp.getColumnSet().addAll(sndOrder);
+ groupProp.getPreferredOrderEnforcer().addAll(sndOrder);
+ goon = false;
+ break;
+ }
+ }
+ }
+ if (!goon) {
+ break;
+ }
+ }
+
+ localProps.add(groupProp);
+
+ if (reqdByParent != null) {
+ // propagate parent requirements
+ List<ILocalStructuralProperty> lpPar = reqdByParent.getLocalProperties();
+ if (lpPar != null) {
+ boolean allOk = true;
+ List<ILocalStructuralProperty> props = new ArrayList<ILocalStructuralProperty>(lpPar.size());
+ for (ILocalStructuralProperty prop : lpPar) {
+ if (prop.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
+ allOk = false;
+ break;
+ }
+ LocalOrderProperty lop = (LocalOrderProperty) prop;
+ LogicalVariable ord = lop.getColumn();
+ Pair<LogicalVariable, Mutable<ILogicalExpression>> p = getGbyPairByRhsVar(gby, ord);
+ if (p == null) {
+ p = getDecorPairByRhsVar(gby, ord);
+ if (p == null) {
+ allOk = false;
+ break;
+ }
+ }
+ ILogicalExpression e = p.second.getValue();
+ if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new IllegalStateException(
+ "Right hand side of group-by assignment should have been normalized to a variable reference.");
+ }
+ LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
+ props.add(new LocalOrderProperty(new OrderColumn(v, lop.getOrder())));
+ }
+ List<FunctionalDependency> fdList = new ArrayList<FunctionalDependency>();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : gby.getDecorList()) {
+ List<LogicalVariable> hd = gby.getGbyVarList();
+ List<LogicalVariable> tl = new ArrayList<LogicalVariable>(1);
+ tl.add(((VariableReferenceExpression) decorPair.second.getValue()).getVariableReference());
+ fdList.add(new FunctionalDependency(hd, tl));
+ }
+ if (allOk
+ && PropertiesUtil.matchLocalProperties(localProps, props,
+ new HashMap<LogicalVariable, EquivalenceClass>(), fdList)) {
+ localProps = props;
+ }
+ }
+ }
+
+ IPartitioningProperty pp = null;
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
+ if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
+ pp = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(columnList), null);
+ }
+ pv[0] = new StructuralPropertiesVector(pp, localProps);
+ return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ private static Pair<LogicalVariable, Mutable<ILogicalExpression>> getGbyPairByRhsVar(GroupByOperator gby,
+ LogicalVariable var) {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) {
+ if (ve.first == var) {
+ return ve;
+ }
+ }
+ return null;
+ }
+
+ private static Pair<LogicalVariable, Mutable<ILogicalExpression>> getDecorPairByRhsVar(GroupByOperator gby,
+ LogicalVariable var) {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getDecorList()) {
+ if (ve.first == var) {
+ return ve;
+ }
+ }
+ return null;
+ }
+
+ private static LogicalVariable getLhsGbyVar(GroupByOperator gby, LogicalVariable var) {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> ve : gby.getGroupByList()) {
+ ILogicalExpression e = ve.second.getValue();
+ if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new IllegalStateException(
+ "Right hand side of group by assignment should have been normalized to a variable reference.");
+ }
+ LogicalVariable v = ((VariableReferenceExpression) e).getVariableReference();
+ if (v == var) {
+ return ve.first;
+ }
+ }
+ return null;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
new file mode 100644
index 0000000..224d94c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractPropagatePropertiesForUsedVariablesPOperator.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+
+public abstract class AbstractPropagatePropertiesForUsedVariablesPOperator extends AbstractPhysicalOperator {
+
+ public void computeDeliveredPropertiesForUsedVariables(ILogicalOperator op, List<LogicalVariable> usedVariables) {
+ ILogicalOperator op2 = op.getInputs().get(0).getValue();
+ IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
+ List<ILocalStructuralProperty> downPropsLocal = op2.getDeliveredPhysicalProperties().getLocalProperties();
+ List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
+ for (ILocalStructuralProperty lsp : downPropsLocal) {
+ LinkedList<LogicalVariable> cols = new LinkedList<LogicalVariable>();
+ lsp.getColumns(cols);
+ if (usedVariables.containsAll(cols)) {
+ propsLocal.add(lsp);
+ }
+ }
+ deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java
new file mode 100644
index 0000000..347d580
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractScanPOperator.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+
+public abstract class AbstractScanPOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
new file mode 100644
index 0000000..ca09b77
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AbstractStableSortPOperator.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public abstract class AbstractStableSortPOperator extends AbstractPhysicalOperator {
+
+ protected OrderColumn[] sortColumns;
+ protected List<ILocalStructuralProperty> orderProps;
+
+ public AbstractStableSortPOperator() {
+ }
+
+ public OrderColumn[] getSortColumns() {
+ return sortColumns;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ // if (orderProps == null) { // to do caching, we need some mechanism to
+ // invalidate cache
+ computeLocalProperties(op);
+ // }
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ StructuralPropertiesVector childProp = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties();
+ deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), orderProps);
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator iop,
+ IPhysicalPropertiesVector reqdByParent) {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
+ if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
+ if (orderProps == null) {
+ computeLocalProperties(op);
+ }
+ StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(
+ IPartitioningProperty.UNPARTITIONED, orderProps) };
+ return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ } else {
+ return emptyUnaryRequirements();
+ }
+ }
+
+ public void computeLocalProperties(ILogicalOperator op) {
+ orderProps = new LinkedList<ILocalStructuralProperty>();
+
+ OrderOperator ord = (OrderOperator) op;
+ for (Pair<IOrder, Mutable<ILogicalExpression>> p : ord.getOrderExpressions()) {
+ ILogicalExpression expr = p.second.getValue();
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+ LogicalVariable var = varRef.getVariableReference();
+ switch (p.first.getKind()) {
+ case ASC: {
+ orderProps.add(new LocalOrderProperty(new OrderColumn(var, OrderKind.ASC)));
+ break;
+ }
+ case DESC: {
+ orderProps.add(new LocalOrderProperty(new OrderColumn(var, OrderKind.DESC)));
+ break;
+ }
+ default: {
+ throw new NotImplementedException();
+ }
+ }
+ } else {
+ throw new IllegalStateException();
+ }
+ }
+
+ int n = orderProps.size();
+ sortColumns = new OrderColumn[n];
+ int i = 0;
+ for (ILocalStructuralProperty prop : orderProps) {
+ sortColumns[i++] = ((LocalOrderProperty) prop).getOrderColumn();
+ }
+ }
+
+ public List<ILocalStructuralProperty> getOrderProperties() {
+ return orderProps;
+ }
+
+ @Override
+ public String toString() {
+ if (orderProps == null) {
+ return getOperatorTag().toString();
+ } else {
+ return getOperatorTag().toString() + " " + orderProps;
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
new file mode 100644
index 0000000..47038ad
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AggregatePOperator.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.AggregateRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class AggregatePOperator extends AbstractPhysicalOperator {
+
+ public AggregatePOperator() {
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.AGGREGATE;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ IPhysicalPropertiesVector childProps = op2.getDeliveredPhysicalProperties();
+ deliveredProperties = new StructuralPropertiesVector(childProps.getPartitioningProperty(),
+ new ArrayList<ILocalStructuralProperty>(0));
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ AggregateOperator aggOp = (AggregateOperator) op;
+ List<LogicalVariable> variables = aggOp.getVariables();
+ List<Mutable<ILogicalExpression>> expressions = aggOp.getExpressions();
+ int[] outColumns = new int[variables.size()];
+ for (int i = 0; i < outColumns.length; i++) {
+ outColumns[i] = opSchema.findVariable(variables.get(i));
+ }
+ IAggregateFunctionFactory[] aggFactories = new IAggregateFunctionFactory[expressions.size()];
+ ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ for (int i = 0; i < aggFactories.length; i++) {
+ AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expressions.get(i)
+ .getValue();
+ aggFactories[i] = exprJobGen.createAggregateFunctionFactory(aggFun,
+ context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
+ }
+
+ AggregateRuntimeFactory runtime = new AggregateRuntimeFactory(aggFactories);
+
+ // contribute one Asterix framewriter
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ builder.contributeMicroOperator(aggOp, runtime, recDesc);
+ // and contribute one edge from its child
+ ILogicalOperator src = aggOp.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, aggOp, 0);
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
new file mode 100644
index 0000000..861765d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/AssignPOperator.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.AssignRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class AssignPOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.ASSIGN;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ ILogicalOperator op2 = op.getInputs().get(0).getValue();
+ deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ AssignOperator assign = (AssignOperator) op;
+ List<LogicalVariable> variables = assign.getVariables();
+ List<Mutable<ILogicalExpression>> expressions = assign.getExpressions();
+ int[] outColumns = new int[variables.size()];
+ for (int i = 0; i < outColumns.length; i++) {
+ outColumns[i] = opSchema.findVariable(variables.get(i));
+ }
+ IEvaluatorFactory[] evalFactories = new IEvaluatorFactory[expressions.size()];
+ ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ for (int i = 0; i < evalFactories.length; i++) {
+ evalFactories[i] = exprJobGen.createEvaluatorFactory(expressions.get(i).getValue(),
+ context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
+ }
+
+ // TODO push projections into the operator
+ int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
+
+ AssignRuntimeFactory runtime = new AssignRuntimeFactory(outColumns, evalFactories, projectionList);
+
+ // contribute one Asterix framewriter
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ builder.contributeMicroOperator(assign, runtime, recDesc);
+ // and contribute one edge from its child
+ ILogicalOperator src = assign.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, assign, 0);
+
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
new file mode 100644
index 0000000..0c780d5
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/BroadcastPOperator.java
@@ -0,0 +1,70 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNReplicatingConnectorDescriptor;
+
+public class BroadcastPOperator extends AbstractExchangePOperator {
+
+ private INodeDomain domain;
+
+ public BroadcastPOperator(INodeDomain domain) {
+ this.domain = domain;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.BROADCAST_EXCHANGE;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ IPartitioningProperty pp = new BroadcastPartitioningProperty(domain);
+ this.deliveredProperties = new StructuralPropertiesVector(pp, op2.getDeliveredPhysicalProperties()
+ .getLocalProperties());
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+ IConnectorDescriptor conn = new MToNReplicatingConnectorDescriptor(spec);
+ return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
new file mode 100644
index 0000000..68a6a24
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/DataSourceScanPOperator.java
@@ -0,0 +1,83 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+
+@SuppressWarnings("rawtypes")
+public class DataSourceScanPOperator extends AbstractScanPOperator {
+
+ private IDataSource<?> dataSource;
+
+ public DataSourceScanPOperator(IDataSource<?> dataSource) {
+ this.dataSource = dataSource;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.DATASOURCE_SCAN;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ // partitioning properties
+ DataSourceScanOperator dssOp = (DataSourceScanOperator) op;
+ IDataSourcePropertiesProvider dspp = dataSource.getPropertiesProvider();
+ deliveredProperties = dspp.computePropertiesVector(dssOp.getVariables());
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ DataSourceScanOperator scan = (DataSourceScanOperator) op;
+ IMetadataProvider mp = context.getMetadataProvider();
+
+ List<LogicalVariable> vars = scan.getVariables();
+ List<LogicalVariable> projectVars = scan.getProjectVariables();
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> p = mp.getScannerRuntime(dataSource, vars,
+ projectVars, scan.isProjectPushed(), context, builder.getJobSpec());
+ builder.contributeHyracksOperator(scan, p.first);
+ if (p.second != null) {
+ builder.contributeAlgebricksPartitionConstraint(p.first, p.second);
+ }
+
+ ILogicalOperator srcExchange = scan.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(srcExchange, 0, scan, 0);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
new file mode 100644
index 0000000..7fc76da
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/EmptyTupleSourcePOperator.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.LinkedList;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class EmptyTupleSourcePOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.EMPTY_TUPLE_SOURCE;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED,
+ new LinkedList<ILocalStructuralProperty>());
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return null;
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ EmptyTupleSourceRuntimeFactory runtime = new EmptyTupleSourceRuntimeFactory();
+ RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
+ builder.contributeMicroOperator(op, runtime, recDesc);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
new file mode 100644
index 0000000..f13b2bb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ExternalGroupByPOperator.java
@@ -0,0 +1,244 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.OperatorSchemaImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.SerializableAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableTableFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+public class ExternalGroupByPOperator extends AbstractPhysicalOperator {
+
+ private int tableSize = 0;
+ private int frameLimit = 0;
+ private List<LogicalVariable> columnSet = new ArrayList<LogicalVariable>();
+
+ public ExternalGroupByPOperator(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList, int frameLimit,
+ int tableSize) {
+ this.tableSize = tableSize;
+ this.frameLimit = frameLimit;
+ computeColumnSet(gbyList);
+ }
+
+ public void computeColumnSet(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList) {
+ columnSet.clear();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
+ ILogicalExpression expr = p.second.getValue();
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression v = (VariableReferenceExpression) expr;
+ columnSet.add(v.getVariableReference());
+ }
+ }
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.EXTERNAL_GROUP_BY;
+ }
+
+ @Override
+ public String toString() {
+ return getOperatorTag().toString() + columnSet;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ public List<LogicalVariable> getGbyColumns() {
+ return columnSet;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ List<ILocalStructuralProperty> propsLocal = new LinkedList<ILocalStructuralProperty>();
+
+ GroupByOperator gOp = (GroupByOperator) op;
+ HashSet<LogicalVariable> columnSet = new HashSet<LogicalVariable>();
+
+ if (!columnSet.isEmpty()) {
+ propsLocal.add(new LocalGroupingProperty(columnSet));
+ }
+ for (ILogicalPlan p : gOp.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ ILogicalOperator rOp = r.getValue();
+ propsLocal.addAll(rOp.getDeliveredPhysicalProperties().getLocalProperties());
+ }
+ }
+
+ ILogicalOperator op2 = op.getInputs().get(0).getValue();
+ IPhysicalPropertiesVector childProp = op2.getDeliveredPhysicalProperties();
+ deliveredProperties = new StructuralPropertiesVector(childProp.getPartitioningProperty(), propsLocal);
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
+ if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
+ StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+ pv[0] = new StructuralPropertiesVector(new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(
+ columnSet), null), null);
+ return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ } else {
+ return emptyUnaryRequirements();
+ }
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ List<LogicalVariable> gbyCols = getGbyColumns();
+ int keys[] = JobGenHelper.variablesToFieldIndexes(gbyCols, inputSchemas[0]);
+ GroupByOperator gby = (GroupByOperator) op;
+ int numFds = gby.getDecorList().size();
+ int fdColumns[] = new int[numFds];
+ int j = 0;
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+ ILogicalExpression expr = p.second.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new AlgebricksException("pre-sorted group-by expects variable references.");
+ }
+ VariableReferenceExpression v = (VariableReferenceExpression) expr;
+ LogicalVariable decor = v.getVariableReference();
+ fdColumns[j++] = inputSchemas[0].findVariable(decor);
+ }
+
+ if (gby.getNestedPlans().size() != 1) {
+ throw new AlgebricksException(
+ "External group-by currently works only for one nested plan with one root containing"
+ + "an aggregate and a nested-tuple-source.");
+ }
+ ILogicalPlan p0 = gby.getNestedPlans().get(0);
+ if (p0.getRoots().size() != 1) {
+ throw new AlgebricksException(
+ "External group-by currently works only for one nested plan with one root containing"
+ + "an aggregate and a nested-tuple-source.");
+ }
+ Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
+ AggregateOperator aggOp = (AggregateOperator) r0.getValue();
+
+ IPartialAggregationTypeComputer partialAggregationTypeComputer = context.getPartialAggregationTypeComputer();
+ List<Object> intermediateTypes = new ArrayList<Object>();
+ int n = aggOp.getExpressions().size();
+ ISerializableAggregateFunctionFactory[] aff = new ISerializableAggregateFunctionFactory[n];
+ int i = 0;
+ ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IVariableTypeEnvironment aggOpInputEnv = context.getTypeEnvironment(aggOp.getInputs().get(0).getValue());
+ IVariableTypeEnvironment outputEnv = context.getTypeEnvironment(op);
+ for (Mutable<ILogicalExpression> exprRef : aggOp.getExpressions()) {
+ AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) exprRef.getValue();
+ aff[i++] = exprJobGen.createSerializableAggregateFunctionFactory(aggFun, aggOpInputEnv, inputSchemas,
+ context);
+ intermediateTypes.add(partialAggregationTypeComputer.getType(aggFun, aggOpInputEnv,
+ context.getMetadataProvider()));
+ }
+
+ int[] keyAndDecFields = new int[keys.length + fdColumns.length];
+ for (i = 0; i < keys.length; ++i) {
+ keyAndDecFields[i] = keys[i];
+ }
+ for (i = 0; i < fdColumns.length; i++) {
+ keyAndDecFields[keys.length + i] = fdColumns[i];
+ }
+
+ List<LogicalVariable> keyAndDecVariables = new ArrayList<LogicalVariable>();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getGroupByList())
+ keyAndDecVariables.add(p.first);
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList())
+ keyAndDecVariables.add(GroupByOperator.getDecorVariable(p));
+
+ for (LogicalVariable var : keyAndDecVariables)
+ aggOpInputEnv.setVarType(var, outputEnv.getVarType(var));
+
+ compileSubplans(inputSchemas[0], gby, opSchema, context);
+ JobSpecification spec = builder.getJobSpec();
+ IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(gbyCols,
+ aggOpInputEnv, context);
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ IBinaryHashFunctionFactory[] hashFunctionFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
+ gbyCols, aggOpInputEnv, context);
+
+ ISerializableAggregateFunctionFactory[] merges = new ISerializableAggregateFunctionFactory[n];
+ List<LogicalVariable> usedVars = new ArrayList<LogicalVariable>();
+ IOperatorSchema[] localInputSchemas = new IOperatorSchema[1];
+ localInputSchemas[0] = new OperatorSchemaImpl();
+ for (i = 0; i < n; i++) {
+ AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
+ .get(i).getValue();
+ aggFun.getUsedVariables(usedVars);
+ }
+ i = 0;
+ for (Object type : intermediateTypes) {
+ aggOpInputEnv.setVarType(usedVars.get(i++), type);
+ }
+ for (LogicalVariable keyVar : keyAndDecVariables)
+ localInputSchemas[0].addVariable(keyVar);
+ for (LogicalVariable usedVar : usedVars)
+ localInputSchemas[0].addVariable(usedVar);
+ for (i = 0; i < n; i++) {
+ AggregateFunctionCallExpression mergeFun = (AggregateFunctionCallExpression) aggOp.getMergeExpressions()
+ .get(i).getValue();
+ merges[i] = exprJobGen.createSerializableAggregateFunctionFactory(mergeFun, aggOpInputEnv,
+ localInputSchemas, context);
+ }
+ IAggregatorDescriptorFactory aggregatorFactory = new SerializableAggregatorDescriptorFactory(aff);
+ IAggregatorDescriptorFactory mergeFactory = new SerializableAggregatorDescriptorFactory(merges);
+
+ ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories);
+ INormalizedKeyComputerFactory normalizedKeyFactory = JobGenHelper.variablesToAscNormalizedKeyComputerFactory(
+ gbyCols, aggOpInputEnv, context);
+ ExternalGroupOperatorDescriptor gbyOpDesc = new ExternalGroupOperatorDescriptor(spec, keyAndDecFields,
+ frameLimit, comparatorFactories, normalizedKeyFactory, aggregatorFactory, mergeFactory,
+ recordDescriptor, new HashSpillableTableFactory(tpcf, tableSize), false);
+
+ contributeOpDesc(builder, gby, gbyOpDesc);
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
new file mode 100644
index 0000000..823ac0b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionExchangePOperator.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.HashSet;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
+
+public class HashPartitionExchangePOperator extends AbstractExchangePOperator {
+
+ private List<LogicalVariable> hashFields;
+ private INodeDomain domain;
+
+ public HashPartitionExchangePOperator(List<LogicalVariable> hashFields, INodeDomain domain) {
+ this.hashFields = hashFields;
+ this.domain = domain;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.HASH_PARTITION_EXCHANGE;
+ }
+
+ public List<LogicalVariable> getHashFields() {
+ return hashFields;
+ }
+
+ public INodeDomain getDomain() {
+ return domain;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ IPartitioningProperty p = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(hashFields), domain);
+ this.deliveredProperties = new StructuralPropertiesVector(p, null);
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public String toString() {
+ return getOperatorTag().toString() + " " + hashFields;
+ }
+
+ @Override
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+ int[] keys = new int[hashFields.size()];
+ IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[hashFields.size()];
+ int i = 0;
+ IBinaryHashFunctionFactoryProvider hashFunProvider = context.getBinaryHashFunctionFactoryProvider();
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ for (LogicalVariable v : hashFields) {
+ keys[i] = opSchema.findVariable(v);
+ hashFunctionFactories[i] = hashFunProvider.getBinaryHashFunctionFactory(env.getVarType(v));
+ ++i;
+ }
+ ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories);
+ IConnectorDescriptor conn = new MToNPartitioningConnectorDescriptor(spec, tpcf);
+ return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
new file mode 100644
index 0000000..a54f116
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HashPartitionMergeExchangePOperator.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
+
+public class HashPartitionMergeExchangePOperator extends AbstractExchangePOperator {
+
+ private List<OrderColumn> orderColumns;
+ private List<LogicalVariable> partitionFields;
+ private INodeDomain domain;
+
+ public HashPartitionMergeExchangePOperator(List<OrderColumn> orderColumns, List<LogicalVariable> partitionFields,
+ INodeDomain domain) {
+ this.orderColumns = orderColumns;
+ this.partitionFields = partitionFields;
+ this.domain = domain;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.HASH_PARTITION_MERGE_EXCHANGE;
+ }
+
+ public List<OrderColumn> getOrderExpressions() {
+ return orderColumns;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ IPartitioningProperty p = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(partitionFields),
+ domain);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ List<ILocalStructuralProperty> op2Locals = op2.getDeliveredPhysicalProperties().getLocalProperties();
+ List<ILocalStructuralProperty> locals = new ArrayList<ILocalStructuralProperty>();
+ for (ILocalStructuralProperty prop : op2Locals) {
+ if (prop.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
+ locals.add(prop);
+ } else {
+ break;
+ }
+ }
+
+ this.deliveredProperties = new StructuralPropertiesVector(p, locals);
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ List<ILocalStructuralProperty> orderProps = new LinkedList<ILocalStructuralProperty>();
+ for (OrderColumn oc : orderColumns) {
+ LogicalVariable var = oc.getColumn();
+ switch (oc.getOrder()) {
+ case ASC: {
+ orderProps.add(new LocalOrderProperty(new OrderColumn(var, OrderKind.ASC)));
+ break;
+ }
+ case DESC: {
+ orderProps.add(new LocalOrderProperty(new OrderColumn(var, OrderKind.DESC)));
+ break;
+ }
+ default: {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(null,
+ orderProps) };
+ return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @Override
+ public String toString() {
+ return getOperatorTag().toString() + " MERGE:" + orderColumns + " HASH:" + partitionFields;
+ }
+
+ @Override
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+ int[] keys = new int[partitionFields.size()];
+ IBinaryHashFunctionFactory[] hashFunctionFactories = new IBinaryHashFunctionFactory[partitionFields.size()];
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ {
+ int i = 0;
+ IBinaryHashFunctionFactoryProvider hashFunProvider = context.getBinaryHashFunctionFactoryProvider();
+ for (LogicalVariable v : partitionFields) {
+ keys[i] = opSchema.findVariable(v);
+ hashFunctionFactories[i] = hashFunProvider.getBinaryHashFunctionFactory(env.getVarType(v));
+ ++i;
+ }
+ }
+ ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(keys, hashFunctionFactories);
+
+ int n = orderColumns.size();
+ int[] sortFields = new int[n];
+ IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[n];
+ {
+ int j = 0;
+ for (OrderColumn oc : orderColumns) {
+ LogicalVariable var = oc.getColumn();
+ sortFields[j] = opSchema.findVariable(var);
+ Object type = env.getVarType(var);
+ IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+ comparatorFactories[j] = bcfp.getBinaryComparatorFactory(type, oc.getOrder());
+ j++;
+ }
+ }
+
+ IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields,
+ comparatorFactories);
+ return new Pair<IConnectorDescriptor, TargetConstraint>(conn, null);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
new file mode 100644
index 0000000..2d3724a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/HybridHashJoinPOperator.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.join.HybridHashJoinOperatorDescriptor;
+
+public class HybridHashJoinPOperator extends AbstractHashJoinPOperator {
+
+ private final int memSizeInFrames;
+ private final int maxInputBuildSizeInFrames;
+ private final int aveRecordsPerFrame;
+ private final double fudgeFactor;
+
+ public HybridHashJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType,
+ List<LogicalVariable> sideLeftOfEqualities, List<LogicalVariable> sideRightOfEqualities,
+ int memSizeInFrames, int maxInputSize0InFrames, int aveRecordsPerFrame, double fudgeFactor) {
+ super(kind, partitioningType, sideLeftOfEqualities, sideRightOfEqualities);
+ this.memSizeInFrames = memSizeInFrames;
+ this.maxInputBuildSizeInFrames = maxInputSize0InFrames;
+ this.aveRecordsPerFrame = aveRecordsPerFrame;
+ this.fudgeFactor = fudgeFactor;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.HYBRID_HASH_JOIN;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ public double getFudgeFactor() {
+ return fudgeFactor;
+ }
+
+ public int getMemSizeInFrames() {
+ return memSizeInFrames;
+ }
+
+ @Override
+ public String toString() {
+ return getOperatorTag().toString() + " " + keysLeftBranch + keysRightBranch;
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ int[] keysLeft = JobGenHelper.variablesToFieldIndexes(keysLeftBranch, inputSchemas[0]);
+ int[] keysRight = JobGenHelper.variablesToFieldIndexes(keysRightBranch, inputSchemas[1]);
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ IBinaryHashFunctionFactory[] hashFunFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
+ keysLeftBranch, env, context);
+ IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keysLeft.length];
+ int i = 0;
+ IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+ for (LogicalVariable v : keysLeftBranch) {
+ Object t = env.getVarType(v);
+ comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, OrderKind.ASC);
+ }
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ JobSpecification spec = builder.getJobSpec();
+ IOperatorDescriptor opDesc = null;
+ try {
+ switch (kind) {
+ case INNER: {
+ opDesc = new HybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
+ maxInputBuildSizeInFrames, aveRecordsPerFrame, getFudgeFactor(), keysLeft, keysRight,
+ hashFunFactories, comparatorFactories, recDescriptor);
+ break;
+ }
+ case LEFT_OUTER: {
+ INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
+ for (int j = 0; j < nullWriterFactories.length; j++) {
+ nullWriterFactories[j] = context.getNullWriterFactory();
+ }
+ opDesc = new HybridHashJoinOperatorDescriptor(spec, getMemSizeInFrames(),
+ maxInputBuildSizeInFrames, aveRecordsPerFrame, getFudgeFactor(), keysLeft, keysRight,
+ hashFunFactories, comparatorFactories, recDescriptor, true, nullWriterFactories);
+ break;
+ }
+ default: {
+ throw new NotImplementedException();
+ }
+ }
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+ contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+ ILogicalOperator src1 = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src1, 0, op, 0);
+ ILogicalOperator src2 = op.getInputs().get(1).getValue();
+ builder.contributeGraphEdge(src2, 0, op, 1);
+ }
+
+ @Override
+ protected List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ return new LinkedList<ILocalStructuralProperty>();
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
new file mode 100644
index 0000000..08406d4
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryHashJoinPOperator.java
@@ -0,0 +1,133 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.join.InMemoryHashJoinOperatorDescriptor;
+
+public class InMemoryHashJoinPOperator extends AbstractHashJoinPOperator {
+
+ private final int tableSize;
+
+ /**
+ * builds on the first operator and probes on the second.
+ */
+
+ public InMemoryHashJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType,
+ List<LogicalVariable> sideLeftOfEqualities, List<LogicalVariable> sideRightOfEqualities, int tableSize) {
+ super(kind, partitioningType, sideLeftOfEqualities, sideRightOfEqualities);
+ this.tableSize = tableSize;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.IN_MEMORY_HASH_JOIN;
+ }
+
+ @Override
+ public String toString() {
+ return getOperatorTag().toString() + " " + keysLeftBranch + keysRightBranch;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ int[] keysLeft = JobGenHelper.variablesToFieldIndexes(keysLeftBranch, inputSchemas[0]);
+ int[] keysRight = JobGenHelper.variablesToFieldIndexes(keysRightBranch, inputSchemas[1]);
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ IBinaryHashFunctionFactory[] hashFunFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(
+ keysLeftBranch, env, context);
+ IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keysLeft.length];
+ int i = 0;
+ IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+ for (LogicalVariable v : keysLeftBranch) {
+ Object t = env.getVarType(v);
+ comparatorFactories[i++] = bcfp.getBinaryComparatorFactory(t, OrderKind.ASC);
+ }
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ JobSpecification spec = builder.getJobSpec();
+ IOperatorDescriptor opDesc = null;
+
+ switch (kind) {
+ case INNER: {
+ opDesc = new InMemoryHashJoinOperatorDescriptor(spec, keysLeft, keysRight, hashFunFactories,
+ comparatorFactories, recDescriptor, tableSize);
+ break;
+ }
+ case LEFT_OUTER: {
+ INullWriterFactory[] nullWriterFactories = new INullWriterFactory[inputSchemas[1].getSize()];
+ for (int j = 0; j < nullWriterFactories.length; j++) {
+ nullWriterFactories[j] = context.getNullWriterFactory();
+ }
+ opDesc = new InMemoryHashJoinOperatorDescriptor(spec, keysLeft, keysRight, hashFunFactories,
+ comparatorFactories, recDescriptor, true, nullWriterFactories, tableSize);
+ break;
+ }
+ default: {
+ throw new NotImplementedException();
+ }
+ }
+ contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+ ILogicalOperator src1 = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src1, 0, op, 0);
+ ILogicalOperator src2 = op.getInputs().get(1).getValue();
+ builder.contributeGraphEdge(src2, 0, op, 1);
+ }
+
+ @Override
+ protected List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties();
+ List<ILocalStructuralProperty> lp0 = pv0.getLocalProperties();
+ if (lp0 != null) {
+ // maintains the local properties on the probe side
+ return new LinkedList<ILocalStructuralProperty>(lp0);
+ }
+ return new LinkedList<ILocalStructuralProperty>();
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
new file mode 100644
index 0000000..8b8138f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InMemoryStableSortPOperator.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.sort.InMemorySortRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class InMemoryStableSortPOperator extends AbstractStableSortPOperator {
+
+ public InMemoryStableSortPOperator() {
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.IN_MEMORY_STABLE_SORT;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ int n = sortColumns.length;
+ int[] sortFields = new int[n];
+ IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
+ int i = 0;
+ INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+ INormalizedKeyComputerFactory nkcf = null;
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ for (OrderColumn oc : sortColumns) {
+ LogicalVariable var = oc.getColumn();
+ sortFields[i] = opSchema.findVariable(var);
+ Object type = env.getVarType(var);
+ OrderKind order = oc.getOrder();
+ if (i == 0 && nkcfProvider != null && type != null) {
+ nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order);
+ }
+
+ IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+ comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder());
+ i++;
+ }
+
+ IPushRuntimeFactory runtime = new InMemorySortRuntimeFactory(sortFields, nkcf, comps, null);
+ builder.contributeMicroOperator(op, runtime, recDescriptor);
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
new file mode 100644
index 0000000..276122f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/IndexInsertDeletePOperator.java
@@ -0,0 +1,108 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class IndexInsertDeletePOperator extends AbstractPhysicalOperator {
+
+ private List<LogicalVariable> primaryKeys;
+ private List<LogicalVariable> secondaryKeys;
+ private IDataSourceIndex<?, ?> dataSourceIndex;
+
+ public IndexInsertDeletePOperator(List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys,
+ IDataSourceIndex<?, ?> dataSourceIndex) {
+ this.primaryKeys = primaryKeys;
+ this.secondaryKeys = secondaryKeys;
+ this.dataSourceIndex = dataSourceIndex;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.INDEX_INSERT_DELETE;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ List<LogicalVariable> scanVariables = new ArrayList<LogicalVariable>();
+ scanVariables.addAll(primaryKeys);
+ scanVariables.add(new LogicalVariable(-1));
+ IPartitioningProperty pp = dataSourceIndex.getDataSource().getPropertiesProvider()
+ .computePropertiesVector(scanVariables).getPartitioningProperty();
+ List<ILocalStructuralProperty> orderProps = new LinkedList<ILocalStructuralProperty>();
+ for (LogicalVariable k : secondaryKeys) {
+ orderProps.add(new LocalOrderProperty(new OrderColumn(k, OrderKind.ASC)));
+ }
+ StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(pp,
+ orderProps) };
+ return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ IndexInsertDeleteOperator insertDeleteOp = (IndexInsertDeleteOperator) op;
+ IMetadataProvider mp = context.getMetadataProvider();
+
+ JobSpecification spec = builder.getJobSpec();
+ RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(), inputSchemas[0],
+ context);
+
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
+ if (insertDeleteOp.getOperation() == Kind.INSERT)
+ runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, primaryKeys,
+ secondaryKeys, inputDesc, context, spec);
+ else
+ runtimeAndConstraints = mp.getIndexDeleteRuntime(dataSourceIndex, propagatedSchema, primaryKeys,
+ secondaryKeys, inputDesc, context, spec);
+
+ builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
+ builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
+ ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
new file mode 100644
index 0000000..2863aa6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/InsertDeletePOperator.java
@@ -0,0 +1,96 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+@SuppressWarnings("rawtypes")
+public class InsertDeletePOperator extends AbstractPhysicalOperator {
+
+ private LogicalVariable payload;
+ private List<LogicalVariable> keys;
+ private IDataSource<?> dataSource;
+
+ public InsertDeletePOperator(LogicalVariable payload, List<LogicalVariable> keys, IDataSource dataSource) {
+ this.payload = payload;
+ this.keys = keys;
+ this.dataSource = dataSource;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.INSERT_DELETE;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ List<LogicalVariable> scanVariables = new ArrayList<LogicalVariable>();
+ scanVariables.addAll(keys);
+ scanVariables.add(new LogicalVariable(-1));
+ IPhysicalPropertiesVector r = dataSource.getPropertiesProvider().computePropertiesVector(scanVariables);
+ IPhysicalPropertiesVector[] requirements = new IPhysicalPropertiesVector[1];
+ requirements[0] = r;
+ return new PhysicalRequirements(requirements, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ InsertDeleteOperator insertDeleteOp = (InsertDeleteOperator) op;
+ IMetadataProvider mp = context.getMetadataProvider();
+ JobSpecification spec = builder.getJobSpec();
+ RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(), inputSchemas[0],
+ context);
+
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = null;
+ if (insertDeleteOp.getOperation() == Kind.INSERT)
+ runtimeAndConstraints = mp.getInsertRuntime(dataSource, propagatedSchema, keys, payload, inputDesc,
+ context, spec);
+ else
+ runtimeAndConstraints = mp.getDeleteRuntime(dataSource, propagatedSchema, keys, payload, inputDesc,
+ context, spec);
+
+ builder.contributeHyracksOperator(insertDeleteOp, runtimeAndConstraints.first);
+ builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
+ ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, insertDeleteOp, 0);
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
new file mode 100644
index 0000000..5b02d83
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/MicroPreclusteredGroupByPOperator.java
@@ -0,0 +1,80 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.group.MicroPreClusteredGroupRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+public class MicroPreclusteredGroupByPOperator extends AbstractPreclusteredGroupByPOperator {
+
+ public MicroPreclusteredGroupByPOperator(List<LogicalVariable> columnList) {
+ super(columnList);
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.MICRO_PRE_CLUSTERED_GROUP_BY;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
+ GroupByOperator gby = (GroupByOperator) op;
+ int numFds = gby.getDecorList().size();
+ int fdColumns[] = new int[numFds];
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ int j = 0;
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+ ILogicalExpression expr = p.second.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new AlgebricksException("pre-sorted group-by expects variable references.");
+ }
+ VariableReferenceExpression v = (VariableReferenceExpression) expr;
+ LogicalVariable decor = v.getVariableReference();
+ fdColumns[j++] = inputSchemas[0].findVariable(decor);
+ }
+ // compile subplans and set the gby op. schema accordingly
+ AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context);
+ IAggregatorDescriptorFactory aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys,
+ fdColumns);
+
+ IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
+ columnList, env, context);
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(),
+ inputSchemas[0], context);
+ MicroPreClusteredGroupRuntimeFactory runtime = new MicroPreClusteredGroupRuntimeFactory(keys,
+ comparatorFactories, aggregatorFactory, inputRecordDesc, recordDescriptor, null);
+ builder.contributeMicroOperator(gby, runtime, recordDescriptor);
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
new file mode 100644
index 0000000..5978b10
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NLJoinPOperator.java
@@ -0,0 +1,275 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator.JoinKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.BroadcastPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.std.join.NestedLoopJoinOperatorDescriptor;
+
+/**
+ *
+ * Left input is broadcast and preserves its local properties.
+ *
+ * Right input can be partitioned in any way.
+ *
+ */
+public class NLJoinPOperator extends AbstractJoinPOperator {
+
+ private final int memSize;
+
+ public NLJoinPOperator(JoinKind kind, JoinPartitioningType partitioningType, int memSize) {
+ super(kind, partitioningType);
+ this.memSize = memSize;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.NESTED_LOOP;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator iop, IOptimizationContext context) {
+ if (partitioningType != JoinPartitioningType.BROADCAST) {
+ throw new NotImplementedException(partitioningType + " nested loop joins are not implemented.");
+ }
+
+ IPartitioningProperty pp;
+
+ AbstractLogicalOperator op = (AbstractLogicalOperator) iop;
+
+ if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(1).getValue();
+ IPhysicalPropertiesVector pv1 = op2.getPhysicalOperator().getDeliveredProperties();
+ if (pv1 == null) {
+ pp = null;
+ } else {
+ pp = pv1.getPartitioningProperty();
+ }
+ } else {
+ pp = null;
+ }
+
+ List<ILocalStructuralProperty> localProps = new LinkedList<ILocalStructuralProperty>();
+ this.deliveredProperties = new StructuralPropertiesVector(pp, localProps);
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ if (partitioningType != JoinPartitioningType.BROADCAST) {
+ throw new NotImplementedException(partitioningType + " nested loop joins are not implemented.");
+ }
+
+ StructuralPropertiesVector[] pv = new StructuralPropertiesVector[2];
+ pv[0] = new StructuralPropertiesVector(new BroadcastPartitioningProperty(null), null);
+ pv[1] = new StructuralPropertiesVector(null, null);
+ return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op;
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ IOperatorSchema[] conditionInputSchemas = new IOperatorSchema[1];
+ conditionInputSchemas[0] = propagatedSchema;
+ ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IEvaluatorFactory cond = exprJobGen.createEvaluatorFactory(join.getCondition().getValue(),
+ context.getTypeEnvironment(op), conditionInputSchemas, context);
+ ITuplePairComparatorFactory comparatorFactory = new TuplePairEvaluatorFactory(cond,
+ context.getBinaryBooleanInspector());
+ JobSpecification spec = builder.getJobSpec();
+ IOperatorDescriptor opDesc = null;
+
+ switch (kind) {
+ case INNER: {
+ opDesc = new NestedLoopJoinOperatorDescriptor(spec, comparatorFactory, recDescriptor, memSize);
+ break;
+ }
+ case LEFT_OUTER:
+ default: {
+ throw new NotImplementedException();
+ }
+ }
+ contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+ ILogicalOperator src1 = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src1, 0, op, 0);
+ ILogicalOperator src2 = op.getInputs().get(1).getValue();
+ builder.contributeGraphEdge(src2, 0, op, 1);
+ }
+
+ public static class TuplePairEvaluatorFactory implements ITuplePairComparatorFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final IEvaluatorFactory cond;
+ private final IBinaryBooleanInspector binaryBooleanInspector;
+
+ public TuplePairEvaluatorFactory(IEvaluatorFactory cond, IBinaryBooleanInspector binaryBooleanInspector) {
+ this.cond = cond;
+ this.binaryBooleanInspector = binaryBooleanInspector;
+ }
+
+ @Override
+ public synchronized ITuplePairComparator createTuplePairComparator() {
+ return new TuplePairEvaluator(cond, binaryBooleanInspector);
+ }
+
+ }
+
+ public static class TuplePairEvaluator implements ITuplePairComparator {
+
+ private IEvaluator condEvaluator;
+ private final IEvaluatorFactory condFactory;
+ private final CompositeFrameTupleReference compositeTupleRef;
+ private final FrameTupleReference leftRef;
+ private final FrameTupleReference rightRef;
+ private final ArrayBackedValueStorage evalOutput;
+ private final IBinaryBooleanInspector binaryBooleanInspector;
+
+ public TuplePairEvaluator(IEvaluatorFactory condFactory, IBinaryBooleanInspector binaryBooleanInspector) {
+ this.condFactory = condFactory;
+ this.binaryBooleanInspector = binaryBooleanInspector;
+ this.evalOutput = new ArrayBackedValueStorage();
+ this.leftRef = new FrameTupleReference();
+ this.rightRef = new FrameTupleReference();
+ this.compositeTupleRef = new CompositeFrameTupleReference(leftRef, rightRef);
+ }
+
+ @Override
+ public int compare(IFrameTupleAccessor outerAccessor, int outerIndex, IFrameTupleAccessor innerAccessor,
+ int innerIndex) throws HyracksDataException {
+ if (condEvaluator == null) {
+ try {
+ this.condEvaluator = condFactory.createEvaluator(evalOutput);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ compositeTupleRef.reset(outerAccessor, outerIndex, innerAccessor, innerIndex);
+ try {
+ condEvaluator.evaluate(compositeTupleRef);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ boolean result = binaryBooleanInspector.getBooleanValue(evalOutput.getBytes(), 0, evalOutput.getLength());
+ evalOutput.reset();
+ if (result)
+ return 0;
+ else
+ return 1;
+ }
+ }
+
+ public static class CompositeFrameTupleReference implements IFrameTupleReference {
+
+ private final FrameTupleReference refLeft;
+ private final FrameTupleReference refRight;
+
+ public CompositeFrameTupleReference(FrameTupleReference refLeft, FrameTupleReference refRight) {
+ this.refLeft = refLeft;
+ this.refRight = refRight;
+ }
+
+ public void reset(IFrameTupleAccessor outerAccessor, int outerIndex, IFrameTupleAccessor innerAccessor,
+ int innerIndex) {
+ refLeft.reset(outerAccessor, outerIndex);
+ refRight.reset(innerAccessor, innerIndex);
+ }
+
+ @Override
+ public int getFieldCount() {
+ return refLeft.getFieldCount() + refRight.getFieldCount();
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ int leftFieldCount = refLeft.getFieldCount();
+ if (fIdx < leftFieldCount)
+ return refLeft.getFieldData(fIdx);
+ else
+ return refRight.getFieldData(fIdx - leftFieldCount);
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ int leftFieldCount = refLeft.getFieldCount();
+ if (fIdx < leftFieldCount)
+ return refLeft.getFieldStart(fIdx);
+ else
+ return refRight.getFieldStart(fIdx - leftFieldCount);
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ int leftFieldCount = refLeft.getFieldCount();
+ if (fIdx < leftFieldCount)
+ return refLeft.getFieldLength(fIdx);
+ else
+ return refRight.getFieldLength(fIdx - leftFieldCount);
+ }
+
+ @Override
+ public IFrameTupleAccessor getFrameTupleAccessor() {
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public int getTupleIndex() {
+ throw new NotImplementedException();
+ }
+
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
new file mode 100644
index 0000000..5a4a204
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/NestedTupleSourcePOperator.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.NestedTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class NestedTupleSourcePOperator extends AbstractPhysicalOperator {
+
+ public NestedTupleSourcePOperator() {
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.NESTED_TUPLE_SOURCE;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ Mutable<ILogicalOperator> dataSource = ((NestedTupleSourceOperator) op).getDataSourceReference();
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) dataSource.getValue().getInputs().get(0).getValue();
+ IPhysicalPropertiesVector inheritedProps = op2.getDeliveredPhysicalProperties();
+ deliveredProperties = (StructuralPropertiesVector) inheritedProps.clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return null;
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ propagatedSchema.addAllVariables(outerPlanSchema);
+ NestedTupleSourceRuntimeFactory runtime = new NestedTupleSourceRuntimeFactory();
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ builder.contributeMicroOperator(op, runtime, recDesc);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
new file mode 100644
index 0000000..0ed17d8
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/OneToOneExchangePOperator.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+
+public class OneToOneExchangePOperator extends AbstractExchangePOperator {
+
+ public OneToOneExchangePOperator() {
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.ONE_TO_ONE_EXCHANGE;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) {
+ IConnectorDescriptor conn = new OneToOneConnectorDescriptor(spec);
+ return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.SAME_COUNT);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
new file mode 100644
index 0000000..d74fb62
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreSortedDistinctByPOperator.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
+
+public class PreSortedDistinctByPOperator extends AbstractPhysicalOperator {
+
+ private List<LogicalVariable> columnList;
+
+ public PreSortedDistinctByPOperator(List<LogicalVariable> columnList) {
+ this.columnList = columnList;
+ }
+
+ public void setDistinctByColumns(List<LogicalVariable> distinctByColumns) {
+ this.columnList = distinctByColumns;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
+ List<ILocalStructuralProperty> propsLocal = op2.getDeliveredPhysicalProperties().getLocalProperties();
+ deliveredProperties = new StructuralPropertiesVector(pp, propsLocal);
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+ List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>(columnList.size());
+ for (LogicalVariable column : columnList) {
+ localProps.add(new LocalOrderProperty(new OrderColumn(column, OrderKind.ASC)));
+ }
+ IPartitioningProperty pp = null;
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) op;
+ if (aop.getExecutionMode() == ExecutionMode.PARTITIONED) {
+ pp = new UnorderedPartitionedProperty(new HashSet<LogicalVariable>(columnList), null);
+ }
+ pv[0] = new StructuralPropertiesVector(pp, localProps);
+ return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+
+ JobSpecification spec = builder.getJobSpec();
+ int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
+ int sz = inputSchemas[0].getSize();
+ int fdSz = sz - columnList.size();
+ int[] fdColumns = new int[fdSz];
+ int j = 0;
+ for (LogicalVariable v : inputSchemas[0]) {
+ if (!columnList.contains(v)) {
+ fdColumns[j++] = inputSchemas[0].findVariable(v);
+ }
+ }
+ IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
+ columnList, context.getTypeEnvironment(op), context);
+ IAggregateFunctionFactory[] aggFactories = new IAggregateFunctionFactory[] {};
+ IAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(
+ aggFactories, keys, fdColumns);
+
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys,
+ comparatorFactories, aggregatorFactory, recordDescriptor);
+
+ contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.PRE_SORTED_DISTINCT_BY;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
new file mode 100644
index 0000000..bb19bb9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/PreclusteredGroupByPOperator.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
+
+public class PreclusteredGroupByPOperator extends AbstractPreclusteredGroupByPOperator {
+
+ public PreclusteredGroupByPOperator(List<LogicalVariable> columnList) {
+ super(columnList);
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]);
+ GroupByOperator gby = (GroupByOperator) op;
+ int numFds = gby.getDecorList().size();
+ int fdColumns[] = new int[numFds];
+ int j = 0;
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+ ILogicalExpression expr = p.second.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new AlgebricksException("pre-sorted group-by expects variable references.");
+ }
+ VariableReferenceExpression v = (VariableReferenceExpression) expr;
+ LogicalVariable decor = v.getVariableReference();
+ fdColumns[j++] = inputSchemas[0].findVariable(decor);
+ }
+ // compile subplans and set the gby op. schema accordingly
+ AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context);
+ IAggregatorDescriptorFactory aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys,
+ fdColumns);
+
+ JobSpecification spec = builder.getJobSpec();
+ IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(
+ columnList, context.getTypeEnvironment(op), context);
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+
+ PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys,
+ comparatorFactories, aggregatorFactory, recordDescriptor);
+
+ contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
new file mode 100644
index 0000000..abfa31f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RandomMergeExchangePOperator.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNReplicatingConnectorDescriptor;
+
+public class RandomMergeExchangePOperator extends AbstractExchangePOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.RANDOM_MERGE_EXCHANGE;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ this.deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED,
+ new ArrayList<ILocalStructuralProperty>(0));
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) {
+ IConnectorDescriptor conn = new MToNReplicatingConnectorDescriptor(spec);
+ return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
new file mode 100644
index 0000000..ff558ec
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RangePartitionPOperator.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class RangePartitionPOperator extends AbstractExchangePOperator {
+
+ private ArrayList<OrderColumn> partitioningFields;
+ private INodeDomain domain;
+
+ public RangePartitionPOperator(ArrayList<OrderColumn> partitioningFields, INodeDomain domain) {
+ this.partitioningFields = partitioningFields;
+ this.domain = domain;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.RANGE_PARTITION_EXCHANGE;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ IPartitioningProperty p = new OrderedPartitionedProperty(partitioningFields, domain);
+ this.deliveredProperties = new StructuralPropertiesVector(p, new LinkedList<ILocalStructuralProperty>());
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+ throw new NotImplementedException();
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
new file mode 100644
index 0000000..b42dd5b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/ReplicatePOperator.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.misc.SplitOperatorDescriptor;
+
+public class ReplicatePOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.SPLIT;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ JobSpecification spec = builder.getJobSpec();
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+
+ ReplicateOperator rop = (ReplicateOperator) op;
+ int outputArity = rop.getOutputArity();
+
+ SplitOperatorDescriptor splitOpDesc = new SplitOperatorDescriptor(spec, recDescriptor, outputArity);
+ contributeOpDesc(builder, (AbstractLogicalOperator) op, splitOpDesc);
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
new file mode 100644
index 0000000..3647deb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/RunningAggregatePOperator.java
@@ -0,0 +1,117 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.RunningAggregateRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class RunningAggregatePOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.RUNNING_AGGREGATE;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ IPartitioningProperty pp = null;
+ RunningAggregateOperator ragg = (RunningAggregateOperator) op;
+ for (Mutable<ILogicalExpression> exprRef : ragg.getExpressions()) {
+ StatefulFunctionCallExpression f = (StatefulFunctionCallExpression) exprRef.getValue();
+ IPartitioningProperty p = f.getRequiredPartitioningProperty();
+ if (p != null) {
+ if (pp == null) {
+ pp = p;
+ } else {
+ throw new IllegalStateException("Two stateful functions want to set partitioning requirements: "
+ + pp + " and " + p);
+ }
+ }
+ }
+ StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(pp, null) };
+ return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ RunningAggregateOperator ragg = (RunningAggregateOperator) op;
+ List<LogicalVariable> variables = ragg.getVariables();
+ List<Mutable<ILogicalExpression>> expressions = ragg.getExpressions();
+ int[] outColumns = new int[variables.size()];
+ for (int i = 0; i < outColumns.length; i++) {
+ outColumns[i] = opSchema.findVariable(variables.get(i));
+ }
+ IRunningAggregateFunctionFactory[] runningAggFuns = new IRunningAggregateFunctionFactory[expressions.size()];
+ ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ for (int i = 0; i < runningAggFuns.length; i++) {
+ StatefulFunctionCallExpression expr = (StatefulFunctionCallExpression) expressions.get(i).getValue();
+ runningAggFuns[i] = exprJobGen.createRunningAggregateFunctionFactory(expr,
+ context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
+ }
+
+ // TODO push projections into the operator
+ int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
+
+ RunningAggregateRuntimeFactory runtime = new RunningAggregateRuntimeFactory(outColumns, runningAggFuns,
+ projectionList);
+
+ // contribute one Asterix framewriter
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ builder.contributeMicroOperator(ragg, runtime, recDesc);
+ // and contribute one edge from its child
+ ILogicalOperator src = ragg.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, ragg, 0);
+
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
new file mode 100644
index 0000000..0375246
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkPOperator.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.SinkRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class SinkPOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.SINK;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ SinkRuntimeFactory runtime = new SinkRuntimeFactory();
+ builder.contributeMicroOperator(op, runtime, recDesc);
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
new file mode 100644
index 0000000..123b0e1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SinkWritePOperator.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class SinkWritePOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.SINK_WRITE;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ ILogicalOperator op2 = op.getInputs().get(0).getValue();
+ deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ WriteOperator write = (WriteOperator) op;
+ IDataSink sink = write.getDataSink();
+ IPartitioningProperty pp = sink.getPartitioningProperty();
+ StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(pp, null) };
+ return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ WriteOperator write = (WriteOperator) op;
+ int[] columns = new int[write.getExpressions().size()];
+ int i = 0;
+ for (Mutable<ILogicalExpression> exprRef : write.getExpressions()) {
+ ILogicalExpression expr = exprRef.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new NotImplementedException("Only writing variable expressions is supported.");
+ }
+ VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+ LogicalVariable v = varRef.getVariableReference();
+ columns[i++] = inputSchemas[0].findVariable(v);
+ }
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(), inputSchemas[0],
+ context);
+
+ IPrinterFactory[] pf = JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op),
+ context, columns);
+
+ IMetadataProvider<?, ?> mp = context.getMetadataProvider();
+
+ Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> runtime = mp.getWriteFileRuntime(write.getDataSink(),
+ columns, pf, inputDesc);
+
+ builder.contributeMicroOperator(write, runtime.first, recDesc, runtime.second);
+ ILogicalOperator src = write.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, write, 0);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
new file mode 100644
index 0000000..98c9be3
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SortMergeExchangePOperator.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder.TargetConstraint;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningMergingConnectorDescriptor;
+
+public class SortMergeExchangePOperator extends AbstractExchangePOperator {
+
+ private OrderColumn[] sortColumns;
+
+ public SortMergeExchangePOperator(OrderColumn[] sortColumns) {
+ this.sortColumns = sortColumns;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.SORT_MERGE_EXCHANGE;
+ }
+
+ public OrderColumn[] getSortColumns() {
+ return sortColumns;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getOperatorTag());
+ sb.append(" [");
+ sb.append(sortColumns[0]);
+ for (int i = 1; i < sortColumns.length; i++) {
+ sb.append(", " + sortColumns[i]);
+ }
+ sb.append(" ]");
+ return sb.toString();
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator inp1 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ IPhysicalPropertiesVector pv1 = inp1.getDeliveredPhysicalProperties();
+ if (pv1 == null) {
+ inp1.computeDeliveredPhysicalProperties(context);
+ pv1 = inp1.getDeliveredPhysicalProperties();
+ }
+ int sortCol = 0;
+ List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>(sortColumns.length);
+ for (ILocalStructuralProperty prop : pv1.getLocalProperties()) {
+ if (prop.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
+ LocalOrderProperty lop = (LocalOrderProperty) prop;
+ if (lop.getOrderColumn().equals(sortColumns[sortCol])) {
+ localProps.add(lop);
+ sortCol++;
+ if (sortCol == sortColumns.length) {
+ break;
+ }
+ }
+ } else {
+ break;
+ }
+ }
+ if (sortCol < sortColumns.length) {
+ localProps = null;
+ }
+ this.deliveredProperties = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, localProps);
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ List<ILocalStructuralProperty> localProps = new ArrayList<ILocalStructuralProperty>(sortColumns.length);
+ for (OrderColumn oc : sortColumns) {
+ localProps.add(new LocalOrderProperty(oc));
+ }
+ StructuralPropertiesVector[] r = new StructuralPropertiesVector[] { new StructuralPropertiesVector(null,
+ localProps) };
+ return new PhysicalRequirements(r, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @Override
+ public Pair<IConnectorDescriptor, TargetConstraint> createConnectorDescriptor(JobSpecification spec,
+ ILogicalOperator op, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException {
+ int n = sortColumns.length;
+ int[] sortFields = new int[n];
+ IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
+ IBinaryHashFunctionFactory[] hashFuns = new IBinaryHashFunctionFactory[n];
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ for (int i = 0; i < n; i++) {
+ sortFields[i] = opSchema.findVariable(sortColumns[i].getColumn());
+ Object type = env.getVarType(sortColumns[i].getColumn());
+ IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+ comps[i] = bcfp.getBinaryComparatorFactory(type, sortColumns[i].getOrder());
+ IBinaryHashFunctionFactoryProvider bhffp = context.getBinaryHashFunctionFactoryProvider();
+ hashFuns[i] = bhffp.getBinaryHashFunctionFactory(type);
+ }
+ ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(sortFields, hashFuns);
+ IConnectorDescriptor conn = new MToNPartitioningMergingConnectorDescriptor(spec, tpcf, sortFields, comps);
+ return new Pair<IConnectorDescriptor, TargetConstraint>(conn, TargetConstraint.ONE);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
new file mode 100644
index 0000000..73c7dd6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StableSortPOperator.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+
+/**
+ *
+ * This will always be attached to an {@link OrderOperator} logical operator.
+ *
+ *
+ */
+
+public class StableSortPOperator extends AbstractStableSortPOperator {
+
+ private int maxNumberOfFrames;
+
+ public StableSortPOperator(int maxNumberOfFrames) {
+ super();
+ this.maxNumberOfFrames = maxNumberOfFrames;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.STABLE_SORT;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ JobSpecification spec = builder.getJobSpec();
+ RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ int n = sortColumns.length;
+ int[] sortFields = new int[n];
+ IBinaryComparatorFactory[] comps = new IBinaryComparatorFactory[n];
+
+ INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+ INormalizedKeyComputerFactory nkcf = null;
+
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ int i = 0;
+ for (OrderColumn oc : sortColumns) {
+ LogicalVariable var = oc.getColumn();
+ sortFields[i] = opSchema.findVariable(var);
+ Object type = env.getVarType(var);
+ OrderKind order = oc.getOrder();
+ if (i == 0 && nkcfProvider != null && type != null) {
+ nkcf = nkcfProvider.getNormalizedKeyComputerFactory(type, order);
+ }
+ IBinaryComparatorFactoryProvider bcfp = context.getBinaryComparatorFactoryProvider();
+ comps[i] = bcfp.getBinaryComparatorFactory(type, oc.getOrder());
+ i++;
+ }
+
+ ExternalSortOperatorDescriptor sortOpDesc = new ExternalSortOperatorDescriptor(spec, maxNumberOfFrames,
+ sortFields, nkcf, comps, recDescriptor);
+ contributeOpDesc(builder, (AbstractLogicalOperator) op, sortOpDesc);
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java
new file mode 100644
index 0000000..82ea31c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamDiePOperator.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamDieRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class StreamDiePOperator extends AbstractPhysicalOperator {
+
+ public StreamDiePOperator() {
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.STREAM_DIE;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ ILogicalOperator op2 = op.getInputs().get(0).getValue();
+ deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+ pv[0] = (StructuralPropertiesVector) reqdByParent;
+ return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ DieOperator die = (DieOperator) op;
+ ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ IEvaluatorFactory afterObjectsFact = exprJobGen.createEvaluatorFactory(die.getAfterObjects().getValue(),
+ env, inputSchemas, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ StreamDieRuntimeFactory runtime = new StreamDieRuntimeFactory(afterObjectsFact, null,
+ context.getBinaryIntegerInspector());
+ builder.contributeMicroOperator(die, runtime, recDesc);
+ ILogicalOperator src = die.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, die, 0);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
new file mode 100644
index 0000000..a31b2da
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamLimitPOperator.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamLimitRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class StreamLimitPOperator extends AbstractPhysicalOperator {
+
+ private boolean global;
+
+ public StreamLimitPOperator(boolean global) {
+ this.global = global;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.STREAM_LIMIT;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ ILogicalOperator op2 = op.getInputs().get(0).getValue();
+ deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ if (global) {
+ StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1];
+ pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null);
+ return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ } else {
+ return emptyUnaryRequirements();
+ }
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ LimitOperator limit = (LimitOperator) op;
+ ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ IEvaluatorFactory maxObjectsFact = exprJobGen.createEvaluatorFactory(limit.getMaxObjects().getValue(),
+ env, inputSchemas, context);
+ ILogicalExpression offsetExpr = limit.getOffset().getValue();
+ IEvaluatorFactory offsetFact = (offsetExpr == null) ? null : exprJobGen.createEvaluatorFactory(offsetExpr, env,
+ inputSchemas, context);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ StreamLimitRuntimeFactory runtime = new StreamLimitRuntimeFactory(maxObjectsFact, offsetFact, null,
+ context.getBinaryIntegerInspector());
+ builder.contributeMicroOperator(limit, runtime, recDesc);
+ // and contribute one edge from its child
+ ILogicalOperator src = limit.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, limit, 0);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
new file mode 100644
index 0000000..94e2135
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamProjectPOperator.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamProjectRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class StreamProjectPOperator extends AbstractPropagatePropertiesForUsedVariablesPOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.STREAM_PROJECT;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ ProjectOperator project = (ProjectOperator) op;
+ int[] projectionList = new int[project.getVariables().size()];
+ int i = 0;
+ for (LogicalVariable v : project.getVariables()) {
+ int pos = inputSchemas[0].findVariable(v);
+ if (pos < 0) {
+ throw new AlgebricksException("Could not find variable " + v + ".");
+ }
+ projectionList[i++] = pos;
+ }
+ StreamProjectRuntimeFactory runtime = new StreamProjectRuntimeFactory(projectionList);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ builder.contributeMicroOperator(project, runtime, recDesc);
+ ILogicalOperator src = project.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, project, 0);
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ ProjectOperator p = (ProjectOperator) op;
+ computeDeliveredPropertiesForUsedVariables(p, p.getVariables());
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
new file mode 100644
index 0000000..b4112c8
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StreamSelectPOperator.java
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamSelectRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class StreamSelectPOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.STREAM_SELECT;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ ILogicalOperator op2 = op.getInputs().get(0).getValue();
+ deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ SelectOperator select = (SelectOperator) op;
+ ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ IEvaluatorFactory cond = exprJobGen.createEvaluatorFactory(select.getCondition().getValue(),
+ context.getTypeEnvironment(op), inputSchemas, context);
+ StreamSelectRuntimeFactory runtime = new StreamSelectRuntimeFactory(cond, null,
+ context.getBinaryBooleanInspector());
+ // contribute one Asterix framewriter
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ builder.contributeMicroOperator(select, runtime, recDesc);
+ // and contribute one edge from its child
+ ILogicalOperator src = select.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, select, 0);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
new file mode 100644
index 0000000..dfab5ed
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/StringStreamingScriptPOperator.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StringStreamingRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription;
+import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.IScriptDescription.ScriptKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.scripting.StringStreamingScriptDescription;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class StringStreamingScriptPOperator extends AbstractPropagatePropertiesForUsedVariablesPOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.STRING_STREAM_SCRIPT;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ ScriptOperator scriptOp = (ScriptOperator) op;
+ IScriptDescription scriptDesc = scriptOp.getScriptDescription();
+ if (scriptDesc.getKind() != ScriptKind.STRING_STREAMING) {
+ throw new IllegalStateException();
+ }
+ StringStreamingScriptDescription sssd = (StringStreamingScriptDescription) scriptDesc;
+ StringStreamingRuntimeFactory runtime = new StringStreamingRuntimeFactory(sssd.getCommand(),
+ sssd.getPrinterFactories(), sssd.getFieldDelimiter(), sssd.getParserFactory());
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, propagatedSchema, context);
+ builder.contributeMicroOperator(scriptOp, runtime, recDesc);
+ // and contribute one edge from its child
+ ILogicalOperator src = scriptOp.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, scriptOp, 0);
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ ScriptOperator s = (ScriptOperator) op;
+ computeDeliveredPropertiesForUsedVariables(s, s.getInputVariables());
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
new file mode 100644
index 0000000..4e6ef69
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/SubplanPOperator.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.SubplanRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class SubplanPOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.SUBPLAN;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ IPhysicalPropertiesVector childsProperties = op2.getPhysicalOperator().getDeliveredProperties();
+ List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>();
+ if (childsProperties.getLocalProperties() != null) {
+ propsLocal.addAll(childsProperties.getLocalProperties());
+ }
+ // ... get local properties for newly created variables...
+ SubplanOperator subplan = (SubplanOperator) op;
+ for (ILogicalPlan plan : subplan.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : plan.getRoots()) {
+ AbstractLogicalOperator rOp = (AbstractLogicalOperator) r.getValue();
+ propsLocal.addAll(rOp.getPhysicalOperator().getDeliveredProperties().getLocalProperties());
+ }
+ }
+
+ deliveredProperties = new StructuralPropertiesVector(childsProperties.getPartitioningProperty(), propsLocal);
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ return emptyUnaryRequirements();
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ SubplanOperator subplan = (SubplanOperator) op;
+ if (subplan.getNestedPlans().size() != 1) {
+ throw new NotImplementedException("Subplan currently works only for one nested plan with one root.");
+ }
+ AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], subplan, opSchema, context);
+ assert (subplans.length == 1);
+ AlgebricksPipeline np = subplans[0];
+ RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor(op.getInputs().get(0).getValue(),
+ inputSchemas[0], context);
+ INullWriterFactory[] nullWriterFactories = new INullWriterFactory[np.getOutputWidth()];
+ for (int i = 0; i < nullWriterFactories.length; i++) {
+ nullWriterFactories[i] = context.getNullWriterFactory();
+ }
+ SubplanRuntimeFactory runtime = new SubplanRuntimeFactory(np, nullWriterFactories, inputRecordDesc, null);
+
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ builder.contributeMicroOperator(subplan, runtime, recDesc);
+
+ ILogicalOperator src = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, op, 0);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
new file mode 100644
index 0000000..da88dd9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnionAllPOperator.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.union.UnionAllOperatorDescriptor;
+
+public class UnionAllPOperator extends AbstractPhysicalOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.UNION_ALL;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return false;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ IPartitioningProperty pp = op2.getDeliveredPhysicalProperties().getPartitioningProperty();
+ this.deliveredProperties = new StructuralPropertiesVector(pp, new ArrayList<ILocalStructuralProperty>(0));
+
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ StructuralPropertiesVector pv0 = StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR;
+ StructuralPropertiesVector pv1 = StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR;
+ return new PhysicalRequirements(new StructuralPropertiesVector[] { pv0, pv1 },
+ IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ UnionAllOperator unionOp = (UnionAllOperator) op;
+ int n = unionOp.getVariableMappings().size();
+ int[] leftColumns = new int[n];
+ int[] rightColumns = new int[n];
+ int i = 0;
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> t : unionOp.getVariableMappings()) {
+ int posLeft = inputSchemas[0].findVariable(t.first);
+ leftColumns[i] = posLeft;
+ int posRight = inputSchemas[1].findVariable(t.second);
+ rightColumns[i] = posRight;
+ ++i;
+ }
+
+ JobSpecification spec = builder.getJobSpec();
+ RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+
+ // at algebricks level, union all only accepts two inputs, although at
+ // hyracks
+ // level, there is no restrictions
+ UnionAllOperatorDescriptor opDesc = new UnionAllOperatorDescriptor(spec, 2, recordDescriptor);
+ contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc);
+ ILogicalOperator src1 = op.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src1, 0, op, 0);
+ ILogicalOperator src2 = op.getInputs().get(1).getValue();
+ builder.contributeGraphEdge(src2, 0, op, 1);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
new file mode 100644
index 0000000..8f838ba
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/UnnestPOperator.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenHelper;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.UnnestRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class UnnestPOperator extends AbstractScanPOperator {
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.UNNEST;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ deliveredProperties = op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ UnnestOperator unnest = (UnnestOperator) op;
+ if (unnest.getPositionalVariable() != null) {
+ throw new AlgebricksException("Cannot generate runtime for unnest with positional variable "
+ + unnest.getPositionalVariable());
+ }
+ int outCol = opSchema.findVariable(unnest.getVariable());
+ ILogicalExpression unnestExpr = unnest.getExpressionRef().getValue();
+ ILogicalExpressionJobGen exprJobGen = context.getExpressionJobGen();
+ boolean exit = false;
+ if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ exit = true;
+ } else {
+ AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) unnestExpr;
+ if (fce.getKind() != FunctionKind.UNNEST) {
+ exit = true;
+ }
+ }
+ if (exit) {
+ throw new AlgebricksException("Unnest expression " + unnestExpr + " is not an unnesting function call.");
+ }
+ UnnestingFunctionCallExpression agg = (UnnestingFunctionCallExpression) unnestExpr;
+ IUnnestingFunctionFactory unnestingFactory = exprJobGen.createUnnestingFunctionFactory(agg,
+ context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context);
+ int[] projectionList = JobGenHelper.projectAllVariables(opSchema);
+ UnnestRuntimeFactory unnestRuntime = new UnnestRuntimeFactory(outCol, unnestingFactory, projectionList);
+ RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(op, opSchema, context);
+ builder.contributeMicroOperator(unnest, unnestRuntime, recDesc);
+ ILogicalOperator src = unnest.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, unnest, 0);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
new file mode 100644
index 0000000..d1f0340
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/operators/physical/WriteResultPOperator.java
@@ -0,0 +1,89 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class WriteResultPOperator extends AbstractPhysicalOperator {
+
+ private LogicalVariable payload;
+ private List<LogicalVariable> keys;
+ private IDataSource<?> dataSource;
+
+ public WriteResultPOperator(IDataSource<?> dataSource, LogicalVariable payload, List<LogicalVariable> keys) {
+ this.dataSource = dataSource;
+ this.payload = payload;
+ this.keys = keys;
+ }
+
+ @Override
+ public String toString() {
+ return getOperatorTag().toString() + " " + payload + " " + keys;
+ }
+
+ @Override
+ public PhysicalOperatorTag getOperatorTag() {
+ return PhysicalOperatorTag.WRITE_RESULT;
+ }
+
+ @Override
+ public boolean isMicroOperator() {
+ return true;
+ }
+
+ @Override
+ public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ deliveredProperties = (StructuralPropertiesVector) op2.getDeliveredPhysicalProperties().clone();
+ }
+
+ @Override
+ public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator op,
+ IPhysicalPropertiesVector reqdByParent) {
+ List<LogicalVariable> scanVariables = new ArrayList<LogicalVariable>();
+ scanVariables.addAll(keys);
+ scanVariables.add(new LogicalVariable(-1));
+ IPhysicalPropertiesVector r = dataSource.getPropertiesProvider().computePropertiesVector(scanVariables);
+ IPhysicalPropertiesVector[] requirements = new IPhysicalPropertiesVector[1];
+ requirements[0] = r;
+ return new PhysicalRequirements(requirements, IPartitioningRequirementsCoordinator.NO_COORDINATION);
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ @Override
+ public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op,
+ IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema)
+ throws AlgebricksException {
+ WriteResultOperator writeResultOp = (WriteResultOperator) op;
+ IMetadataProvider mp = context.getMetadataProvider();
+
+ JobSpecification spec = builder.getJobSpec();
+ Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getWriteResultRuntime(
+ dataSource, propagatedSchema, keys, payload, context, spec);
+
+ builder.contributeHyracksOperator(writeResultOp, runtimeAndConstraints.first);
+ builder.contributeAlgebricksPartitionConstraint(runtimeAndConstraints.first, runtimeAndConstraints.second);
+ ILogicalOperator src = writeResultOp.getInputs().get(0).getValue();
+ builder.contributeGraphEdge(src, 0, writeResultOp, 0);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
new file mode 100644
index 0000000..103f534
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/plan/ALogicalPlanImpl.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.plan;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+
+/*
+ * Author: Guangqiang Li
+ * Created on Jul 9, 2009
+ */
+public class ALogicalPlanImpl implements ILogicalPlan {
+ private List<Mutable<ILogicalOperator>> roots;
+
+ public ALogicalPlanImpl() {
+ this.roots = new ArrayList<Mutable<ILogicalOperator>>();
+ }
+
+ public ALogicalPlanImpl(List<Mutable<ILogicalOperator>> roots) {
+ this.roots = roots;
+ }
+
+ public ALogicalPlanImpl(Mutable<ILogicalOperator> root) {
+ roots = new ArrayList<Mutable<ILogicalOperator>>(1);
+ roots.add(root);
+ }
+
+ public List<Mutable<ILogicalOperator>> getRoots() {
+ return roots;
+ }
+
+ public void setRoots(List<Mutable<ILogicalOperator>> roots) {
+ this.roots = roots;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
new file mode 100644
index 0000000..05ae1ec
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/LogicalOperatorPrettyPrintVisitor.java
@@ -0,0 +1,349 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator.Kind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+
+public class LogicalOperatorPrettyPrintVisitor implements ILogicalOperatorVisitor<String, Integer> {
+
+ public LogicalOperatorPrettyPrintVisitor() {
+ }
+
+ @Override
+ public String visitAggregateOperator(AggregateOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("aggregate ").append(op.getVariables()).append(" <- ");
+ pprintExprList(op.getExpressions(), buffer);
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitRunningAggregateOperator(RunningAggregateOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("running-aggregate ").append(op.getVariables()).append(" <- ");
+ pprintExprList(op.getExpressions(), buffer);
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("empty-tuple-source");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitGroupByOperator(GroupByOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("group by (").append(op.gByListToString()).append(") decor (")
+ .append(op.decorListToString()).append(") {");
+ printNestedPlans(op, indent, buffer);
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitDistinctOperator(DistinctOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("distinct " + "(");
+ pprintExprList(op.getExpressions(), buffer);
+ buffer.append(")");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitInnerJoinOperator(InnerJoinOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("join (").append(op.getCondition().getValue()).append(")");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("left outer join (").append(op.getCondition().getValue()).append(")");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitNestedTupleSourceOperator(NestedTupleSourceOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("nested tuple source");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitOrderOperator(OrderOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("order ");
+ for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) {
+ String fst;
+ switch (p.first.getKind()) {
+ case ASC: {
+ fst = "ASC";
+ break;
+ }
+ case DESC: {
+ fst = "DESC";
+ break;
+ }
+ default: {
+ fst = p.first.getExpressionRef().toString();
+ }
+ }
+ buffer.append("(" + fst + ", " + p.second.getValue() + ") ");
+ }
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitAssignOperator(AssignOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("assign ").append(op.getVariables()).append(" <- ");
+ pprintExprList(op.getExpressions(), buffer);
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitWriteOperator(WriteOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("write ").append(op.getExpressions());
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitWriteResultOperator(WriteResultOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("load ").append(op.getDataSource()).append(" from ")
+ .append(op.getPayloadExpression()).append(" partitioned by ").append(op.getKeyExpressions().toString());
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitSelectOperator(SelectOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("select " + "(" + op.getCondition().getValue() + ")");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitProjectOperator(ProjectOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("project " + "(" + op.getVariables() + ")");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitPartitioningSplitOperator(PartitioningSplitOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("partitioning-split (" + op.getExpressions() + ")");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitSubplanOperator(SubplanOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("subplan {");
+ printNestedPlans(op, indent, buffer);
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitUnionOperator(UnionAllOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("union");
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> v : op.getVariableMappings()) {
+ buffer.append(" (" + v.first + ", " + v.second + ", " + v.third + ")");
+ }
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitUnnestOperator(UnnestOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("unnest " + op.getVariable());
+ if (op.getPositionalVariable() != null) {
+ buffer.append(" at " + op.getPositionalVariable());
+ }
+ buffer.append(" <- " + op.getExpressionRef().getValue());
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitUnnestMapOperator(UnnestMapOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append(
+ "unnest-map " + op.getVariables() + " <- " + op.getExpressionRef().getValue());
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitDataScanOperator(DataSourceScanOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append(
+ "data-scan " + op.getProjectVariables() + "<-" + op.getVariables() + " <- " + op.getDataSource());
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitLimitOperator(LimitOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("limit " + op.getMaxObjects().getValue());
+ ILogicalExpression offset = op.getOffset().getValue();
+ if (offset != null) {
+ buffer.append(", " + offset);
+ }
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitDieOperator(DieOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("die after " + op.getAfterObjects().getValue());
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitExchangeOperator(ExchangeOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("exchange ");
+ return buffer.toString();
+ }
+
+ protected static final StringBuilder addIndent(StringBuilder buffer, int level) {
+ for (int i = 0; i < level; ++i) {
+ buffer.append(' ');
+ }
+ return buffer;
+ }
+
+ private void printNestedPlans(AbstractOperatorWithNestedPlans op, Integer indent, StringBuilder buffer)
+ throws AlgebricksException {
+ boolean first = true;
+ if (op.getNestedPlans().isEmpty()) {
+ buffer.append("}");
+ } else {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ // PrettyPrintUtil.indent(buffer, level + 10).append("var " +
+ // p.first + ":\n");
+ buffer.append("\n");
+ if (first) {
+ first = false;
+ } else {
+ addIndent(buffer, indent).append(" {\n");
+ }
+ PlanPrettyPrinter.printPlan(p, buffer, this, indent + 10);
+ addIndent(buffer, indent).append(" }");
+ }
+ }
+ }
+
+ @Override
+ public String visitScriptOperator(ScriptOperator op, Integer indent) {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append(
+ "script (in: " + op.getInputVariables() + ") (out: " + op.getOutputVariables() + ")");
+ return buffer.toString();
+ }
+
+ private void pprintExprList(List<Mutable<ILogicalExpression>> expressions, StringBuilder buffer) {
+ buffer.append("[");
+ boolean first = true;
+ for (Mutable<ILogicalExpression> exprRef : expressions) {
+ if (first) {
+ first = false;
+ } else {
+ buffer.append(", ");
+ }
+ buffer.append(exprRef.getValue());
+ }
+ buffer.append("]");
+ }
+
+ @Override
+ public String visitReplicateOperator(ReplicateOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("replicate ");
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitInsertDeleteOperator(InsertDeleteOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
+ addIndent(buffer, indent).append(header).append(op.getDataSource()).append(" from ")
+ .append(op.getPayloadExpression()).append(" partitioned by ")
+ .append(op.getPrimaryKeyExpressions().toString());
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, Integer indent)
+ throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ String header = op.getOperation() == Kind.INSERT ? "insert into " : "delete from ";
+ addIndent(buffer, indent).append(header).append(op.getDataSourceIndex()).append(" from ")
+ .append(op.getSecondaryKeyExpressions().toString()).append(" ")
+ .append(op.getPrimaryKeyExpressions().toString());
+ return buffer.toString();
+ }
+
+ @Override
+ public String visitSinkOperator(SinkOperator op, Integer indent) throws AlgebricksException {
+ StringBuilder buffer = new StringBuilder();
+ addIndent(buffer, indent).append("sink");
+ return buffer.toString();
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
new file mode 100644
index 0000000..174db0b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/prettyprint/PlanPrettyPrinter.java
@@ -0,0 +1,90 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class PlanPrettyPrinter {
+ public static void printPlan(ILogicalPlan plan, StringBuilder out, LogicalOperatorPrettyPrintVisitor pvisitor,
+ int indent) throws AlgebricksException {
+ for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+ printOperator((AbstractLogicalOperator) root.getValue(), out, pvisitor, indent);
+ }
+ }
+
+ public static void printPhysicalOps(ILogicalPlan plan, StringBuilder out, int indent) {
+ for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+ printPhysicalOperator((AbstractLogicalOperator) root.getValue(), indent, out);
+ }
+ }
+
+ public static void printOperator(AbstractLogicalOperator op, StringBuilder out,
+ LogicalOperatorPrettyPrintVisitor pvisitor, int indent) throws AlgebricksException {
+ out.append(op.accept(pvisitor, indent));
+ IPhysicalOperator pOp = op.getPhysicalOperator();
+
+ if (pOp != null) {
+ out.append("\n");
+ pad(out, indent);
+ appendln(out, "-- " + pOp.toString() + " |" + op.getExecutionMode() + "|");
+ } else {
+ appendln(out, " -- |" + op.getExecutionMode() + "|");
+ }
+
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ printOperator((AbstractLogicalOperator) i.getValue(), out, pvisitor, indent + 2);
+ }
+
+ }
+
+ public static void printPhysicalOperator(AbstractLogicalOperator op, int indent, StringBuilder out) {
+ IPhysicalOperator pOp = op.getPhysicalOperator();
+ pad(out, indent);
+ appendln(out, "-- " + pOp.toString() + " |" + op.getExecutionMode() + "|");
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans opNest = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : opNest.getNestedPlans()) {
+ pad(out, indent + 8);
+ appendln(out, "{");
+ printPhysicalOps(p, out, indent + 10);
+ pad(out, indent + 8);
+ appendln(out, "}");
+ }
+ }
+
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ printPhysicalOperator((AbstractLogicalOperator) i.getValue(), indent + 2, out);
+ }
+
+ }
+
+ private static void appendln(StringBuilder buf, String s) {
+ buf.append(s);
+ buf.append("\n");
+ }
+
+ private static void pad(StringBuilder buf, int indent) {
+ for (int i = 0; i < indent; ++i) {
+ buf.append(' ');
+ }
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
new file mode 100644
index 0000000..c11131a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AbstractGroupingProperty.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public abstract class AbstractGroupingProperty {
+ protected Set<LogicalVariable> columnSet;
+
+ public AbstractGroupingProperty(Set<LogicalVariable> columnSet) {
+ this.columnSet = columnSet;
+ }
+
+ public Set<LogicalVariable> getColumnSet() {
+ return columnSet;
+ }
+
+ public final void normalizeGroupingColumns(Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+ List<FunctionalDependency> fds) {
+ replaceGroupingColumnsByEqClasses(equivalenceClasses);
+ applyFDsToGroupingColumns(fds);
+ }
+
+ private void replaceGroupingColumnsByEqClasses(Map<LogicalVariable, EquivalenceClass> equivalenceClasses) {
+ if (equivalenceClasses == null || equivalenceClasses.isEmpty()) {
+ return;
+ }
+ HashSet<LogicalVariable> norm = new HashSet<LogicalVariable>();
+ for (LogicalVariable v : columnSet) {
+ EquivalenceClass ec = equivalenceClasses.get(v);
+ if (ec == null) {
+ norm.add(v);
+ } else {
+ if (ec.representativeIsConst()) {
+ // trivially satisfied, so the var. can be removed
+ } else {
+ norm.add(ec.getVariableRepresentative());
+ }
+ }
+ }
+ columnSet = norm;
+ }
+
+ private void applyFDsToGroupingColumns(List<FunctionalDependency> fds) {
+ // the set of vars. is unordered
+ // so we try all FDs on all variables (incomplete algo?)
+ if (fds == null || fds.isEmpty()) {
+ return;
+ }
+ HashSet<LogicalVariable> norm = new HashSet<LogicalVariable>();
+ for (LogicalVariable v : columnSet) {
+ boolean isImpliedByAnFD = false;
+ for (FunctionalDependency fdep : fds) {
+ if (columnSet.containsAll(fdep.getHead()) && fdep.getTail().contains(v)) {
+ isImpliedByAnFD = true;
+ norm.addAll(fdep.getHead());
+ break;
+ }
+
+ }
+ if (!isImpliedByAnFD) {
+ norm.add(v);
+ }
+ }
+ columnSet = norm;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AsterixNodeGroupDomain.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AsterixNodeGroupDomain.java
new file mode 100644
index 0000000..d27534a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/AsterixNodeGroupDomain.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public class AsterixNodeGroupDomain implements INodeDomain {
+
+ private String groupName;
+
+ public AsterixNodeGroupDomain(String groupName) {
+ this.groupName = groupName;
+ }
+
+ @Override
+ public boolean sameAs(INodeDomain domain) {
+ if (!(domain instanceof AsterixNodeGroupDomain)) {
+ return false;
+ }
+ AsterixNodeGroupDomain dom2 = (AsterixNodeGroupDomain) domain;
+ return groupName.equals(dom2.groupName);
+ }
+
+ @Override
+ public String toString() {
+ return "AsterixDomain(" + groupName + ")";
+ }
+
+ @Override
+ public Integer cardinality() {
+ return null;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
new file mode 100644
index 0000000..8a38cfb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/BroadcastPartitioningProperty.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class BroadcastPartitioningProperty implements IPartitioningProperty {
+
+ private INodeDomain domain;
+
+ public BroadcastPartitioningProperty(INodeDomain domain) {
+ this.domain = domain;
+ }
+
+ @Override
+ public PartitioningType getPartitioningType() {
+ return PartitioningType.BROADCAST;
+ }
+
+ @Override
+ public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+ // do nothing
+ }
+
+ @Override
+ public void getColumns(Collection<LogicalVariable> columns) {
+ }
+
+ @Override
+ public INodeDomain getNodeDomain() {
+ return domain;
+ }
+
+ @Override
+ public void setNodeDomain(INodeDomain domain) {
+ this.domain = domain;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
new file mode 100644
index 0000000..876ee52
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FileSplitDomain.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+
+public class FileSplitDomain implements INodeDomain {
+
+ private FileSplit[] splits;
+
+ public FileSplitDomain(FileSplit[] splits) {
+ this.splits = splits;
+ }
+
+ @Override
+ public Integer cardinality() {
+ return splits.length;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("FileSplitDomain[");
+ boolean fst = true;
+ for (FileSplit fs : splits) {
+ if (fst) {
+ fst = false;
+ } else {
+ sb.append(", ");
+ }
+ sb.append(fs.getNodeName() + ":" + fs.getLocalFile());
+ }
+ sb.append(']');
+ return sb.toString();
+ }
+
+ @Override
+ public boolean sameAs(INodeDomain domain) {
+ if (!(domain instanceof FileSplitDomain)) {
+ return false;
+ }
+ FileSplitDomain fsd = (FileSplitDomain) domain;
+ if (fsd.splits.length != splits.length) {
+ return false;
+ }
+ // conservative approach...
+ for (int i = 0; i < splits.length; i++) {
+ if (!ncEq(splits[i], fsd.splits[i])) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private boolean ncEq(FileSplit fs1, FileSplit fs2) {
+ return fs1.getNodeName().equals(fs2.getNodeName());
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java
new file mode 100644
index 0000000..40bbc36
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FilteredVariablePropagationPolicy.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+
+public class FilteredVariablePropagationPolicy extends VariablePropagationPolicy {
+
+ List<LogicalVariable> toPropagate;
+
+ public FilteredVariablePropagationPolicy(List<LogicalVariable> varList) {
+ toPropagate = varList;
+ }
+
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
+ for (LogicalVariable v : toPropagate) {
+ target.addVariable(v);
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java
new file mode 100644
index 0000000..1c76171
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/FunctionalDependency.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public final class FunctionalDependency {
+ private List<LogicalVariable> head;
+ private List<LogicalVariable> tail;
+
+ public FunctionalDependency(List<LogicalVariable> head, List<LogicalVariable> tail) {
+ this.head = head;
+ this.tail = tail;
+ }
+
+ public List<LogicalVariable> getHead() {
+ return head;
+ }
+
+ public List<LogicalVariable> getTail() {
+ return tail;
+ }
+
+ @Override
+ public String toString() {
+ return head + "->" + tail;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof FunctionalDependency)) {
+ return false;
+ } else {
+ FunctionalDependency fd = (FunctionalDependency) obj;
+ return fd.getHead().equals(this.head) && fd.getTail().equals(this.tail);
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java
new file mode 100644
index 0000000..fbe30f5
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILocalStructuralProperty.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface ILocalStructuralProperty extends IStructuralProperty {
+ public enum PropertyType {
+ LOCAL_GROUPING_PROPERTY,
+ LOCAL_ORDER_PROPERTY
+ }
+
+ public void getVariables(Collection<LogicalVariable> variables);
+
+ public PropertyType getPropertyType();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java
new file mode 100644
index 0000000..214a579
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/ILogicalPropertiesVector.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public interface ILogicalPropertiesVector {
+ public Integer getNumberOfTuples();
+
+ public Integer getMaxOutputFrames();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/INodeDomain.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/INodeDomain.java
new file mode 100644
index 0000000..76057f6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/INodeDomain.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public interface INodeDomain {
+ public boolean sameAs(INodeDomain domain);
+
+ /** @return the number of nodes in that domain or null if unknown */
+ public Integer cardinality();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
new file mode 100644
index 0000000..8a0cab7
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningProperty.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IPartitioningProperty extends IStructuralProperty {
+ public enum PartitioningType {
+ UNPARTITIONED, RANDOM, BROADCAST, UNORDERED_PARTITIONED, ORDERED_PARTITIONED
+ }
+
+ static final INodeDomain DOMAIN_FOR_UNPARTITIONED_DATA = new INodeDomain() {
+ @Override
+ public boolean sameAs(INodeDomain domain) {
+ return domain == this;
+ }
+
+ @Override
+ public Integer cardinality() {
+ return null;
+ }
+ };
+
+ public static final IPartitioningProperty UNPARTITIONED = new IPartitioningProperty() {
+
+ @Override
+ public PartitioningType getPartitioningType() {
+ return PartitioningType.UNPARTITIONED;
+ }
+
+ @Override
+ public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+ // do nothing
+ }
+
+ @Override
+ public void getColumns(Collection<LogicalVariable> columns) {
+ }
+
+ @Override
+ public INodeDomain getNodeDomain() {
+ return DOMAIN_FOR_UNPARTITIONED_DATA;
+ }
+
+ @Override
+ public String toString() {
+ return getPartitioningType().toString();
+ }
+
+ @Override
+ public void setNodeDomain(INodeDomain domain) {
+ throw new IllegalStateException();
+ }
+ };
+
+ public abstract PartitioningType getPartitioningType();
+
+ public abstract void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+ List<FunctionalDependency> fds);
+
+ public abstract INodeDomain getNodeDomain();
+
+ public abstract void setNodeDomain(INodeDomain domain);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
new file mode 100644
index 0000000..d884a17
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPartitioningRequirementsCoordinator.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+/**
+ *
+ * Implements constraints in between requirements for the children of the same
+ * operator.
+ *
+ */
+
+public interface IPartitioningRequirementsCoordinator {
+
+ public static IPartitioningRequirementsCoordinator NO_COORDINATION = new IPartitioningRequirementsCoordinator() {
+
+ @Override
+ public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty requirements,
+ IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context) {
+ return new Pair<Boolean, IPartitioningProperty>(true, requirements);
+ }
+ };
+
+ public static IPartitioningRequirementsCoordinator EQCLASS_PARTITIONING_COORDINATOR = new IPartitioningRequirementsCoordinator() {
+
+ @Override
+ public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty rqdpp,
+ IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ if (firstDeliveredPartitioning != null
+ && firstDeliveredPartitioning.getPartitioningType() == rqdpp.getPartitioningType()) {
+ switch (rqdpp.getPartitioningType()) {
+ case UNORDERED_PARTITIONED: {
+ UnorderedPartitionedProperty upp1 = (UnorderedPartitionedProperty) firstDeliveredPartitioning;
+ Set<LogicalVariable> set1 = upp1.getColumnSet();
+ UnorderedPartitionedProperty uppreq = (UnorderedPartitionedProperty) rqdpp;
+ Set<LogicalVariable> modifuppreq = new HashSet<LogicalVariable>();
+ Map<LogicalVariable, EquivalenceClass> eqmap = context.getEquivalenceClassMap(op);
+ Set<LogicalVariable> covered = new HashSet<LogicalVariable>();
+ for (LogicalVariable r : uppreq.getColumnSet()) {
+ EquivalenceClass ec = eqmap.get(r);
+ for (LogicalVariable v : set1) {
+ EquivalenceClass ecFirst = eqmap.get(v);
+ if (ecFirst == ec) {
+ covered.add(v);
+ modifuppreq.add(r);
+ break;
+ }
+ }
+ }
+ if (!covered.equals(set1)) {
+ throw new AlgebricksException("Could not modify " + rqdpp
+ + " to agree with partitioning property " + firstDeliveredPartitioning
+ + " delivered by previous input operator.");
+ }
+ UnorderedPartitionedProperty upp2 = new UnorderedPartitionedProperty(modifuppreq,
+ rqdpp.getNodeDomain());
+ return new Pair<Boolean, IPartitioningProperty>(false, upp2);
+ }
+ case ORDERED_PARTITIONED: {
+ throw new NotImplementedException();
+ }
+ }
+ }
+ return new Pair<Boolean, IPartitioningProperty>(true, rqdpp);
+ }
+
+ };
+
+ public Pair<Boolean, IPartitioningProperty> coordinateRequirements(IPartitioningProperty requirements,
+ IPartitioningProperty firstDeliveredPartitioning, ILogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java
new file mode 100644
index 0000000..72f9683
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPhysicalPropertiesVector.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IPhysicalPropertiesVector {
+
+ public IPartitioningProperty getPartitioningProperty();
+
+ public List<ILocalStructuralProperty> getLocalProperties();
+
+ /**
+ *
+ * @param reqd
+ * vector of required properties
+ * @param equivalenceClasses
+ * @param fds
+ * @return a vector of properties from pvector that are not delivered by the
+ * current vector or null if none
+ */
+ public IPhysicalPropertiesVector getUnsatisfiedPropertiesFrom(IPhysicalPropertiesVector reqd,
+ boolean mayExpandProperties, Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+ List<FunctionalDependency> fds);
+
+ public IPhysicalPropertiesVector clone();
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java
new file mode 100644
index 0000000..02afec1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IPropertiesComputer.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+
+public interface IPropertiesComputer {
+
+ // can return null
+ public IPartitioningProperty computePartitioningProperty(ILogicalExpression expr);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java
new file mode 100644
index 0000000..aabf9e9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/IStructuralProperty.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public interface IStructuralProperty {
+ public void getColumns(Collection<LogicalVariable> columns);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java
new file mode 100644
index 0000000..5c82ee7
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalGroupingProperty.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class LocalGroupingProperty extends AbstractGroupingProperty implements ILocalStructuralProperty {
+
+ // preferredOrderEnforcer, if not null, is guaranteed to enforce grouping on
+ // columnSet
+ private List<LogicalVariable> preferredOrderEnforcer;
+
+ public LocalGroupingProperty(Set<LogicalVariable> columnSet) {
+ super(columnSet);
+ }
+
+ public LocalGroupingProperty(Set<LogicalVariable> columnSet, List<LogicalVariable> preferredOrderEnforcer) {
+ this(columnSet);
+ this.preferredOrderEnforcer = preferredOrderEnforcer;
+ }
+
+ @Override
+ public PropertyType getPropertyType() {
+ return PropertyType.LOCAL_GROUPING_PROPERTY;
+ }
+
+ @Override
+ public void getColumns(Collection<LogicalVariable> columns) {
+ columns.addAll(columnSet);
+ }
+
+ @Override
+ public String toString() {
+ return columnSet.toString();
+ }
+
+ @Override
+ public void getVariables(Collection<LogicalVariable> variables) {
+ variables.addAll(columnSet);
+ }
+
+ public List<LogicalVariable> getPreferredOrderEnforcer() {
+ return preferredOrderEnforcer;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java
new file mode 100644
index 0000000..7810df7
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LocalOrderProperty.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+
+public final class LocalOrderProperty implements ILocalStructuralProperty {
+
+ private OrderColumn orderColumn;
+
+ public LocalOrderProperty(OrderColumn orderColumn) {
+ this.orderColumn = orderColumn;
+ }
+
+ public OrderColumn getOrderColumn() {
+ return orderColumn;
+ }
+
+ public void setOrderColumn(OrderColumn orderColumn) {
+ this.orderColumn = orderColumn;
+ }
+
+ public LogicalVariable getColumn() {
+ return orderColumn.getColumn();
+ }
+
+ public OrderKind getOrder() {
+ return orderColumn.getOrder();
+ }
+
+ @Override
+ public PropertyType getPropertyType() {
+ return PropertyType.LOCAL_ORDER_PROPERTY;
+ }
+
+ @Override
+ public void getColumns(Collection<LogicalVariable> columns) {
+ columns.add(getColumn());
+ }
+
+ @Override
+ public String toString() {
+ return orderColumn.toString();
+ }
+
+ @Override
+ public void getVariables(Collection<LogicalVariable> variables) {
+ variables.add(orderColumn.getColumn());
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java
new file mode 100644
index 0000000..ae25f41
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/LogicalPropertiesVectorImpl.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public class LogicalPropertiesVectorImpl implements ILogicalPropertiesVector {
+
+ private Integer numTuples, maxOutputFrames;
+
+ @Override
+ public String toString() {
+ return "LogicalPropertiesVector [ num.tuples: " + numTuples + ", maxOutputFrames: " + maxOutputFrames + " ]";
+ }
+
+ @Override
+ public Integer getNumberOfTuples() {
+ return numTuples;
+ }
+
+ public void setNumberOfTuples(Integer numTuples) {
+ this.numTuples = numTuples;
+ }
+
+ @Override
+ public Integer getMaxOutputFrames() {
+ return maxOutputFrames;
+ }
+
+ public void setMaxOutputFrames(Integer maxOutputFrames) {
+ this.maxOutputFrames = maxOutputFrames;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderColumn.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderColumn.java
new file mode 100644
index 0000000..5f22b95
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderColumn.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+
+public final class OrderColumn {
+
+ private LogicalVariable column;
+ private OrderKind order;
+
+ public OrderColumn(LogicalVariable column, OrderKind order) {
+ this.column = column;
+ this.order = order;
+ }
+
+ public LogicalVariable getColumn() {
+ return column;
+ }
+
+ public OrderKind getOrder() {
+ return order;
+ }
+
+ public void setColumn(LogicalVariable column) {
+ this.column = column;
+ }
+
+ public void setOrder(OrderKind order) {
+ this.order = order;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof OrderColumn)) {
+ return false;
+ } else {
+ OrderColumn oc = (OrderColumn) obj;
+ return column.equals(oc.getColumn()) && order == oc.getOrder();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return column.toString() + "(" + order + ")";
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
new file mode 100644
index 0000000..137c46d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/OrderedPartitionedProperty.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class OrderedPartitionedProperty implements IPartitioningProperty {
+
+ private ArrayList<OrderColumn> orderColumns;
+ private INodeDomain domain;
+
+ public OrderedPartitionedProperty(ArrayList<OrderColumn> orderColumns, INodeDomain domain) {
+ this.domain = domain;
+ this.orderColumns = orderColumns;
+ }
+
+ public ArrayList<OrderColumn> getOrderColumns() {
+ return orderColumns;
+ }
+
+ public ArrayList<LogicalVariable> getColumns() {
+ ArrayList<LogicalVariable> cols = new ArrayList<LogicalVariable>(orderColumns.size());
+ for (OrderColumn oc : orderColumns) {
+ cols.add(oc.getColumn());
+ }
+ return cols;
+ }
+
+ @Override
+ public PartitioningType getPartitioningType() {
+ return PartitioningType.ORDERED_PARTITIONED;
+ }
+
+ @Override
+ public String toString() {
+ return getPartitioningType().toString() + orderColumns;
+ }
+
+ @Override
+ public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+ orderColumns = PropertiesUtil.replaceOrderColumnsByEqClasses(orderColumns, equivalenceClasses);
+ orderColumns = PropertiesUtil.applyFDsToOrderColumns(orderColumns, fds);
+ }
+
+ @Override
+ public void getColumns(Collection<LogicalVariable> columns) {
+ for (OrderColumn oc : orderColumns) {
+ columns.add(oc.getColumn());
+ }
+ }
+
+ @Override
+ public INodeDomain getNodeDomain() {
+ return domain;
+ }
+
+ @Override
+ public void setNodeDomain(INodeDomain domain) {
+ this.domain = domain;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java
new file mode 100644
index 0000000..7503a3c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PhysicalRequirements.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+public class PhysicalRequirements {
+ private final IPhysicalPropertiesVector[] requiredProperties;
+ private final IPartitioningRequirementsCoordinator partitioningCoordinator;
+
+ public PhysicalRequirements(IPhysicalPropertiesVector[] requiredProperties,
+ IPartitioningRequirementsCoordinator partitioningCoordinator) {
+ this.requiredProperties = requiredProperties;
+ this.partitioningCoordinator = partitioningCoordinator;
+ }
+
+ public IPhysicalPropertiesVector[] getRequiredProperties() {
+ return requiredProperties;
+ }
+
+ public IPartitioningRequirementsCoordinator getPartitioningCoordinator() {
+ return partitioningCoordinator;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
new file mode 100644
index 0000000..ad80e46
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/PropertiesUtil.java
@@ -0,0 +1,323 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+
+public class PropertiesUtil {
+
+ public Set<LogicalVariable> closureUnderFDs(Collection<LogicalVariable> vars, List<FunctionalDependency> fdList) {
+ Set<LogicalVariable> k = new HashSet<LogicalVariable>(vars);
+ boolean change;
+ do {
+ change = false;
+ for (FunctionalDependency fd : fdList) {
+ List<LogicalVariable> h = fd.getHead();
+ if (k.containsAll(h)) {
+ List<LogicalVariable> t = fd.getTail();
+ for (LogicalVariable v : t) {
+ if (!(k.contains(v))) {
+ k.add(v);
+ change = true;
+ }
+ }
+ }
+ }
+ } while (change);
+ return k;
+ }
+
+ public static boolean matchLocalProperties(List<ILocalStructuralProperty> reqd,
+ List<ILocalStructuralProperty> dlvd, Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+ List<FunctionalDependency> fds) {
+ if (reqd == null) {
+ return true;
+ }
+ if (dlvd == null) {
+ return false;
+ }
+ normalizeLocals(reqd, equivalenceClasses, fds);
+ normalizeLocals(dlvd, equivalenceClasses, fds);
+
+ ListIterator<ILocalStructuralProperty> dlvdIter = dlvd.listIterator();
+
+ Set<LogicalVariable> rqdCols = new HashSet<LogicalVariable>();
+ Set<LogicalVariable> dlvdCols = new HashSet<LogicalVariable>();
+ for (ILocalStructuralProperty r : reqd) {
+ if (r.getPropertyType() == PropertyType.LOCAL_GROUPING_PROPERTY) {
+ rqdCols.clear();
+ r.getVariables(rqdCols);
+ }
+ boolean implied = false;
+ while (!implied && dlvdIter.hasNext()) {
+ ILocalStructuralProperty d = dlvdIter.next();
+ switch (r.getPropertyType()) {
+ case LOCAL_ORDER_PROPERTY: {
+ if (d.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
+ return false;
+ }
+ LocalOrderProperty lop = (LocalOrderProperty) d;
+ if (lop.getColumn() == ((LocalOrderProperty) r).getColumn()
+ && lop.getOrder() == ((LocalOrderProperty) r).getOrder()) {
+ implied = true;
+ } else {
+ return false;
+ }
+ break;
+ }
+ case LOCAL_GROUPING_PROPERTY: {
+ dlvdCols.clear();
+ d.getColumns(dlvdCols);
+ for (LogicalVariable v : dlvdCols) {
+ if (rqdCols.contains(v)) {
+ rqdCols.remove(v);
+ } else {
+ return false;
+ }
+ }
+ if (rqdCols.isEmpty()) {
+ implied = true;
+ }
+ break;
+ }
+ default: {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ if (!implied) {
+ return false;
+ }
+ }
+ return true;
+
+ }
+
+ public static boolean matchPartitioningProps(IPartitioningProperty reqd, IPartitioningProperty dlvd,
+ boolean mayExpandProperties) {
+
+ INodeDomain dom1 = reqd.getNodeDomain();
+ INodeDomain dom2 = dlvd.getNodeDomain();
+ if (dom1 != null && dom2 != null && !dom1.sameAs(dom2)) {
+ return false;
+ }
+
+ switch (reqd.getPartitioningType()) {
+ case RANDOM: {
+ // anything matches RANDOM
+ return true;
+ }
+ case UNORDERED_PARTITIONED: {
+ switch (dlvd.getPartitioningType()) {
+ case UNORDERED_PARTITIONED: {
+ UnorderedPartitionedProperty ur = (UnorderedPartitionedProperty) reqd;
+ UnorderedPartitionedProperty ud = (UnorderedPartitionedProperty) dlvd;
+ if (mayExpandProperties) {
+ return ur.getColumnSet().containsAll(ud.getColumnSet());
+ } else {
+ return ur.getColumnSet().equals(ud.getColumnSet());
+ }
+ }
+ case ORDERED_PARTITIONED: {
+ UnorderedPartitionedProperty ur = (UnorderedPartitionedProperty) reqd;
+ OrderedPartitionedProperty od = (OrderedPartitionedProperty) dlvd;
+ if (mayExpandProperties) {
+ return ur.getColumnSet().containsAll(od.getOrderColumns());
+ } else {
+ return ur.getColumnSet().containsAll(od.getOrderColumns())
+ && od.getOrderColumns().containsAll(ur.getColumnSet());
+ }
+ }
+ default: {
+ return false;
+ }
+ }
+ }
+ case ORDERED_PARTITIONED: {
+ switch (dlvd.getPartitioningType()) {
+ case ORDERED_PARTITIONED: {
+ OrderedPartitionedProperty or = (OrderedPartitionedProperty) reqd;
+ OrderedPartitionedProperty od = (OrderedPartitionedProperty) dlvd;
+ if (mayExpandProperties) {
+ return isPrefixOf(od.getOrderColumns(), or.getOrderColumns());
+ } else {
+ return od.getOrderColumns().equals(or.getOrderColumns());
+ }
+ }
+ default: {
+ return false;
+ }
+ }
+ }
+ default: {
+ return (dlvd.getPartitioningType() == reqd.getPartitioningType());
+ }
+ }
+ }
+
+ /**
+ *
+ * @param pref
+ * @param target
+ * @return true iff pref is a prefix of target
+ */
+ private static boolean isPrefixOf(List<OrderColumn> pref, List<OrderColumn> target) {
+ Iterator<OrderColumn> iter = target.iterator();
+ for (OrderColumn v : pref) {
+ if (!iter.hasNext()) {
+ return false;
+ }
+ if (!v.equals(iter.next())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static ArrayList<OrderColumn> applyFDsToOrderColumns(ArrayList<OrderColumn> orderColumns,
+ List<FunctionalDependency> fds) {
+ // the set of vars. is ordered
+ // so we try the variables in order from last to first
+ if (fds == null || fds.isEmpty()) {
+ return orderColumns;
+ }
+
+ int deleted = 0;
+ for (int i = orderColumns.size() - 1; i >= 0; i--) {
+ for (FunctionalDependency fdep : fds) {
+ if (impliedByPrefix(orderColumns, i, fdep)) {
+ orderColumns.set(i, null);
+ deleted++;
+ break;
+ }
+ }
+ }
+ ArrayList<OrderColumn> norm = new ArrayList<OrderColumn>(orderColumns.size() - deleted);
+ for (OrderColumn oc : orderColumns) {
+ if (oc != null) {
+ norm.add(oc);
+ }
+ }
+ return norm;
+ }
+
+ public static ArrayList<OrderColumn> replaceOrderColumnsByEqClasses(ArrayList<OrderColumn> orderColumns,
+ Map<LogicalVariable, EquivalenceClass> equivalenceClasses) {
+ if (equivalenceClasses == null || equivalenceClasses.isEmpty()) {
+ return orderColumns;
+ }
+ ArrayList<OrderColumn> norm = new ArrayList<OrderColumn>();
+ for (OrderColumn v : orderColumns) {
+ EquivalenceClass ec = equivalenceClasses.get(v.getColumn());
+ if (ec == null) {
+ norm.add(v);
+ } else {
+ if (ec.representativeIsConst()) {
+ // trivially satisfied, so the var. can be removed
+ } else {
+ norm.add(new OrderColumn(ec.getVariableRepresentative(), v.getOrder()));
+ }
+ }
+ }
+ return norm;
+ }
+
+ private static boolean impliedByPrefix(ArrayList<OrderColumn> vars, int i, FunctionalDependency fdep) {
+ if (!fdep.getTail().contains(vars.get(i).getColumn())) {
+ return false;
+ }
+ boolean fdSat = true;
+ for (LogicalVariable pv : fdep.getHead()) {
+ boolean isInPrefix = false;
+ for (int j = 0; j < i; j++) {
+ if (vars.get(j).getColumn().equals(pv)) {
+ isInPrefix = true;
+ break;
+ }
+ }
+ if (!isInPrefix) {
+ fdSat = false;
+ break;
+ }
+ }
+ return fdSat;
+ }
+
+ private static void normalizeLocals(List<ILocalStructuralProperty> props,
+ Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+ ListIterator<ILocalStructuralProperty> propIter = props.listIterator();
+ int pos = -1;
+ while (propIter.hasNext()) {
+ ILocalStructuralProperty p = propIter.next();
+ if (p.getPropertyType() == PropertyType.LOCAL_GROUPING_PROPERTY) {
+ ((LocalGroupingProperty) p).normalizeGroupingColumns(equivalenceClasses, fds);
+ pos++;
+ } else {
+ LocalOrderProperty ord = (LocalOrderProperty) p;
+ EquivalenceClass ec = equivalenceClasses.get(ord.getColumn());
+ if (ec != null) {
+ if (ec.representativeIsConst()) {
+ propIter.remove();
+ } else {
+ ord.getOrderColumn().setColumn(ec.getVariableRepresentative());
+ pos++;
+ }
+ } else {
+ pos++;
+ }
+ }
+ }
+
+ if (pos < 1) {
+ return;
+ }
+
+ while (propIter.hasPrevious()) {
+ ILocalStructuralProperty p = propIter.previous();
+ ListIterator<ILocalStructuralProperty> secondIter = props.listIterator(pos);
+ pos--;
+ Set<LogicalVariable> cols = new HashSet<LogicalVariable>();
+ while (secondIter.hasPrevious()) {
+ secondIter.previous().getColumns(cols);
+ }
+ secondIter = null;
+ for (FunctionalDependency fdep : fds) {
+ LinkedList<LogicalVariable> columnsOfP = new LinkedList<LogicalVariable>();
+ p.getColumns(columnsOfP);
+ if (impliedByPrefix(columnsOfP, cols, fdep)) {
+ propIter.remove();
+ break;
+ }
+ }
+ }
+ }
+
+ private static boolean impliedByPrefix(List<LogicalVariable> colsOfProp, Set<LogicalVariable> colsOfPrefix,
+ FunctionalDependency fdep) {
+ return fdep.getTail().containsAll(colsOfProp) && colsOfPrefix.containsAll(fdep.getHead());
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
new file mode 100644
index 0000000..66e900e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/RandomPartitioningProperty.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class RandomPartitioningProperty implements IPartitioningProperty {
+
+ private INodeDomain domain;
+
+ public RandomPartitioningProperty(INodeDomain domain) {
+ this.domain = domain;
+ }
+
+ @Override
+ public PartitioningType getPartitioningType() {
+ return PartitioningType.RANDOM;
+ }
+
+ @Override
+ public String toString() {
+ return getPartitioningType() + " domain:" + domain;
+ }
+
+ @Override
+ public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+ // do nothing
+ }
+
+ @Override
+ public void getColumns(Collection<LogicalVariable> columns) {
+ }
+
+ @Override
+ public INodeDomain getNodeDomain() {
+ return domain;
+ }
+
+ @Override
+ public void setNodeDomain(INodeDomain domain) {
+ this.domain = domain;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java
new file mode 100644
index 0000000..d8d8f35
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/StructuralPropertiesVector.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public class StructuralPropertiesVector implements IPhysicalPropertiesVector {
+ private List<ILocalStructuralProperty> propsLocal;
+ private IPartitioningProperty propPartitioning;
+
+ public static final StructuralPropertiesVector EMPTY_PROPERTIES_VECTOR = new StructuralPropertiesVector(null,
+ new ArrayList<ILocalStructuralProperty>());
+
+ public StructuralPropertiesVector(IPartitioningProperty propPartitioning, List<ILocalStructuralProperty> propsLocal) {
+ this.propPartitioning = propPartitioning;
+ this.propsLocal = propsLocal;
+ }
+
+ @Override
+ public String toString() {
+ return "propsLocal=" + propsLocal + "\tpropPartioning=" + propPartitioning;
+ }
+
+ @Override
+ public IPartitioningProperty getPartitioningProperty() {
+ return propPartitioning;
+ }
+
+ @Override
+ public List<ILocalStructuralProperty> getLocalProperties() {
+ return propsLocal;
+ }
+
+ @Override
+ public IPhysicalPropertiesVector clone() {
+ List<ILocalStructuralProperty> propsCopy = new LinkedList<ILocalStructuralProperty>();
+ if (propsLocal != null) {
+ propsCopy.addAll(propsLocal);
+ }
+ return new StructuralPropertiesVector(propPartitioning, propsCopy);
+ }
+
+ /**
+ *
+ * @param reqd
+ * vector of required properties
+ * @return a vector of properties from pvector that are not delivered by the
+ * current vector or null if none
+ */
+ @Override
+ public IPhysicalPropertiesVector getUnsatisfiedPropertiesFrom(IPhysicalPropertiesVector reqd,
+ boolean mayExpandProperties, Map<LogicalVariable, EquivalenceClass> equivalenceClasses,
+ List<FunctionalDependency> fds) {
+ List<ILocalStructuralProperty> plist = reqd.getLocalProperties();
+ List<ILocalStructuralProperty> diffLocals = null;
+ if (plist != null && !plist.isEmpty()) {
+ if (!PropertiesUtil.matchLocalProperties(plist, propsLocal, equivalenceClasses, fds)) {
+ diffLocals = plist;
+ }
+ }
+
+ IPartitioningProperty diffPart = null;
+ IPartitioningProperty reqdPart = reqd.getPartitioningProperty();
+ if (reqdPart != null) {
+ if (mayExpandProperties) {
+ reqdPart.normalize(equivalenceClasses, fds);
+ } else {
+ reqdPart.normalize(equivalenceClasses, null);
+ }
+ propPartitioning.normalize(equivalenceClasses, fds);
+ if (!PropertiesUtil.matchPartitioningProps(reqdPart, propPartitioning, mayExpandProperties)) {
+ diffPart = reqdPart;
+ }
+ }
+
+ if (diffLocals == null && diffPart == null) {
+ return null;
+ } else {
+ return new StructuralPropertiesVector(diffPart, diffLocals);
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
new file mode 100644
index 0000000..6b14a9f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/TypePropagationPolicy.java
@@ -0,0 +1,68 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypeEnvPointer;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public abstract class TypePropagationPolicy {
+ public static final TypePropagationPolicy ALL = new TypePropagationPolicy() {
+
+ @Override
+ public Object getVarType(LogicalVariable var, INullableTypeComputer ntc,
+ List<LogicalVariable> nonNullVariableList, ITypeEnvPointer... typeEnvs) throws AlgebricksException {
+ for (ITypeEnvPointer p : typeEnvs) {
+ IVariableTypeEnvironment env = p.getTypeEnv();
+ if (env == null) {
+ throw new AlgebricksException("Null environment for pointer " + p + " in getVarType for var=" + var);
+ }
+ Object t = env.getVarType(var, nonNullVariableList);
+ if (t != null) {
+ return t;
+ }
+ }
+ return null;
+ }
+ };
+
+ public static final TypePropagationPolicy LEFT_OUTER = new TypePropagationPolicy() {
+
+ @Override
+ public Object getVarType(LogicalVariable var, INullableTypeComputer ntc,
+ List<LogicalVariable> nonNullVariableList, ITypeEnvPointer... typeEnvs) throws AlgebricksException {
+ int n = typeEnvs.length;
+ for (int i = 0; i < n; i++) {
+ Object t = typeEnvs[i].getTypeEnv().getVarType(var, nonNullVariableList);
+ if (t != null) {
+ if (i == 0) { // inner branch
+ return t;
+ } else { // outer branch
+ boolean nonNullVarIsProduced = false;
+ for (LogicalVariable v : nonNullVariableList) {
+ if (v == var) {
+ nonNullVarIsProduced = true;
+ break;
+ }
+ if (typeEnvs[i].getTypeEnv().getVarType(v) != null) {
+ nonNullVarIsProduced = true;
+ break;
+ }
+ }
+ if (nonNullVarIsProduced) {
+ return t;
+ } else {
+ return ntc.makeNullableType(t);
+ }
+ }
+ }
+ }
+ return null;
+ }
+ };
+
+ public abstract Object getVarType(LogicalVariable var, INullableTypeComputer ntc,
+ List<LogicalVariable> nonNullVariableList, ITypeEnvPointer... typeEnvs) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
new file mode 100644
index 0000000..520e7db
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnorderedPartitionedProperty.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+
+public final class UnorderedPartitionedProperty extends AbstractGroupingProperty implements IPartitioningProperty {
+
+ private INodeDomain domain;
+
+ public UnorderedPartitionedProperty(Set<LogicalVariable> partitioningVariables, INodeDomain domain) {
+ super(partitioningVariables);
+ this.domain = domain;
+ }
+
+ @Override
+ public PartitioningType getPartitioningType() {
+ return PartitioningType.UNORDERED_PARTITIONED;
+ }
+
+ @Override
+ public void normalize(Map<LogicalVariable, EquivalenceClass> equivalenceClasses, List<FunctionalDependency> fds) {
+ normalizeGroupingColumns(equivalenceClasses, fds);
+ }
+
+ @Override
+ public String toString() {
+ return getPartitioningType().toString() + columnSet;
+ }
+
+ @Override
+ public void getColumns(Collection<LogicalVariable> columns) {
+ columns.addAll(columnSet);
+ }
+
+ @Override
+ public INodeDomain getNodeDomain() {
+ return domain;
+ }
+
+ @Override
+ public void setNodeDomain(INodeDomain domain) {
+ this.domain = domain;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java
new file mode 100644
index 0000000..acdecba
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/UnpartitionedPropertyComputer.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+
+public class UnpartitionedPropertyComputer implements IPropertiesComputer {
+
+ public static final UnpartitionedPropertyComputer INSTANCE = new UnpartitionedPropertyComputer();
+
+ private UnpartitionedPropertyComputer() {
+ }
+
+ @Override
+ public IPartitioningProperty computePartitioningProperty(ILogicalExpression expr) {
+ return IPartitioningProperty.UNPARTITIONED;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
new file mode 100644
index 0000000..6f936c0
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/properties/VariablePropagationPolicy.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.properties;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public abstract class VariablePropagationPolicy {
+ public static final VariablePropagationPolicy ALL = new VariablePropagationPolicy() {
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
+ int n = sources.length;
+ for (int i = 0; i < n; i++) {
+ target.addAllNewVariables(sources[i]);
+ }
+ }
+ };
+
+ public static final VariablePropagationPolicy NONE = new VariablePropagationPolicy() {
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
+ // do nothing
+ }
+ };
+
+ /**
+ * Adds, from each source, only variables that do not already appear in the
+ * target.
+ *
+ *
+ */
+ public static final VariablePropagationPolicy ADDNEWVARIABLES = new VariablePropagationPolicy() {
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources) {
+ for (IOperatorSchema s : sources) {
+ for (LogicalVariable v : s) {
+ if (target.findVariable(v) < 0) {
+ target.addVariable(v);
+ }
+ }
+ }
+ }
+ };
+
+ public abstract void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException;
+
+ public static VariablePropagationPolicy concat(final VariablePropagationPolicy... policies) {
+ return new VariablePropagationPolicy() {
+ @Override
+ public void propagateVariables(IOperatorSchema target, IOperatorSchema... sources)
+ throws AlgebricksException {
+ if (policies.length != sources.length) {
+ throw new IllegalArgumentException();
+ }
+ for (int i = 0; i < policies.length; ++i) {
+ VariablePropagationPolicy p = policies[i];
+ IOperatorSchema s = sources[i];
+ p.propagateVariables(target, s);
+ }
+ }
+ };
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountAggregateFunctionFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountAggregateFunctionFactory.java
new file mode 100644
index 0000000..acb2e0e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountAggregateFunctionFactory.java
@@ -0,0 +1,54 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.aggregators;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class TupleCountAggregateFunctionFactory implements IAggregateFunctionFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IAggregateFunction createAggregateFunction(IDataOutputProvider provider) throws AlgebricksException {
+
+ final DataOutput out = provider.getDataOutput();
+ return new IAggregateFunction() {
+
+ int cnt;
+
+ @Override
+ public void step(IFrameTupleReference tuple) throws AlgebricksException {
+ ++cnt;
+ }
+
+ @Override
+ public void init() throws AlgebricksException {
+ cnt = 0;
+ }
+
+ @Override
+ public void finish() throws AlgebricksException {
+ try {
+ out.writeInt(cnt);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ @Override
+ public void finishPartial() throws AlgebricksException {
+ try {
+ out.writeInt(cnt);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java
new file mode 100644
index 0000000..02fe250
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/aggregators/TupleCountRunningAggregateFunctionFactory.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.aggregators;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class TupleCountRunningAggregateFunctionFactory implements IRunningAggregateFunctionFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IRunningAggregateFunction createRunningAggregateFunction(IDataOutputProvider provider)
+ throws AlgebricksException {
+
+ final DataOutput out = provider.getDataOutput();
+
+ return new IRunningAggregateFunction() {
+
+ int cnt;
+
+ @Override
+ public void step(IFrameTupleReference tuple) throws AlgebricksException {
+ ++cnt;
+ try {
+ out.writeInt(cnt);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ @Override
+ public void init() throws AlgebricksException {
+ cnt = 0;
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java
new file mode 100644
index 0000000..ecd3e96
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/AlgebricksPipeline.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class AlgebricksPipeline implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ private final IPushRuntimeFactory[] runtimeFactories;
+ private final RecordDescriptor[] recordDescriptors;
+
+ public AlgebricksPipeline(IPushRuntimeFactory[] runtimeFactories, RecordDescriptor[] recordDescriptors) {
+ this.runtimeFactories = runtimeFactories;
+ this.recordDescriptors = recordDescriptors;
+ // this.projectedColumns = projectedColumns;
+ }
+
+ public IPushRuntimeFactory[] getRuntimeFactories() {
+ return runtimeFactories;
+ }
+
+ public RecordDescriptor[] getRecordDescriptors() {
+ return recordDescriptors;
+ }
+
+ public int getOutputWidth() {
+ return recordDescriptors[recordDescriptors.length - 1].getFieldCount();
+ }
+
+ // public int[] getProjectedColumns() {
+ // return projectedColumns;
+ // }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunction.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunction.java
new file mode 100644
index 0000000..e68ae52
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunction.java
@@ -0,0 +1,15 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public interface IAggregateFunction {
+ /** should be called each time a new aggregate value is computed */
+ public void init() throws AlgebricksException;
+
+ public void step(IFrameTupleReference tuple) throws AlgebricksException;
+
+ public void finish() throws AlgebricksException;
+
+ public void finishPartial() throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunctionFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunctionFactory.java
new file mode 100644
index 0000000..e70c052
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IAggregateFunctionFactory.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+
+public interface IAggregateFunctionFactory extends Serializable {
+ public IAggregateFunction createAggregateFunction(IDataOutputProvider provider) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluator.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluator.java
new file mode 100644
index 0000000..566651d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluator.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public interface IEvaluator {
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluatorFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluatorFactory.java
new file mode 100644
index 0000000..46db040
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IEvaluatorFactory.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+
+public interface IEvaluatorFactory extends Serializable {
+ public IEvaluator createEvaluator(IDataOutputProvider output) throws AlgebricksException;
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntime.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntime.java
new file mode 100644
index 0000000..01ae150
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntime.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public interface IPushRuntime extends IFrameWriter {
+ public void setFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc);
+
+ public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java
new file mode 100644
index 0000000..4e2d113
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IPushRuntimeFactory.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IPushRuntimeFactory extends Serializable {
+ public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunction.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunction.java
new file mode 100644
index 0000000..ac381cf
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunction.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public interface IRunningAggregateFunction {
+ public void init() throws AlgebricksException;
+
+ public void step(IFrameTupleReference tuple) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunctionFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunctionFactory.java
new file mode 100644
index 0000000..638f5f8
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IRunningAggregateFunctionFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+
+public interface IRunningAggregateFunctionFactory extends Serializable {
+ public IRunningAggregateFunction createRunningAggregateFunction(IDataOutputProvider provider)
+ throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunction.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunction.java
new file mode 100644
index 0000000..4e9c5fb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunction.java
@@ -0,0 +1,44 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import java.io.DataOutput;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public interface ISerializableAggregateFunction {
+ /**
+ * initialize the space occupied by internal state
+ *
+ * @param state
+ * @throws AlgebricksException
+ * @return length of the intermediate state
+ */
+ public void init(DataOutput state) throws AlgebricksException;
+
+ /**
+ * update the internal state
+ *
+ * @param tuple
+ * @param state
+ * @throws AlgebricksException
+ */
+ public void step(IFrameTupleReference tuple, byte[] data, int start, int len) throws AlgebricksException;
+
+ /**
+ * output the state to result
+ *
+ * @param state
+ * @param result
+ * @throws AlgebricksException
+ */
+ public void finish(byte[] data, int start, int len, DataOutput result) throws AlgebricksException;
+
+ /**
+ * output the partial state to partial result
+ *
+ * @param state
+ * @param partialResult
+ * @throws AlgebricksException
+ */
+ public void finishPartial(byte[] data, int start, int len, DataOutput partialResult) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunctionFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunctionFactory.java
new file mode 100644
index 0000000..f9f871e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/ISerializableAggregateFunctionFactory.java
@@ -0,0 +1,9 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface ISerializableAggregateFunctionFactory extends Serializable {
+ public ISerializableAggregateFunction createAggregateFunction() throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java
new file mode 100644
index 0000000..5b891f2
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunction.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public interface IUnnestingFunction {
+ public void init(IFrameTupleReference tuple) throws AlgebricksException;
+
+ public boolean step() throws AlgebricksException;
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunctionFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunctionFactory.java
new file mode 100644
index 0000000..2134fd6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/base/IUnnestingFunctionFactory.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+
+public interface IUnnestingFunctionFactory extends Serializable {
+ public IUnnestingFunction createUnnestingFunction(IDataOutputProvider provider) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/AsterixBTreeRegistry.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/AsterixBTreeRegistry.java
new file mode 100644
index 0000000..4ea64d1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/AsterixBTreeRegistry.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context;
+
+import java.util.HashMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+
+public class AsterixBTreeRegistry {
+
+ private HashMap<Integer, BTree> map = new HashMap<Integer, BTree>();
+ private Lock registryLock = new ReentrantLock();
+
+ public BTree get(int fileId) {
+ return map.get(fileId);
+ }
+
+ // TODO: not very high concurrency, but good enough for now
+ public void lock() {
+ registryLock.lock();
+ }
+
+ public void unlock() {
+ registryLock.unlock();
+ }
+
+ public void register(int fileId, BTree btree) {
+ map.put(fileId, btree);
+ }
+
+ public void unregister(int fileId) {
+ map.remove(fileId);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/RuntimeContext.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/RuntimeContext.java
new file mode 100644
index 0000000..5afd17f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/context/RuntimeContext.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public class RuntimeContext {
+ private IHyracksTaskContext hyracksContext;
+
+ public RuntimeContext() {
+ }
+
+ public IHyracksTaskContext getHyracksContext() {
+ return hyracksContext;
+ }
+
+ public void setHyracksContext(IHyracksTaskContext hyracksContext) {
+ this.hyracksContext = hyracksContext;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ColumnAccessEvalFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ColumnAccessEvalFactory.java
new file mode 100644
index 0000000..4d8bf96
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ColumnAccessEvalFactory.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class ColumnAccessEvalFactory implements IEvaluatorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int fieldIndex;
+
+ public ColumnAccessEvalFactory(int fieldIndex) {
+ this.fieldIndex = fieldIndex;
+ }
+
+ @Override
+ public String toString() {
+ return "ColumnAccess(" + fieldIndex + ")";
+ }
+
+ @Override
+ public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new IEvaluator() {
+
+ private DataOutput out = output.getDataOutput();
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ byte[] buffer = tuple.getFieldData(fieldIndex);
+ int start = tuple.getFieldStart(fieldIndex);
+ int length = tuple.getFieldLength(fieldIndex);
+ try {
+ out.write(buffer, start, length);
+ } catch (IOException ioe) {
+ throw new AlgebricksException(ioe);
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ConstantEvalFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ConstantEvalFactory.java
new file mode 100644
index 0000000..f49f2ee
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/evaluators/ConstantEvalFactory.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class ConstantEvalFactory implements IEvaluatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private byte[] value;
+
+ public ConstantEvalFactory(byte[] value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ return "Constant";
+ }
+
+ @Override
+ public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new IEvaluator() {
+
+ private DataOutput out = output.getDataOutput();
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ try {
+ out.write(value, 0, value.length);
+ } catch (IOException ioe) {
+ throw new AlgebricksException(ioe);
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryBooleanInspectorImpl.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryBooleanInspectorImpl.java
new file mode 100644
index 0000000..fdb3663
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryBooleanInspectorImpl.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
+
+public class BinaryBooleanInspectorImpl implements IBinaryBooleanInspector {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final BinaryBooleanInspectorImpl INSTANCE = new BinaryBooleanInspectorImpl();
+
+ private BinaryBooleanInspectorImpl() {
+ }
+
+ @Override
+ public boolean getBooleanValue(byte[] bytes, int offset, int length) {
+ return bytes[offset] == 1;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryIntegerInspectorImpl.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryIntegerInspectorImpl.java
new file mode 100644
index 0000000..9d0587d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/BinaryIntegerInspectorImpl.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+public class BinaryIntegerInspectorImpl implements IBinaryIntegerInspector {
+
+ private static final long serialVersionUID = 1L;
+ public static final BinaryIntegerInspectorImpl INSTANCE = new BinaryIntegerInspectorImpl();
+
+ @Override
+ public int getIntegerValue(byte[] bytes, int offset, int length) {
+ return IntegerSerializerDeserializer.getInt(bytes, offset);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/IntegerPrinterFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/IntegerPrinterFactory.java
new file mode 100644
index 0000000..db47af6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/IntegerPrinterFactory.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.WriteValueTools;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+public class IntegerPrinterFactory implements IPrinterFactory {
+
+ private static final long serialVersionUID = 1L;
+ public static final IntegerPrinterFactory INSTANCE = new IntegerPrinterFactory();
+
+ private IntegerPrinterFactory() {
+ }
+
+ @Override
+ public IPrinter createPrinter() {
+ return new IPrinter() {
+
+ @Override
+ public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+ int d = IntegerSerializerDeserializer.getInt(b, s);
+ try {
+ WriteValueTools.writeInt(d, ps);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ @Override
+ public void init() throws AlgebricksException {
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/NoopNullWriterFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/NoopNullWriterFactory.java
new file mode 100644
index 0000000..e064832
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/NoopNullWriterFactory.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
+
+import java.io.DataOutput;
+
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class NoopNullWriterFactory implements INullWriterFactory {
+
+ private static final long serialVersionUID = 1L;
+ public static final NoopNullWriterFactory INSTANCE = new NoopNullWriterFactory();
+
+ private NoopNullWriterFactory() {
+ }
+
+ @Override
+ public INullWriter createNullWriter() {
+
+ return new INullWriter() {
+
+ @Override
+ public void writeNull(DataOutput out) throws HyracksDataException {
+ // do nothing
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/UTF8StringPrinterFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/UTF8StringPrinterFactory.java
new file mode 100644
index 0000000..4a900f1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/data/UTF8StringPrinterFactory.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data;
+
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+
+public class UTF8StringPrinterFactory implements IPrinterFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final UTF8StringPrinterFactory INSTANCE = new UTF8StringPrinterFactory();
+
+ private UTF8StringPrinterFactory() {
+ }
+
+ @Override
+ public IPrinter createPrinter() {
+ return new IPrinter() {
+
+ @Override
+ public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+ int strlen = UTF8StringPointable.getUTFLen(b, s);
+ int pos = s + 2;
+ int maxPos = pos + strlen;
+ ps.print("\"");
+ while (pos < maxPos) {
+ char c = UTF8StringPointable.charAt(b, pos);
+ switch (c) {
+ case '\\':
+ case '"':
+ ps.print('\\');
+ break;
+ }
+ ps.print(c);
+ pos += UTF8StringPointable.charSize(b, pos);
+ }
+ ps.print("\"");
+ }
+
+ @Override
+ public void init() throws AlgebricksException {
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java
new file mode 100644
index 0000000..33102cb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobBuilder.java
@@ -0,0 +1,323 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksCountPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraintHelper;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class JobBuilder implements IHyracksJobBuilder {
+
+ private JobSpecification jobSpec;
+ private AlgebricksPartitionConstraint clusterLocations;
+
+ private Map<ILogicalOperator, ArrayList<ILogicalOperator>> outEdges = new HashMap<ILogicalOperator, ArrayList<ILogicalOperator>>();
+ private Map<ILogicalOperator, ArrayList<ILogicalOperator>> inEdges = new HashMap<ILogicalOperator, ArrayList<ILogicalOperator>>();
+ private Map<ILogicalOperator, Pair<IConnectorDescriptor, TargetConstraint>> connectors = new HashMap<ILogicalOperator, Pair<IConnectorDescriptor, TargetConstraint>>();
+
+ private Map<ILogicalOperator, Pair<IPushRuntimeFactory, RecordDescriptor>> microOps = new HashMap<ILogicalOperator, Pair<IPushRuntimeFactory, RecordDescriptor>>();
+ private Map<IPushRuntimeFactory, ILogicalOperator> revMicroOpMap = new HashMap<IPushRuntimeFactory, ILogicalOperator>();
+ private Map<ILogicalOperator, IOperatorDescriptor> hyracksOps = new HashMap<ILogicalOperator, IOperatorDescriptor>();
+ private Map<ILogicalOperator, AlgebricksPartitionConstraint> pcForMicroOps = new HashMap<ILogicalOperator, AlgebricksPartitionConstraint>();
+
+ private int aodCounter = 0;
+ private Map<ILogicalOperator, Integer> algebraicOpBelongingToMetaAsterixOp = new HashMap<ILogicalOperator, Integer>();
+ private Map<Integer, List<Pair<IPushRuntimeFactory, RecordDescriptor>>> metaAsterixOpSkeletons = new HashMap<Integer, List<Pair<IPushRuntimeFactory, RecordDescriptor>>>();
+ private Map<Integer, AlgebricksMetaOperatorDescriptor> metaAsterixOps = new HashMap<Integer, AlgebricksMetaOperatorDescriptor>();
+ private final Map<IOperatorDescriptor, AlgebricksPartitionConstraint> partitionConstraintMap = new HashMap<IOperatorDescriptor, AlgebricksPartitionConstraint>();
+
+ public JobBuilder(JobSpecification jobSpec, AlgebricksPartitionConstraint clusterLocations) {
+ this.jobSpec = jobSpec;
+ this.clusterLocations = clusterLocations;
+ }
+
+ @Override
+ public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc) {
+ contributeMicroOperator(op, runtime, recDesc, null);
+ }
+
+ @Override
+ public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc,
+ AlgebricksPartitionConstraint pc) {
+ microOps.put(op, new Pair<IPushRuntimeFactory, RecordDescriptor>(runtime, recDesc));
+ revMicroOpMap.put(runtime, op);
+ if (pc != null) {
+ pcForMicroOps.put(op, pc);
+ }
+ }
+
+ @Override
+ public void contributeConnector(ILogicalOperator exchgOp, IConnectorDescriptor conn) {
+ connectors.put(exchgOp, new Pair<IConnectorDescriptor, TargetConstraint>(conn, null));
+ }
+
+ @Override
+ public void contributeConnectorWithTargetConstraint(ILogicalOperator exchgOp, IConnectorDescriptor conn,
+ TargetConstraint numberOfTargetPartitions) {
+ connectors.put(exchgOp, new Pair<IConnectorDescriptor, TargetConstraint>(conn, numberOfTargetPartitions));
+ }
+
+ @Override
+ public void contributeGraphEdge(ILogicalOperator src, int srcOutputIndex, ILogicalOperator dest, int destInputIndex) {
+ ArrayList<ILogicalOperator> outputs = outEdges.get(src);
+ if (outputs == null) {
+ outputs = new ArrayList<ILogicalOperator>();
+ outEdges.put(src, outputs);
+ }
+ addAtPos(outputs, dest, srcOutputIndex);
+
+ ArrayList<ILogicalOperator> inp = inEdges.get(dest);
+ if (inp == null) {
+ inp = new ArrayList<ILogicalOperator>();
+ inEdges.put(dest, inp);
+ }
+ addAtPos(inp, src, destInputIndex);
+ }
+
+ @Override
+ public void contributeHyracksOperator(ILogicalOperator op, IOperatorDescriptor opDesc) {
+ hyracksOps.put(op, opDesc);
+ }
+
+ @Override
+ public void contributeAlgebricksPartitionConstraint(IOperatorDescriptor opDesc, AlgebricksPartitionConstraint apc) {
+ partitionConstraintMap.put(opDesc, apc);
+ }
+
+ @Override
+ public JobSpecification getJobSpec() {
+ return jobSpec;
+ }
+
+ @Override
+ public void buildSpec(List<ILogicalOperator> roots) throws AlgebricksException {
+ buildAsterixComponents();
+ Map<IConnectorDescriptor, TargetConstraint> tgtConstraints = setupConnectors();
+ for (ILogicalOperator r : roots) {
+ IOperatorDescriptor opDesc = findOpDescForAlgebraicOp(r);
+ jobSpec.addRoot(opDesc);
+ }
+ setAllPartitionConstraints(tgtConstraints);
+ }
+
+ private void setAllPartitionConstraints(Map<IConnectorDescriptor, TargetConstraint> tgtConstraints) {
+ List<OperatorDescriptorId> roots = jobSpec.getRoots();
+ setSpecifiedPartitionConstraints();
+ for (OperatorDescriptorId rootId : roots) {
+ setPartitionConstraintsDFS(rootId, tgtConstraints, null);
+ }
+ }
+
+ private void setSpecifiedPartitionConstraints() {
+ for (ILogicalOperator op : pcForMicroOps.keySet()) {
+ AlgebricksPartitionConstraint pc = pcForMicroOps.get(op);
+ Integer k = algebraicOpBelongingToMetaAsterixOp.get(op);
+ AlgebricksMetaOperatorDescriptor amod = metaAsterixOps.get(k);
+ partitionConstraintMap.put(amod, pc);
+ }
+ for (IOperatorDescriptor opDesc : partitionConstraintMap.keySet()) {
+ AlgebricksPartitionConstraint pc = partitionConstraintMap.get(opDesc);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, pc);
+ }
+ }
+
+ private void setPartitionConstraintsDFS(OperatorDescriptorId opId,
+ Map<IConnectorDescriptor, TargetConstraint> tgtConstraints, IOperatorDescriptor parentOp) {
+ List<IConnectorDescriptor> opInputs = jobSpec.getOperatorInputMap().get(opId);
+ AlgebricksPartitionConstraint opConstraint = null;
+ IOperatorDescriptor opDesc = jobSpec.getOperatorMap().get(opId);
+ if (opInputs != null) {
+ for (IConnectorDescriptor conn : opInputs) {
+ ConnectorDescriptorId cid = conn.getConnectorId();
+ org.apache.commons.lang3.tuple.Pair<org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>, org.apache.commons.lang3.tuple.Pair<IOperatorDescriptor, Integer>> p = jobSpec
+ .getConnectorOperatorMap().get(cid);
+ IOperatorDescriptor src = p.getLeft().getLeft();
+ // DFS
+ setPartitionConstraintsDFS(src.getOperatorId(), tgtConstraints, opDesc);
+
+ TargetConstraint constraint = tgtConstraints.get(conn);
+ if (constraint != null) {
+ switch (constraint) {
+ case ONE: {
+ opConstraint = new AlgebricksCountPartitionConstraint(1);
+ break;
+ }
+ case SAME_COUNT: {
+ opConstraint = partitionConstraintMap.get(src);
+ break;
+ }
+ }
+ }
+ }
+ }
+ if (partitionConstraintMap.get(opDesc) == null) {
+ if (opConstraint == null) {
+ if (parentOp != null) {
+ AlgebricksPartitionConstraint pc = partitionConstraintMap.get(parentOp);
+ if (pc != null) {
+ opConstraint = pc;
+ } else if (opInputs == null || opInputs.size() == 0) {
+ opConstraint = new AlgebricksCountPartitionConstraint(1);
+ }
+ }
+ if (opConstraint == null) {
+ opConstraint = clusterLocations;
+ }
+ }
+ partitionConstraintMap.put(opDesc, opConstraint);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, opDesc, opConstraint);
+ }
+ }
+
+ private Map<IConnectorDescriptor, TargetConstraint> setupConnectors() throws AlgebricksException {
+ Map<IConnectorDescriptor, TargetConstraint> tgtConstraints = new HashMap<IConnectorDescriptor, TargetConstraint>();
+ for (ILogicalOperator exchg : connectors.keySet()) {
+ ILogicalOperator inOp = inEdges.get(exchg).get(0);
+ ILogicalOperator outOp = outEdges.get(exchg).get(0);
+ IOperatorDescriptor inOpDesc = findOpDescForAlgebraicOp(inOp);
+ IOperatorDescriptor outOpDesc = findOpDescForAlgebraicOp(outOp);
+ Pair<IConnectorDescriptor, TargetConstraint> connPair = connectors.get(exchg);
+ IConnectorDescriptor conn = connPair.first;
+ int producerPort = outEdges.get(inOp).indexOf(exchg);
+ int consumerPort = inEdges.get(outOp).indexOf(exchg);
+ jobSpec.connect(conn, inOpDesc, producerPort, outOpDesc, consumerPort);
+ if (connPair.second != null) {
+ tgtConstraints.put(conn, connPair.second);
+ }
+ }
+ return tgtConstraints;
+ }
+
+ private IOperatorDescriptor findOpDescForAlgebraicOp(ILogicalOperator op) throws AlgebricksException {
+ IOperatorDescriptor hOpDesc = hyracksOps.get(op);
+ if (hOpDesc != null) {
+ return hOpDesc;
+ }
+ Integer metaOpKey = algebraicOpBelongingToMetaAsterixOp.get(op);
+ if (metaOpKey == null) {
+ throw new AlgebricksException("Could not generate operator descriptor for operator " + op);
+ }
+ return metaAsterixOps.get(metaOpKey);
+ }
+
+ private void buildAsterixComponents() {
+ for (ILogicalOperator aop : microOps.keySet()) {
+ addMicroOpToMetaRuntimeOp(aop);
+ }
+ for (Integer k : metaAsterixOpSkeletons.keySet()) {
+ List<Pair<IPushRuntimeFactory, RecordDescriptor>> opContents = metaAsterixOpSkeletons.get(k);
+ AlgebricksMetaOperatorDescriptor amod = buildMetaAsterixOpDesc(opContents);
+ metaAsterixOps.put(k, amod);
+ }
+ }
+
+ private AlgebricksMetaOperatorDescriptor buildMetaAsterixOpDesc(
+ List<Pair<IPushRuntimeFactory, RecordDescriptor>> opContents) {
+ // RecordDescriptor outputRecordDesc = null;
+ int n = opContents.size();
+ IPushRuntimeFactory[] runtimeFactories = new IPushRuntimeFactory[n];
+ RecordDescriptor[] internalRecordDescriptors = new RecordDescriptor[n];
+ int i = 0;
+ for (Pair<IPushRuntimeFactory, RecordDescriptor> p : opContents) {
+ runtimeFactories[i] = p.first;
+ internalRecordDescriptors[i] = p.second;
+ // if (i == n - 1) {
+ // outputRecordDesc = p.second;
+ // }
+ i++;
+ }
+ ILogicalOperator lastLogicalOp = revMicroOpMap.get(runtimeFactories[n - 1]);
+ ArrayList<ILogicalOperator> outOps = outEdges.get(lastLogicalOp);
+ int outArity = (outOps == null) ? 0 : outOps.size();
+ ILogicalOperator firstLogicalOp = revMicroOpMap.get(runtimeFactories[0]);
+ ArrayList<ILogicalOperator> inOps = inEdges.get(firstLogicalOp);
+ int inArity = (inOps == null) ? 0 : inOps.size();
+ // boolean isLeafOp = inEdges.get(firstLogicalOp) == null;
+ return new AlgebricksMetaOperatorDescriptor(jobSpec, inArity, outArity, runtimeFactories,
+ internalRecordDescriptors);
+ }
+
+ private void addMicroOpToMetaRuntimeOp(ILogicalOperator aop) {
+ Integer k = algebraicOpBelongingToMetaAsterixOp.get(aop);
+ if (k == null) {
+ k = createNewMetaOpInfo(aop);
+ }
+ ArrayList<ILogicalOperator> destList = outEdges.get(aop);
+ if (destList == null || destList.size() != 1) {
+ // for now, we only support linear plans inside meta-ops.
+ return;
+ }
+ ILogicalOperator dest = destList.get(0);
+ Integer j = algebraicOpBelongingToMetaAsterixOp.get(dest);
+ if (j == null && microOps.get(dest) != null) {
+ algebraicOpBelongingToMetaAsterixOp.put(dest, k);
+ List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent1 = metaAsterixOpSkeletons.get(k);
+ aodContent1.add(microOps.get(dest));
+ } else if (j != null && j.intValue() != k.intValue()) {
+ // merge the j component into the k component
+ List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent1 = metaAsterixOpSkeletons.get(k);
+ List<Pair<IPushRuntimeFactory, RecordDescriptor>> aodContent2 = metaAsterixOpSkeletons.get(j);
+ aodContent1.addAll(aodContent2);
+ metaAsterixOpSkeletons.remove(j);
+ for (ILogicalOperator m : algebraicOpBelongingToMetaAsterixOp.keySet()) {
+ Integer g = algebraicOpBelongingToMetaAsterixOp.get(m);
+ if (g.intValue() == j.intValue()) {
+ algebraicOpBelongingToMetaAsterixOp.put(m, k);
+ }
+ }
+ }
+
+ }
+
+ private int createNewMetaOpInfo(ILogicalOperator aop) {
+ int n = aodCounter;
+ aodCounter++;
+ List<Pair<IPushRuntimeFactory, RecordDescriptor>> metaOpContents = new ArrayList<Pair<IPushRuntimeFactory, RecordDescriptor>>();
+ metaOpContents.add(microOps.get(aop));
+ metaAsterixOpSkeletons.put(n, metaOpContents);
+ algebraicOpBelongingToMetaAsterixOp.put(aop, n);
+ return n;
+ }
+
+ private <E> void addAtPos(ArrayList<E> a, E elem, int pos) {
+ int n = a.size();
+ if (n > pos) {
+ a.set(pos, elem);
+ } else {
+ for (int k = n; k < pos; k++) {
+ a.add(null);
+ }
+ a.add(elem);
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenContext.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenContext.java
new file mode 100644
index 0000000..870ce91
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenContext.java
@@ -0,0 +1,191 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.ITypeTraitProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IPartialAggregationTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+
+public class JobGenContext {
+ private final IOperatorSchema outerFlowSchema;
+ private final Map<ILogicalOperator, IOperatorSchema> schemaMap = new HashMap<ILogicalOperator, IOperatorSchema>();
+ private final ISerializerDeserializerProvider serializerDeserializerProvider;
+ private final IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider;
+ private final IBinaryComparatorFactoryProvider comparatorFactoryProvider;
+ private final IPrinterFactoryProvider printerFactoryProvider;
+ private final ITypeTraitProvider typeTraitProvider;
+ private final IMetadataProvider<?, ?> metadataProvider;
+ private final INullWriterFactory nullWriterFactory;
+ private final INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider;
+ private final Object appContext;
+ private final IBinaryBooleanInspector booleanInspector;
+ private final IBinaryIntegerInspector integerInspector;
+ private final ILogicalExpressionJobGen exprJobGen;
+ private final IExpressionTypeComputer expressionTypeComputer;
+ private final IExpressionEvalSizeComputer expressionEvalSizeComputer;
+ private final IPartialAggregationTypeComputer partialAggregationTypeComputer;
+ private final int frameSize;
+ private AlgebricksPartitionConstraint clusterLocations;
+ private int varCounter;
+ private final ITypingContext typingContext;
+
+ public JobGenContext(IOperatorSchema outerFlowSchema, IMetadataProvider<?, ?> metadataProvider, Object appContext,
+ ISerializerDeserializerProvider serializerDeserializerProvider,
+ IBinaryHashFunctionFactoryProvider hashFunctionFactoryProvider,
+ IBinaryComparatorFactoryProvider comparatorFactoryProvider, ITypeTraitProvider typeTraitProvider,
+ IBinaryBooleanInspector booleanInspector, IBinaryIntegerInspector integerInspector,
+ IPrinterFactoryProvider printerFactoryProvider, INullWriterFactory nullWriterFactory,
+ INormalizedKeyComputerFactoryProvider normalizedKeyComputerFactoryProvider,
+ ILogicalExpressionJobGen exprJobGen, IExpressionTypeComputer expressionTypeComputer,
+ INullableTypeComputer nullableTypeComputer, ITypingContext typingContext,
+ IExpressionEvalSizeComputer expressionEvalSizeComputer,
+ IPartialAggregationTypeComputer partialAggregationTypeComputer, int frameSize,
+ AlgebricksPartitionConstraint clusterLocations) {
+ this.outerFlowSchema = outerFlowSchema;
+ this.metadataProvider = metadataProvider;
+ this.appContext = appContext;
+ this.serializerDeserializerProvider = serializerDeserializerProvider;
+ this.hashFunctionFactoryProvider = hashFunctionFactoryProvider;
+ this.comparatorFactoryProvider = comparatorFactoryProvider;
+ this.typeTraitProvider = typeTraitProvider;
+ this.booleanInspector = booleanInspector;
+ this.integerInspector = integerInspector;
+ this.printerFactoryProvider = printerFactoryProvider;
+ this.clusterLocations = clusterLocations;
+ this.normalizedKeyComputerFactoryProvider = normalizedKeyComputerFactoryProvider;
+ this.nullWriterFactory = nullWriterFactory;
+ this.exprJobGen = exprJobGen;
+ this.expressionTypeComputer = expressionTypeComputer;
+ this.typingContext = typingContext;
+ this.expressionEvalSizeComputer = expressionEvalSizeComputer;
+ this.partialAggregationTypeComputer = partialAggregationTypeComputer;
+ this.frameSize = frameSize;
+ this.varCounter = 0;
+ }
+
+ public IOperatorSchema getOuterFlowSchema() {
+ return outerFlowSchema;
+ }
+
+ public AlgebricksPartitionConstraint getClusterLocations() {
+ return clusterLocations;
+ }
+
+ public IMetadataProvider<?, ?> getMetadataProvider() {
+ return metadataProvider;
+ }
+
+ public Object getAppContext() {
+ return appContext;
+ }
+
+ public ISerializerDeserializerProvider getSerializerDeserializerProvider() {
+ return serializerDeserializerProvider;
+ }
+
+ public IBinaryHashFunctionFactoryProvider getBinaryHashFunctionFactoryProvider() {
+ return hashFunctionFactoryProvider;
+ }
+
+ public IBinaryComparatorFactoryProvider getBinaryComparatorFactoryProvider() {
+ return comparatorFactoryProvider;
+ }
+
+ public ITypeTraitProvider getTypeTraitProvider() {
+ return typeTraitProvider;
+ }
+
+ public IBinaryBooleanInspector getBinaryBooleanInspector() {
+ return booleanInspector;
+ }
+
+ public IBinaryIntegerInspector getBinaryIntegerInspector() {
+ return integerInspector;
+ }
+
+ public IPrinterFactoryProvider getPrinterFactoryProvider() {
+ return printerFactoryProvider;
+ }
+
+ public ILogicalExpressionJobGen getExpressionJobGen() {
+ return exprJobGen;
+ }
+
+ public IOperatorSchema getSchema(ILogicalOperator op) {
+ return schemaMap.get(op);
+ }
+
+ public void putSchema(ILogicalOperator op, IOperatorSchema schema) {
+ schemaMap.put(op, schema);
+ }
+
+ public LogicalVariable createNewVar() {
+ varCounter++;
+ LogicalVariable var = new LogicalVariable(-varCounter);
+ return var;
+ }
+
+ public Object getType(ILogicalExpression expr, IVariableTypeEnvironment env) throws AlgebricksException {
+ return expressionTypeComputer.getType(expr, typingContext.getMetadataProvider(), env);
+ }
+
+ public INullWriterFactory getNullWriterFactory() {
+ return nullWriterFactory;
+ }
+
+ public INormalizedKeyComputerFactoryProvider getNormalizedKeyComputerFactoryProvider() {
+ return normalizedKeyComputerFactoryProvider;
+ }
+
+ public IExpressionEvalSizeComputer getExpressionEvalSizeComputer() {
+ return expressionEvalSizeComputer;
+ }
+
+ public int getFrameSize() {
+ return frameSize;
+ }
+
+ public IPartialAggregationTypeComputer getPartialAggregationTypeComputer() {
+ return partialAggregationTypeComputer;
+ }
+
+ public IVariableTypeEnvironment getTypeEnvironment(ILogicalOperator op) {
+ return typingContext.getOutputTypeEnvironment(op);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenHelper.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenHelper.java
new file mode 100644
index 0000000..90bf47c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/JobGenHelper.java
@@ -0,0 +1,145 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+
+import java.util.Collection;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryComparatorFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryHashFunctionFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.INormalizedKeyComputerFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.ITypeTraitProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public final class JobGenHelper {
+
+ private static final Logger LOGGER = Logger.getLogger(JobGenHelper.class.getName());
+
+ @SuppressWarnings("unchecked")
+ public static RecordDescriptor mkRecordDescriptor(ILogicalOperator op, IOperatorSchema opSchema,
+ JobGenContext context) throws AlgebricksException {
+ ISerializerDeserializer[] fields = new ISerializerDeserializer[opSchema.getSize()];
+ ISerializerDeserializerProvider sdp = context.getSerializerDeserializerProvider();
+ int i = 0;
+ IVariableTypeEnvironment env = context.getTypeEnvironment(op);
+ for (LogicalVariable var : opSchema) {
+ Object t = env.getVarType(var);
+ if (t == null) {
+ LOGGER.warning("No type for variable " + var);
+ // throw new AlgebricksException("No type for variable " + var);
+ }
+ fields[i] = sdp.getSerializerDeserializer(t);
+ i++;
+ }
+ return new RecordDescriptor(fields);
+ }
+
+ public static IPrinterFactory[] mkPrinterFactories(IOperatorSchema opSchema, IVariableTypeEnvironment env,
+ JobGenContext context, int[] printColumns) throws AlgebricksException {
+ IPrinterFactory[] pf = new IPrinterFactory[printColumns.length];
+ IPrinterFactoryProvider pff = context.getPrinterFactoryProvider();
+ for (int i = 0; i < pf.length; i++) {
+ LogicalVariable v = opSchema.getVariable(printColumns[i]);
+ Object t = env.getVarType(v);
+ pf[i] = pff.getPrinterFactory(t);
+ }
+ return pf;
+ }
+
+ public static int[] variablesToFieldIndexes(Collection<LogicalVariable> varLogical, IOperatorSchema opSchema) {
+ int[] tuplePos = new int[varLogical.size()];
+ int i = 0;
+ for (LogicalVariable var : varLogical) {
+ tuplePos[i] = opSchema.findVariable(var);
+ i++;
+ }
+ return tuplePos;
+ }
+
+ public static IBinaryHashFunctionFactory[] variablesToBinaryHashFunctionFactories(
+ Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
+ throws AlgebricksException {
+ IBinaryHashFunctionFactory[] funFactories = new IBinaryHashFunctionFactory[varLogical.size()];
+ int i = 0;
+ IBinaryHashFunctionFactoryProvider bhffProvider = context.getBinaryHashFunctionFactoryProvider();
+ for (LogicalVariable var : varLogical) {
+ Object type = env.getVarType(var);
+ funFactories[i++] = bhffProvider.getBinaryHashFunctionFactory(type);
+ }
+ return funFactories;
+ }
+
+ public static IBinaryComparatorFactory[] variablesToAscBinaryComparatorFactories(
+ Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
+ throws AlgebricksException {
+ IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[varLogical.size()];
+ IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider();
+ int i = 0;
+ for (LogicalVariable v : varLogical) {
+ Object type = env.getVarType(v);
+ compFactories[i++] = bcfProvider.getBinaryComparatorFactory(type, OrderKind.ASC);
+ }
+ return compFactories;
+ }
+
+ public static INormalizedKeyComputerFactory variablesToAscNormalizedKeyComputerFactory(
+ Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context)
+ throws AlgebricksException {
+ INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider();
+ if (nkcfProvider == null)
+ return null;
+ for (LogicalVariable v : varLogical) {
+ Object type = env.getVarType(v);
+ return nkcfProvider.getNormalizedKeyComputerFactory(type, OrderKind.ASC);
+ }
+ return null;
+ }
+
+ public static ITypeTraits[] variablesToTypeTraits(Collection<LogicalVariable> varLogical,
+ IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException {
+ ITypeTraits[] typeTraits = new ITypeTraits[varLogical.size()];
+ ITypeTraitProvider typeTraitProvider = context.getTypeTraitProvider();
+ int i = 0;
+ for (LogicalVariable v : varLogical) {
+ Object type = env.getVarType(v);
+ typeTraits[i++] = typeTraitProvider.getTypeTrait(type);
+ }
+ return typeTraits;
+ }
+
+ public static int[] projectAllVariables(IOperatorSchema opSchema) {
+ int[] projectionList = new int[opSchema.getSize()];
+ int k = 0;
+ for (LogicalVariable v : opSchema) {
+ projectionList[k++] = opSchema.findVariable(v);
+ }
+ return projectionList;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/OperatorSchemaImpl.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/OperatorSchemaImpl.java
new file mode 100644
index 0000000..1d874f3
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/OperatorSchemaImpl.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+
+public class OperatorSchemaImpl implements IOperatorSchema {
+ private final Map<LogicalVariable, Integer> varMap;
+
+ private final List<LogicalVariable> varList;
+
+ public OperatorSchemaImpl() {
+ varMap = new HashMap<LogicalVariable, Integer>();
+ varList = new ArrayList<LogicalVariable>();
+ }
+
+ @Override
+ public void addAllVariables(IOperatorSchema source) {
+ for (LogicalVariable v : source) {
+ varMap.put(v, varList.size());
+ varList.add(v);
+ }
+ }
+
+ @Override
+ public void addAllNewVariables(IOperatorSchema source) {
+ for (LogicalVariable v : source) {
+ if (varMap.get(v) == null) {
+ varMap.put(v, varList.size());
+ varList.add(v);
+ }
+ }
+ }
+
+ @Override
+ public int addVariable(LogicalVariable var) {
+ int idx = varList.size();
+ varMap.put(var, idx);
+ varList.add(var);
+ return idx;
+ }
+
+ @Override
+ public void clear() {
+ varMap.clear();
+ varList.clear();
+ }
+
+ @Override
+ public int findVariable(LogicalVariable var) {
+ Integer i = varMap.get(var);
+ if (i == null) {
+ return -1;
+ }
+ return i;
+ }
+
+ @Override
+ public int getSize() {
+ return varList.size();
+ }
+
+ @Override
+ public LogicalVariable getVariable(int index) {
+ return varList.get(index);
+ }
+
+ @Override
+ public Iterator<LogicalVariable> iterator() {
+ return varList.iterator();
+ }
+
+ @Override
+ public String toString() {
+ return varMap.toString();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java
new file mode 100644
index 0000000..ab170c9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/jobgen/impl/PlanCompiler.java
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IHyracksJobBuilder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class PlanCompiler {
+ private JobGenContext context;
+ private Map<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> operatorVisitedToParents = new HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>>();
+
+ public PlanCompiler(JobGenContext context) {
+ this.context = context;
+ }
+
+ public JobGenContext getContext() {
+ return context;
+ }
+
+ public JobSpecification compilePlan(ILogicalPlan plan, IOperatorSchema outerPlanSchema) throws AlgebricksException {
+ JobSpecification spec = new JobSpecification();
+ List<ILogicalOperator> rootOps = new ArrayList<ILogicalOperator>();
+ IHyracksJobBuilder builder = new JobBuilder(spec, context.getClusterLocations());
+ for (Mutable<ILogicalOperator> opRef : plan.getRoots()) {
+ compileOpRef(opRef, spec, builder, outerPlanSchema);
+ rootOps.add(opRef.getValue());
+ }
+ reviseEdges(builder);
+ operatorVisitedToParents.clear();
+ builder.buildSpec(rootOps);
+ return spec;
+ }
+
+ private void compileOpRef(Mutable<ILogicalOperator> opRef, JobSpecification spec, IHyracksJobBuilder builder,
+ IOperatorSchema outerPlanSchema) throws AlgebricksException {
+ ILogicalOperator op = opRef.getValue();
+ int n = op.getInputs().size();
+ IOperatorSchema[] schemas = new IOperatorSchema[n];
+ int i = 0;
+ for (Mutable<ILogicalOperator> opRef2 : op.getInputs()) {
+ List<Mutable<ILogicalOperator>> parents = operatorVisitedToParents.get(opRef2);
+ if (parents == null) {
+ parents = new ArrayList<Mutable<ILogicalOperator>>();
+ operatorVisitedToParents.put(opRef2, parents);
+ parents.add(opRef);
+ compileOpRef(opRef2, spec, builder, outerPlanSchema);
+ schemas[i++] = context.getSchema(opRef2.getValue());
+ } else {
+ if (!parents.contains(opRef))
+ parents.add(opRef);
+ schemas[i++] = context.getSchema(opRef2.getValue());
+ continue;
+ }
+ }
+
+ IOperatorSchema opSchema = new OperatorSchemaImpl();
+ context.putSchema(op, opSchema);
+ op.getVariablePropagationPolicy().propagateVariables(opSchema, schemas);
+ op.contributeRuntimeOperator(builder, context, opSchema, schemas, outerPlanSchema);
+ }
+
+ private void reviseEdges(IHyracksJobBuilder builder) {
+ /**
+ * revise the edges for the case of replicate operator
+ */
+ for (Entry<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> entry : operatorVisitedToParents
+ .entrySet()) {
+ Mutable<ILogicalOperator> child = entry.getKey();
+ List<Mutable<ILogicalOperator>> parents = entry.getValue();
+ if (parents.size() > 1) {
+ int i = 0;
+ for (Mutable<ILogicalOperator> parent : parents) {
+ builder.contributeGraphEdge(child.getValue(), i, parent.getValue(), 0);
+ i++;
+ }
+ }
+ }
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/AggregateRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/AggregateRuntimeFactory.java
new file mode 100644
index 0000000..5184e12
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/AggregateRuntimeFactory.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+
+public class AggregateRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ // private int[] outColumns;
+ private IAggregateFunctionFactory[] aggregFactories;
+
+ public AggregateRuntimeFactory(IAggregateFunctionFactory[] aggregFactories) {
+ super(null);
+ // this.outColumns = outColumns;
+ this.aggregFactories = aggregFactories;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("assign [");
+ for (int i = 0; i < aggregFactories.length; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(aggregFactories[i]);
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ throws AlgebricksException {
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ private IAggregateFunction[] aggregs = new IAggregateFunction[aggregFactories.length];
+ private ArrayBackedValueStorage evalOutput = new ArrayBackedValueStorage();
+ private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(aggregs.length);
+
+ private boolean first = true;
+
+ @Override
+ public void open() throws HyracksDataException {
+ try {
+ if (first) {
+ first = false;
+ initAccessAppendRef(context);
+ for (int i = 0; i < aggregFactories.length; i++) {
+ aggregs[i] = aggregFactories[i].createAggregateFunction(evalOutput);
+ }
+ }
+ for (int i = 0; i < aggregFactories.length; i++) {
+ aggregs[i].init();
+ }
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ tRef.reset(tAccess, t);
+ processTuple(tRef);
+ }
+
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ computeAggregate();
+ appendToFrameFromTupleBuilder(tupleBuilder);
+ super.close();
+ }
+
+ private void computeAggregate() throws HyracksDataException {
+ tupleBuilder.reset();
+ for (int f = 0; f < aggregs.length; f++) {
+ evalOutput.reset();
+ try {
+ aggregs[f].finish();
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ tupleBuilder.addField(evalOutput.getBytes(), evalOutput.getStartIndex(), evalOutput.getLength());
+ }
+ }
+
+ private void processTuple(FrameTupleReference tupleRef) throws HyracksDataException {
+ for (int f = 0; f < aggregs.length; f++) {
+ try {
+ aggregs[f].step(tupleRef);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
new file mode 100644
index 0000000..b7b64ab
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/NestedPlansAccumulatingAggregatorFactory.java
@@ -0,0 +1,239 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.NestedTupleSourceRuntimeFactory.NestedTupleSourceRuntime;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+public class NestedPlansAccumulatingAggregatorFactory implements IAggregatorDescriptorFactory {
+
+ private static final long serialVersionUID = 1L;
+ private AlgebricksPipeline[] subplans;
+ private int[] keyFieldIdx;
+ private int[] decorFieldIdx;
+
+ public NestedPlansAccumulatingAggregatorFactory(AlgebricksPipeline[] subplans, int[] keyFieldIdx,
+ int[] decorFieldIdx) {
+ this.subplans = subplans;
+ this.keyFieldIdx = keyFieldIdx;
+ this.decorFieldIdx = decorFieldIdx;
+ }
+
+ @Override
+ public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDesc,
+ RecordDescriptor outRecordDescriptor, int[] keys, int[] partialKeys) throws HyracksDataException {
+
+ final RuntimeContext rc = new RuntimeContext();
+ rc.setHyracksContext(ctx);
+ final AggregatorOutput outputWriter = new AggregatorOutput(ctx.getFrameSize(), subplans, keyFieldIdx.length,
+ decorFieldIdx.length);
+ final NestedTupleSourceRuntime[] pipelines = new NestedTupleSourceRuntime[subplans.length];
+ for (int i = 0; i < subplans.length; i++) {
+ try {
+ pipelines[i] = (NestedTupleSourceRuntime) assemblePipeline(subplans[i], outputWriter, rc);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ return new IAggregatorDescriptor() {
+
+ @Override
+ public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ ArrayTupleBuilder tb = outputWriter.getTupleBuilder();
+ tb.reset();
+ for (int i = 0; i < keyFieldIdx.length; ++i) {
+ tb.addField(accessor, tIndex, keyFieldIdx[i]);
+ }
+ for (int i = 0; i < decorFieldIdx.length; ++i) {
+ tb.addField(accessor, tIndex, decorFieldIdx[i]);
+ }
+ for (int i = 0; i < pipelines.length; ++i) {
+ pipelines[i].open();
+ }
+
+ // aggregate the first tuple
+ for (int i = 0; i < pipelines.length; i++) {
+ pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
+ }
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
+ int stateTupleIndex, AggregateState state) throws HyracksDataException {
+ // it only works if the output of the aggregator fits in one
+ // frame
+ for (int i = 0; i < pipelines.length; i++) {
+ pipelines[i].writeTuple(accessor.getBuffer(), tIndex);
+ }
+ }
+
+ @Override
+ public void outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ for (int i = 0; i < pipelines.length; i++) {
+ outputWriter.setInputIdx(i);
+ pipelines[i].close();
+ }
+ // outputWriter.writeTuple(appender);
+ tupleBuilder.reset();
+ ArrayTupleBuilder tb = outputWriter.getTupleBuilder();
+ byte[] data = tb.getByteArray();
+ int[] fieldEnds = tb.getFieldEndOffsets();
+ int start = 0;
+ int offset = 0;
+ for (int i = 0; i < fieldEnds.length; i++) {
+ if (i > 0)
+ start = fieldEnds[i - 1];
+ offset = fieldEnds[i] - start;
+ tupleBuilder.addField(data, start, offset);
+ }
+ }
+
+ @Override
+ public AggregateState createAggregateStates() {
+ return new AggregateState();
+ }
+
+ @Override
+ public void reset() {
+
+ }
+
+ @Override
+ public void outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ throw new IllegalStateException("this method should not be called");
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ };
+ }
+
+ private IFrameWriter assemblePipeline(AlgebricksPipeline subplan, IFrameWriter writer, RuntimeContext rc)
+ throws AlgebricksException {
+ // plug the operators
+ IFrameWriter start = writer;
+ IPushRuntimeFactory[] runtimeFactories = subplan.getRuntimeFactories();
+ RecordDescriptor[] recordDescriptors = subplan.getRecordDescriptors();
+ for (int i = runtimeFactories.length - 1; i >= 0; i--) {
+ IPushRuntime newRuntime = runtimeFactories[i].createPushRuntime(rc);
+ newRuntime.setFrameWriter(0, start, recordDescriptors[i]);
+ if (i > 0) {
+ newRuntime.setInputRecordDescriptor(0, recordDescriptors[i - 1]);
+ } else {
+ // the nts has the same input and output rec. desc.
+ newRuntime.setInputRecordDescriptor(0, recordDescriptors[0]);
+ }
+ start = newRuntime;
+ }
+ return start;
+ }
+
+ /**
+ *
+ *
+ * We suppose for now, that each subplan only produces one tuple.
+ *
+ */
+ private static class AggregatorOutput implements IFrameWriter {
+
+ // private ByteBuffer frame;
+ private FrameTupleAccessor[] tAccess;
+ private RecordDescriptor[] inputRecDesc;
+ private int inputIdx;
+ private ArrayTupleBuilder tb;
+ private AlgebricksPipeline[] subplans;
+
+ public AggregatorOutput(int frameSize, AlgebricksPipeline[] subplans, int numKeys, int numDecors) {
+ this.subplans = subplans;
+ // this.keyFieldIndexes = keyFieldIndexes;
+ int totalAggFields = 0;
+ this.inputRecDesc = new RecordDescriptor[subplans.length];
+ for (int i = 0; i < subplans.length; i++) {
+ RecordDescriptor[] rd = subplans[i].getRecordDescriptors();
+ this.inputRecDesc[i] = rd[rd.length - 1];
+ totalAggFields += subplans[i].getOutputWidth();
+ }
+ tb = new ArrayTupleBuilder(numKeys + numDecors + totalAggFields);
+
+ this.tAccess = new FrameTupleAccessor[inputRecDesc.length];
+ for (int i = 0; i < inputRecDesc.length; i++) {
+ tAccess[i] = new FrameTupleAccessor(frameSize, inputRecDesc[i]);
+ }
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+
+ }
+
+ /**
+ *
+ * Since each pipeline only produces one tuple, this method is only
+ * called by the close method of the pipelines.
+ *
+ */
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ int tIndex = 0;
+ int w = subplans[inputIdx].getOutputWidth();
+ IFrameTupleAccessor accessor = tAccess[inputIdx];
+ accessor.reset(buffer);
+ for (int f = 0; f < w; f++) {
+ tb.addField(accessor, tIndex, f);
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // clearFrame();
+ }
+
+ public void setInputIdx(int inputIdx) {
+ this.inputIdx = inputIdx;
+ }
+
+ public ArrayTupleBuilder getTupleBuilder() {
+ return tb;
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
new file mode 100644
index 0000000..ffc8c54
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SerializableAggregatorDescriptorFactory.java
@@ -0,0 +1,146 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg;
+
+import java.io.DataOutput;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+public class SerializableAggregatorDescriptorFactory implements IAggregatorDescriptorFactory {
+ private static final long serialVersionUID = 1L;
+ private ISerializableAggregateFunctionFactory[] aggFactories;
+
+ public SerializableAggregatorDescriptorFactory(ISerializableAggregateFunctionFactory[] aggFactories) {
+ this.aggFactories = aggFactories;
+ }
+
+ @Override
+ public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor, int[] keyFields, final int[] keyFieldsInPartialResults)
+ throws HyracksDataException {
+ final int[] keys = keyFields;
+
+ /**
+ * one IAggregatorDescriptor instance per Gby operator
+ */
+ return new IAggregatorDescriptor() {
+ private FrameTupleReference ftr = new FrameTupleReference();
+ private ISerializableAggregateFunction[] aggs = new ISerializableAggregateFunction[aggFactories.length];
+ private int offsetFieldIndex = keys.length;
+ private int stateFieldLength[] = new int[aggFactories.length];
+
+ @Override
+ public AggregateState createAggregateStates() {
+ return new AggregateState();
+ }
+
+ @Override
+ public void init(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex, AggregateState state)
+ throws HyracksDataException {
+ DataOutput output = tb.getDataOutput();
+ ftr.reset(accessor, tIndex);
+ for (int i = 0; i < aggs.length; i++) {
+ try {
+ int begin = tb.getSize();
+ if (aggs[i] == null) {
+ aggs[i] = aggFactories[i].createAggregateFunction();
+ }
+ aggs[i].init(output);
+ tb.addFieldEndOffset();
+ stateFieldLength[i] = tb.getSize() - begin;
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ // doing initial aggregate
+ ftr.reset(accessor, tIndex);
+ for (int i = 0; i < aggs.length; i++) {
+ try {
+ byte[] data = tb.getByteArray();
+ int prevFieldPos = i + keys.length - 1;
+ int start = prevFieldPos >= 0 ? tb.getFieldEndOffsets()[prevFieldPos] : 0;
+ aggs[i].step(ftr, data, start, stateFieldLength[i]);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
+ int stateTupleIndex, AggregateState state) throws HyracksDataException {
+ ftr.reset(accessor, tIndex);
+ int stateTupleStart = stateAccessor.getTupleStartOffset(stateTupleIndex);
+ for (int i = 0; i < aggs.length; i++) {
+ try {
+ byte[] data = stateAccessor.getBuffer().array();
+ int start = stateAccessor.getFieldStartOffset(stateTupleIndex, i + keys.length)
+ + stateTupleStart;
+ aggs[i].step(ftr, data, start, stateFieldLength[i]);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public void outputPartialResult(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ byte[] data = accessor.getBuffer().array();
+ int startOffset = accessor.getTupleStartOffset(tIndex);
+ int aggFieldOffset = accessor.getFieldStartOffset(tIndex, offsetFieldIndex);
+ int refOffset = startOffset + accessor.getFieldSlotsLength() + aggFieldOffset;
+ int start = refOffset;
+ for (int i = 0; i < aggs.length; i++) {
+ try {
+ aggs[i].finishPartial(data, start, stateFieldLength[i], tb.getDataOutput());
+ start += stateFieldLength[i];
+ tb.addFieldEndOffset();
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public void outputFinalResult(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ byte[] data = accessor.getBuffer().array();
+ int startOffset = accessor.getTupleStartOffset(tIndex);
+ int aggFieldOffset = accessor.getFieldStartOffset(tIndex, offsetFieldIndex);
+ int refOffset = startOffset + accessor.getFieldSlotsLength() + aggFieldOffset;
+ int start = refOffset;
+ for (int i = 0; i < aggs.length; i++) {
+ try {
+ aggs[i].finish(data, start, stateFieldLength[i], tb.getDataOutput());
+ start += stateFieldLength[i];
+ tb.addFieldEndOffset();
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public void reset() {
+
+ }
+
+ @Override
+ public void close() {
+ reset();
+ }
+
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
new file mode 100644
index 0000000..1743dfc
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/aggreg/SimpleAlgebricksAccumulatingAggregatorFactory.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+
+public class SimpleAlgebricksAccumulatingAggregatorFactory implements IAggregatorDescriptorFactory {
+
+ private static final long serialVersionUID = 1L;
+ private IAggregateFunctionFactory[] aggFactories;
+
+ public SimpleAlgebricksAccumulatingAggregatorFactory(IAggregateFunctionFactory[] aggFactories, int[] keys,
+ int[] fdColumns) {
+ this.aggFactories = aggFactories;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDesc,
+ RecordDescriptor outRecordDescriptor, int[] aggKeys, int[] partialKeys) throws HyracksDataException {
+
+ return new IAggregatorDescriptor() {
+
+ private FrameTupleReference ftr = new FrameTupleReference();
+
+ @Override
+ public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ Pair<ArrayBackedValueStorage[], IAggregateFunction[]> aggState = (Pair<ArrayBackedValueStorage[], IAggregateFunction[]>) state.state;
+ ArrayBackedValueStorage[] aggOutput = aggState.first;
+ IAggregateFunction[] agg = aggState.second;
+
+ // initialize aggregate functions
+ for (int i = 0; i < agg.length; i++) {
+ aggOutput[i].reset();
+ try {
+ agg[i].init();
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ ftr.reset(accessor, tIndex);
+ for (int i = 0; i < agg.length; i++) {
+ try {
+ agg[i].step(ftr);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
+ int stateTupleIndex, AggregateState state) throws HyracksDataException {
+ Pair<ArrayBackedValueStorage[], IAggregateFunction[]> aggState = (Pair<ArrayBackedValueStorage[], IAggregateFunction[]>) state.state;
+ IAggregateFunction[] agg = aggState.second;
+ ftr.reset(accessor, tIndex);
+ for (int i = 0; i < agg.length; i++) {
+ try {
+ agg[i].step(ftr);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public void outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ Pair<ArrayBackedValueStorage[], IAggregateFunction[]> aggState = (Pair<ArrayBackedValueStorage[], IAggregateFunction[]>) state.state;
+ ArrayBackedValueStorage[] aggOutput = aggState.first;
+ IAggregateFunction[] agg = aggState.second;
+ for (int i = 0; i < agg.length; i++) {
+ try {
+ agg[i].finish();
+ tupleBuilder.addField(aggOutput[i].getBytes(), aggOutput[i].getStartIndex(),
+ aggOutput[i].getLength());
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public AggregateState createAggregateStates() {
+ IAggregateFunction[] agg = new IAggregateFunction[aggFactories.length];
+ ArrayBackedValueStorage[] aggOutput = new ArrayBackedValueStorage[aggFactories.length];
+ for (int i = 0; i < agg.length; i++) {
+ aggOutput[i] = new ArrayBackedValueStorage();
+ try {
+ agg[i] = aggFactories[i].createAggregateFunction(aggOutput[i]);
+ } catch (AlgebricksException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+ return new AggregateState(new Pair<ArrayBackedValueStorage[], IAggregateFunction[]>(aggOutput, agg));
+ }
+
+ @Override
+ public void reset() {
+
+ }
+
+ @Override
+ public void outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ throw new IllegalStateException("this method should not be called");
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
new file mode 100644
index 0000000..5d5e7fb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputOneFramePushRuntime.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+
+public abstract class AbstractOneInputOneOutputOneFramePushRuntime extends AbstractOneInputOneOutputPushRuntime {
+
+ protected FrameTupleAppender appender;
+ protected ByteBuffer frame;
+ protected FrameTupleAccessor tAccess;
+ protected FrameTupleReference tRef;
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (appender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(frame, writer);
+ }
+ writer.close();
+ appender.reset(frame, true);
+ }
+
+ protected void appendToFrameFromTupleBuilder(ArrayTupleBuilder tb) throws HyracksDataException {
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ FrameUtils.flushFrame(frame, writer);
+ appender.reset(frame, true);
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException(
+ "Could not write frame (AbstractOneInputOneOutputOneFramePushRuntime.appendToFrameFromTupleBuilder).");
+ }
+ }
+ }
+
+ protected void appendProjectionToFrame(int tIndex, int[] projectionList) throws HyracksDataException {
+ if (!appender.appendProjection(tAccess, tIndex, projectionList)) {
+ FrameUtils.flushFrame(frame, writer);
+ appender.reset(frame, true);
+ if (!appender.appendProjection(tAccess, tIndex, projectionList)) {
+ throw new IllegalStateException(
+ "Could not write frame (AbstractOneInputOneOutputOneFramePushRuntime.appendProjectionToFrame).");
+ }
+ }
+ }
+
+ protected void appendTupleToFrame(int tIndex) throws HyracksDataException {
+ if (!appender.append(tAccess, tIndex)) {
+ FrameUtils.flushFrame(frame, writer);
+ appender.reset(frame, true);
+ if (!appender.append(tAccess, tIndex)) {
+ throw new IllegalStateException(
+ "Could not write frame (AbstractOneInputOneOutputOneFramePushRuntime.appendTupleToFrame).");
+ }
+ }
+ }
+
+ protected final void initAccessAppend(RuntimeContext context) {
+ IHyracksTaskContext hCtx = context.getHyracksContext();
+ // if (allocFrame) {
+ frame = hCtx.allocateFrame();
+ appender = new FrameTupleAppender(hCtx.getFrameSize());
+ appender.reset(frame, true);
+ // }
+ tAccess = new FrameTupleAccessor(hCtx.getFrameSize(), inputRecordDesc);
+ }
+
+ protected final void initAccessAppendRef(RuntimeContext context) {
+ initAccessAppend(context);
+ tRef = new FrameTupleReference();
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
new file mode 100644
index 0000000..1034232
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputPushRuntime.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public abstract class AbstractOneInputOneOutputPushRuntime extends AbstractOneInputPushRuntime {
+
+ protected RecordDescriptor inputRecordDesc;
+
+ @Override
+ public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
+ this.inputRecordDesc = recordDescriptor;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
new file mode 100644
index 0000000..23f4f6f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputOneOutputRuntimeFactory.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public abstract class AbstractOneInputOneOutputRuntimeFactory implements IPushRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ protected int[] projectionList;
+
+ public AbstractOneInputOneOutputRuntimeFactory(int[] projectionList) {
+ this.projectionList = projectionList;
+ }
+
+ @Override
+ public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException {
+ return createOneOutputPushRuntime(context);
+ }
+
+ public abstract AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(RuntimeContext context)
+ throws AlgebricksException;
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputPushRuntime.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputPushRuntime.java
new file mode 100644
index 0000000..5660b2c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputPushRuntime.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public abstract class AbstractOneInputPushRuntime implements IPushRuntime {
+ protected IFrameWriter writer;
+ protected RecordDescriptor outputRecordDesc;
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
+ this.writer = writer;
+ this.outputRecordDesc = recordDesc;
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSinkPushRuntime.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
new file mode 100644
index 0000000..82cf050
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSinkPushRuntime.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public abstract class AbstractOneInputSinkPushRuntime implements IPushRuntime {
+ protected RecordDescriptor inputRecordDesc;
+
+ @Override
+ public void setFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
+ this.inputRecordDesc = recordDescriptor;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSourcePushRuntime.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
new file mode 100644
index 0000000..3f1c0f3
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/AbstractOneInputSourcePushRuntime.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public abstract class AbstractOneInputSourcePushRuntime extends AbstractOneInputPushRuntime {
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ @Override
+ public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
+ throw new UnsupportedOperationException();
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/SinkRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/SinkRuntimeFactory.java
new file mode 100644
index 0000000..791d9a1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/base/SinkRuntimeFactory.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class SinkRuntimeFactory implements IPushRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public SinkRuntimeFactory() {
+ }
+
+ @Override
+ public String toString() {
+ return "sink";
+ }
+
+ @Override
+ public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException {
+ return new AbstractOneInputSinkPushRuntime() {
+
+ @Override
+ public void open() throws HyracksDataException {
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
new file mode 100644
index 0000000..b15d63d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/group/MicroPreClusteredGroupRuntimeFactory.java
@@ -0,0 +1,94 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.group;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupWriter;
+
+public class MicroPreClusteredGroupRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final int[] groupFields;
+ private final IBinaryComparatorFactory[] comparatorFactories;
+ private final IAggregatorDescriptorFactory aggregatorFactory;
+ private final RecordDescriptor inRecordDesc;
+ private final RecordDescriptor outRecordDesc;
+
+ public MicroPreClusteredGroupRuntimeFactory(int[] groupFields, IBinaryComparatorFactory[] comparatorFactories,
+ IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDesc,
+ RecordDescriptor outRecordDesc, int[] projectionList) {
+ super(projectionList);
+ // Obs: the projection list is currently ignored.
+ if (projectionList != null) {
+ throw new NotImplementedException("Cannot push projection into InMemorySortRuntime.");
+ }
+ this.groupFields = groupFields;
+ this.comparatorFactories = comparatorFactories;
+ this.aggregatorFactory = aggregatorFactory;
+ this.inRecordDesc = inRecordDesc;
+ this.outRecordDesc = outRecordDesc;
+ }
+
+ @Override
+ public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(RuntimeContext context)
+ throws AlgebricksException {
+ try {
+ final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ final IHyracksTaskContext ctx = context.getHyracksContext();
+ final IAggregatorDescriptor aggregator = aggregatorFactory.createAggregator(ctx, inRecordDesc,
+ outRecordDesc, groupFields, groupFields);
+ final ByteBuffer copyFrame = ctx.allocateFrame();
+ final FrameTupleAccessor copyFrameAccessor = new FrameTupleAccessor(ctx.getFrameSize(), inRecordDesc);
+ copyFrameAccessor.reset(copyFrame);
+ ByteBuffer outFrame = ctx.allocateFrame();
+ final FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ appender.reset(outFrame, true);
+
+ return new AbstractOneInputOneOutputPushRuntime() {
+
+ private PreclusteredGroupWriter pgw;
+
+ @Override
+ public void open() throws HyracksDataException {
+ pgw = new PreclusteredGroupWriter(ctx, groupFields, comparators, aggregator, inRecordDesc,
+ outRecordDesc, writer);
+ pgw.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ pgw.nextFrame(buffer);
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ pgw.fail();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ pgw.close();
+ }
+ };
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
new file mode 100644
index 0000000..2fdeb23
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/AlgebricksMetaOperatorDescriptor.java
@@ -0,0 +1,154 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta;
+
+import java.nio.ByteBuffer;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
+
+public class AlgebricksMetaOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ // array of factories for building the local runtime pipeline
+ private final AlgebricksPipeline pipeline;
+
+ public AlgebricksMetaOperatorDescriptor(JobSpecification spec, int inputArity, int outputArity,
+ IPushRuntimeFactory[] runtimeFactories, RecordDescriptor[] internalRecordDescriptors) {
+ super(spec, inputArity, outputArity);
+ if (outputArity == 1) {
+ this.recordDescriptors[0] = internalRecordDescriptors[internalRecordDescriptors.length - 1];
+ }
+ this.pipeline = new AlgebricksPipeline(runtimeFactories, internalRecordDescriptors);
+ }
+
+ public AlgebricksPipeline getPipeline() {
+ return pipeline;
+ }
+
+ @Override
+ public JSONObject toJSON() throws JSONException {
+ JSONObject json = super.toJSON();
+ json.put("micro-operators", pipeline.getRuntimeFactories());
+ return json;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Asterix { \n");
+ for (IPushRuntimeFactory f : pipeline.getRuntimeFactories()) {
+ sb.append(" " + f.toString() + ";\n");
+ }
+ sb.append("}");
+ // sb.append(super.getInputArity());
+ // sb.append(";");
+ // sb.append(super.getOutputArity());
+ // sb.append(";");
+ return sb.toString();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ if (inputArity == 0) {
+ return createSourceInputPushRuntime(ctx, recordDescProvider, partition, nPartitions);
+ } else {
+ return createOneInputOneOutputPushRuntime(ctx, recordDescProvider, partition, nPartitions);
+ }
+ }
+
+ private IOperatorNodePushable createSourceInputPushRuntime(final IHyracksTaskContext ctx,
+ final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new AbstractUnaryOutputSourceOperatorNodePushable() {
+
+ public void initialize() throws HyracksDataException {
+ IFrameWriter startOfPipeline;
+ RecordDescriptor pipelineOutputRecordDescriptor = outputArity > 0 ? AlgebricksMetaOperatorDescriptor.this.recordDescriptors[0]
+ : null;
+
+ PipelineAssembler pa = new PipelineAssembler(pipeline, inputArity, outputArity, null,
+ pipelineOutputRecordDescriptor);
+ try {
+ RuntimeContext rc = new RuntimeContext();
+ rc.setHyracksContext(ctx);
+ startOfPipeline = pa.assemblePipeline(writer, rc);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ startOfPipeline.open();
+ startOfPipeline.close();
+ }
+ };
+ }
+
+ private IOperatorNodePushable createOneInputOneOutputPushRuntime(final IHyracksTaskContext ctx,
+ final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
+
+ private IFrameWriter startOfPipeline;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (startOfPipeline == null) {
+ RecordDescriptor pipelineOutputRecordDescriptor = outputArity > 0 ? AlgebricksMetaOperatorDescriptor.this.recordDescriptors[0]
+ : null;
+ RecordDescriptor pipelineInputRecordDescriptor = recordDescProvider.getInputRecordDescriptor(
+ AlgebricksMetaOperatorDescriptor.this.getOperatorId(), 0);
+ PipelineAssembler pa = new PipelineAssembler(pipeline, inputArity, outputArity,
+ pipelineInputRecordDescriptor, pipelineOutputRecordDescriptor);
+ try {
+ RuntimeContext rc = new RuntimeContext();
+ rc.setHyracksContext(ctx);
+ startOfPipeline = pa.assemblePipeline(writer, rc);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ startOfPipeline.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ startOfPipeline.nextFrame(buffer);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ startOfPipeline.close();
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/PipelineAssembler.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/PipelineAssembler.java
new file mode 100644
index 0000000..f0f5029
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/PipelineAssembler.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class PipelineAssembler {
+
+ // array of factories for building the local runtime pipeline
+ private final RecordDescriptor pipelineInputRecordDescriptor;
+ private final RecordDescriptor pipelineOutputRecordDescriptor;
+
+ private final int inputArity;
+ private final int outputArity;
+ private final AlgebricksPipeline pipeline;
+
+ public PipelineAssembler(AlgebricksPipeline pipeline, int inputArity, int outputArity,
+ RecordDescriptor pipelineInputRecordDescriptor, RecordDescriptor pipelineOutputRecordDescriptor) {
+ this.pipeline = pipeline;
+ this.pipelineInputRecordDescriptor = pipelineInputRecordDescriptor;
+ this.pipelineOutputRecordDescriptor = pipelineOutputRecordDescriptor;
+ this.inputArity = inputArity;
+ this.outputArity = outputArity;
+ }
+
+ public IFrameWriter assemblePipeline(IFrameWriter writer, RuntimeContext rc) throws AlgebricksException {
+ // plug the operators
+ IFrameWriter start = writer;// this.writer;
+ for (int i = pipeline.getRuntimeFactories().length - 1; i >= 0; i--) {
+ IPushRuntime newRuntime = pipeline.getRuntimeFactories()[i].createPushRuntime(rc);
+ if (i == pipeline.getRuntimeFactories().length - 1) {
+ if (outputArity == 1) {
+ newRuntime.setFrameWriter(0, start, pipelineOutputRecordDescriptor);
+ }
+ } else {
+ newRuntime.setFrameWriter(0, start, pipeline.getRecordDescriptors()[i]);
+ }
+ if (i > 0) {
+ newRuntime.setInputRecordDescriptor(0, pipeline.getRecordDescriptors()[i - 1]);
+ } else if (inputArity > 0) {
+ newRuntime.setInputRecordDescriptor(0, pipelineInputRecordDescriptor);
+ }
+ start = newRuntime;
+ }
+ return start;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/SubplanRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/SubplanRuntimeFactory.java
new file mode 100644
index 0000000..53808e1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/meta/SubplanRuntimeFactory.java
@@ -0,0 +1,177 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta;
+
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.NestedTupleSourceRuntimeFactory.NestedTupleSourceRuntime;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+
+public class SubplanRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final AlgebricksPipeline pipeline;
+ private final RecordDescriptor inputRecordDesc;
+ private final INullWriterFactory[] nullWriterFactories;
+
+ public SubplanRuntimeFactory(AlgebricksPipeline pipeline, INullWriterFactory[] nullWriterFactories,
+ RecordDescriptor inputRecordDesc, int[] projectionList) {
+ super(projectionList);
+ this.pipeline = pipeline;
+ this.nullWriterFactories = nullWriterFactories;
+ this.inputRecordDesc = inputRecordDesc;
+ if (projectionList != null) {
+ throw new NotImplementedException();
+ }
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Subplan { \n");
+ for (IPushRuntimeFactory f : pipeline.getRuntimeFactories()) {
+ sb.append(" " + f.toString() + ";\n");
+ }
+ sb.append("}");
+ return sb.toString();
+ }
+
+ @Override
+ public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ throws AlgebricksException {
+
+ RecordDescriptor pipelineOutputRecordDescriptor = null;
+
+ final PipelineAssembler pa = new PipelineAssembler(pipeline, 1, 1, inputRecordDesc,
+ pipelineOutputRecordDescriptor);
+ final INullWriter[] nullWriters = new INullWriter[nullWriterFactories.length];
+ for (int i = 0; i < nullWriterFactories.length; i++) {
+ nullWriters[i] = nullWriterFactories[i].createNullWriter();
+ }
+
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ /**
+ *
+ * Computes the outer product between a given tuple and the frames
+ * passed.
+ *
+ */
+ class TupleOuterProduct implements IFrameWriter {
+
+ private boolean smthWasWritten = false;
+ private IHyracksTaskContext hCtx = context.getHyracksContext();
+ private int frameSize = hCtx.getFrameSize();
+ private FrameTupleAccessor ta = new FrameTupleAccessor(frameSize,
+ pipeline.getRecordDescriptors()[pipeline.getRecordDescriptors().length - 1]);
+ private ArrayTupleBuilder tb = new ArrayTupleBuilder(nullWriters.length);
+
+ @Override
+ public void open() throws HyracksDataException {
+ smthWasWritten = false;
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ ta.reset(buffer);
+ int nTuple = ta.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ if (!appender.appendConcat(tRef.getFrameTupleAccessor(), tRef.getTupleIndex(), ta, t)) {
+ FrameUtils.flushFrame(frame, writer);
+ appender.reset(frame, true);
+ if (!appender.appendConcat(tRef.getFrameTupleAccessor(), tRef.getTupleIndex(), ta, t)) {
+ throw new IllegalStateException("Could not write frame.");
+ }
+ }
+ }
+ smthWasWritten = true;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (!smthWasWritten) {
+ // the case when we need to write nulls
+ appendNullsToTuple();
+ appendToFrameFromTupleBuilder(tb);
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ private void appendNullsToTuple() throws HyracksDataException {
+ tb.reset();
+ int n0 = tRef.getFieldCount();
+ for (int f = 0; f < n0; f++) {
+ tb.addField(tRef.getFrameTupleAccessor(), tRef.getTupleIndex(), f);
+ }
+ DataOutput dos = tb.getDataOutput();
+ for (int i = 0; i < nullWriters.length; i++) {
+ nullWriters[i].writeNull(dos);
+ tb.addFieldEndOffset();
+ }
+ }
+
+ }
+
+ IFrameWriter endPipe = new TupleOuterProduct();
+
+ NestedTupleSourceRuntime startOfPipeline = (NestedTupleSourceRuntime) pa.assemblePipeline(endPipe, context);
+
+ boolean first = true;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (first) {
+ first = false;
+ initAccessAppendRef(context);
+ }
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ tRef.reset(tAccess, t);
+ startOfPipeline.writeTuple(buffer, t);
+ startOfPipeline.open();
+ startOfPipeline.close();
+ }
+ }
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/sort/InMemorySortRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/sort/InMemorySortRuntimeFactory.java
new file mode 100644
index 0000000..2a6c196
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/sort/InMemorySortRuntimeFactory.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.sort;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.std.sort.FrameSorter;
+
+public class InMemorySortRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int[] sortFields;
+ private INormalizedKeyComputerFactory firstKeyNormalizerFactory;
+ private IBinaryComparatorFactory[] comparatorFactories;
+
+ public InMemorySortRuntimeFactory(int[] sortFields, INormalizedKeyComputerFactory firstKeyNormalizerFactory,
+ IBinaryComparatorFactory[] comparatorFactories, int[] projectionList) {
+ super(projectionList);
+ // Obs: the projection list is currently ignored.
+ if (projectionList != null) {
+ throw new NotImplementedException("Cannot push projection into InMemorySortRuntime.");
+ }
+ this.sortFields = sortFields;
+ this.firstKeyNormalizerFactory = firstKeyNormalizerFactory;
+ this.comparatorFactories = comparatorFactories;
+ }
+
+ @Override
+ public AbstractOneInputOneOutputPushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ throws AlgebricksException {
+
+ return new AbstractOneInputOneOutputPushRuntime() {
+
+ FrameSorter frameSorter = null;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (frameSorter == null) {
+ frameSorter = new FrameSorter(context.getHyracksContext(), sortFields, firstKeyNormalizerFactory,
+ comparatorFactories, outputRecordDesc);
+ }
+ frameSorter.reset();
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ frameSorter.insertFrame(buffer);
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ frameSorter.sortFrames();
+ frameSorter.flushFrames(writer);
+ writer.close();
+ }
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/AssignRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/AssignRuntimeFactory.java
new file mode 100644
index 0000000..04dc7d6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/AssignRuntimeFactory.java
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+
+public class AssignRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private int[] outColumns;
+ private IEvaluatorFactory[] evalFactories;
+
+ /**
+ *
+ * @param outColumns
+ * a sorted array of columns into which the result is written to
+ * @param evalFactories
+ * @param projectionList
+ * an array of columns to be projected
+ */
+
+ public AssignRuntimeFactory(int[] outColumns, IEvaluatorFactory[] evalFactories, int[] projectionList) {
+ super(projectionList);
+ this.outColumns = outColumns;
+ this.evalFactories = evalFactories;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("assign [");
+ for (int i = 0; i < outColumns.length; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(outColumns[i]);
+ }
+ sb.append("] := [");
+ for (int i = 0; i < evalFactories.length; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(evalFactories[i]);
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ throws AlgebricksException {
+ final int[] projectionToOutColumns = new int[projectionList.length];
+ for (int j = 0; j < projectionList.length; j++) {
+ projectionToOutColumns[j] = Arrays.binarySearch(outColumns, projectionList[j]);
+ }
+
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ private ArrayBackedValueStorage evalOutput = new ArrayBackedValueStorage();
+ private IEvaluator[] eval = new IEvaluator[evalFactories.length];
+ private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(projectionList.length);
+ private boolean first = true;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (first) {
+ initAccessAppendRef(context);
+ first = false;
+ int n = evalFactories.length;
+ for (int i = 0; i < n; i++) {
+ try {
+ eval[i] = evalFactories[i].createEvaluator(evalOutput);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ }
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ tRef.reset(tAccess, t);
+ produceTuple(tupleBuilder, tAccess, t, tRef);
+ appendToFrameFromTupleBuilder(tupleBuilder);
+ }
+ }
+
+ private void produceTuple(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex,
+ FrameTupleReference tupleRef) throws HyracksDataException {
+ tb.reset();
+ for (int f = 0; f < projectionList.length; f++) {
+ int k = projectionToOutColumns[f];
+ if (k >= 0) {
+ evalOutput.reset();
+ try {
+ eval[k].evaluate(tupleRef);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ tb.addField(evalOutput.getBytes(), evalOutput.getStartIndex(), evalOutput.getLength());
+ } else {
+ tb.addField(accessor, tIndex, projectionList[f]);
+ }
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
new file mode 100644
index 0000000..2d0a038
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/EmptyTupleSourceRuntimeFactory.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputSourcePushRuntime;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+
+public class EmptyTupleSourceRuntimeFactory implements IPushRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public EmptyTupleSourceRuntimeFactory() {
+ }
+
+ @Override
+ public String toString() {
+ return "ets";
+ }
+
+ @Override
+ public IPushRuntime createPushRuntime(final RuntimeContext context) {
+ return new AbstractOneInputSourcePushRuntime() {
+
+ private IHyracksTaskContext hCtx = context.getHyracksContext();
+ private ByteBuffer frame = hCtx.allocateFrame();
+ private ArrayTupleBuilder tb = new ArrayTupleBuilder(0);
+ private FrameTupleAppender appender = new FrameTupleAppender(hCtx.getFrameSize());
+
+ @Override
+ public void open() throws HyracksDataException {
+ writer.open();
+ appender.reset(frame, true);
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ FrameUtils.flushFrame(frame, writer);
+ writer.close();
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/NestedTupleSourceRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
new file mode 100644
index 0000000..8c650b5
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/NestedTupleSourceRuntimeFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class NestedTupleSourceRuntimeFactory implements IPushRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public NestedTupleSourceRuntimeFactory() {
+ }
+
+ @Override
+ public String toString() {
+ return "nts";
+ }
+
+ @Override
+ public IPushRuntime createPushRuntime(RuntimeContext context) {
+ return new NestedTupleSourceRuntime(context);
+ }
+
+ public static class NestedTupleSourceRuntime extends AbstractOneInputOneOutputOneFramePushRuntime {
+
+ public NestedTupleSourceRuntime(RuntimeContext rc) {
+ initAccessAppend(rc);
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ writer.open();
+ }
+
+ public void writeTuple(ByteBuffer inputBuffer, int tIndex) throws HyracksDataException {
+ tAccess.reset(inputBuffer);
+ appendTupleToFrame(tIndex);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ throw new IllegalStateException();
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/PrinterRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/PrinterRuntimeFactory.java
new file mode 100644
index 0000000..787e001
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/PrinterRuntimeFactory.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers.PrinterBasedWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class PrinterRuntimeFactory implements IPushRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int[] printColumns;
+ private final IPrinterFactory[] printerFactories;
+ private final RecordDescriptor inputRecordDesc;
+
+ public PrinterRuntimeFactory(int[] printColumns, IPrinterFactory[] printerFactories,
+ RecordDescriptor inputRecordDesc) {
+ this.printColumns = printColumns;
+ this.printerFactories = printerFactories;
+ this.inputRecordDesc = inputRecordDesc;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append("print [");
+ for (int i = 0; i < printColumns.length; i++) {
+ if (i > 0) {
+ buf.append("; ");
+ }
+ buf.append(printColumns[i]);
+ }
+ buf.append("]");
+ return buf.toString();
+ }
+
+ @Override
+ public IPushRuntime createPushRuntime(final RuntimeContext context) {
+ IAWriter w = PrinterBasedWriterFactory.INSTANCE.createWriter(printColumns, System.out, printerFactories,
+ inputRecordDesc);
+ return new SinkWriterRuntime(w, context, System.out, inputRecordDesc);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/RunningAggregateRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/RunningAggregateRuntimeFactory.java
new file mode 100644
index 0000000..bca8f2f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/RunningAggregateRuntimeFactory.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunction;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+
+public class RunningAggregateRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private int[] outColumns;
+ private IRunningAggregateFunctionFactory[] runningAggregates;
+
+ /**
+ *
+ * @param outColumns
+ * a sorted array of columns into which the result is written to
+ * @param runningAggregates
+ * @param projectionList
+ * an array of columns to be projected
+ */
+
+ public RunningAggregateRuntimeFactory(int[] outColumns, IRunningAggregateFunctionFactory[] runningAggregates,
+ int[] projectionList) {
+ super(projectionList);
+ this.outColumns = outColumns;
+ this.runningAggregates = runningAggregates;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("running-aggregate [");
+ for (int i = 0; i < outColumns.length; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(outColumns[i]);
+ }
+ sb.append("] := [");
+ for (int i = 0; i < runningAggregates.length; i++) {
+ if (i > 0) {
+ sb.append(", ");
+ }
+ sb.append(runningAggregates[i]);
+ }
+ sb.append("]");
+ return sb.toString();
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ throws AlgebricksException {
+ final int[] projectionToOutColumns = new int[projectionList.length];
+ for (int j = 0; j < projectionList.length; j++) {
+ projectionToOutColumns[j] = Arrays.binarySearch(outColumns, projectionList[j]);
+ }
+
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ private ArrayBackedValueStorage evalOutput = new ArrayBackedValueStorage();
+ private IRunningAggregateFunction[] raggs = new IRunningAggregateFunction[runningAggregates.length];
+ private ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(projectionList.length);
+ private boolean first = true;
+
+ @Override
+ public void open() throws HyracksDataException {
+ initAccessAppendRef(context);
+ if (first) {
+ first = false;
+ int n = runningAggregates.length;
+ for (int i = 0; i < n; i++) {
+ try {
+ raggs[i] = runningAggregates[i].createRunningAggregateFunction(evalOutput);
+ raggs[i].init();
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ }
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ tRef.reset(tAccess, t);
+ produceTuple(tupleBuilder, tAccess, t, tRef);
+ appendToFrameFromTupleBuilder(tupleBuilder);
+ }
+ }
+
+ private void produceTuple(ArrayTupleBuilder tb, IFrameTupleAccessor accessor, int tIndex,
+ FrameTupleReference tupleRef) throws HyracksDataException {
+ tb.reset();
+ for (int f = 0; f < projectionList.length; f++) {
+ int k = projectionToOutColumns[f];
+ if (k >= 0) {
+ evalOutput.reset();
+ try {
+ raggs[k].step(tupleRef);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ tb.addField(evalOutput.getBytes(), evalOutput.getStartIndex(), evalOutput.getLength());
+ } else {
+ tb.addField(accessor, tIndex, projectionList[f]);
+ }
+ }
+ }
+
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntime.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntime.java
new file mode 100644
index 0000000..8101a8a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntime.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.io.PrintStream;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputSinkPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+
+public class SinkWriterRuntime extends AbstractOneInputSinkPushRuntime {
+
+ private final RuntimeContext context;
+ private final PrintStream printStream;
+ private final IAWriter writer;
+ private RecordDescriptor inputRecordDesc;
+ private FrameTupleAccessor tAccess;
+ private boolean autoClose = false;
+ private boolean first = true;
+
+ public SinkWriterRuntime(IAWriter writer, RuntimeContext context, PrintStream printStream,
+ RecordDescriptor inputRecordDesc) {
+ this.writer = writer;
+ this.context = context;
+ this.printStream = printStream;
+ this.inputRecordDesc = inputRecordDesc;
+ this.tAccess = new FrameTupleAccessor(context.getHyracksContext().getFrameSize(), inputRecordDesc);
+ }
+
+ public SinkWriterRuntime(IAWriter writer, RuntimeContext context, PrintStream printStream,
+ RecordDescriptor inputRecordDesc, boolean autoClose) {
+ this(writer, context, printStream, inputRecordDesc);
+ this.autoClose = autoClose;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (first) {
+ first = false;
+ tAccess = new FrameTupleAccessor(context.getHyracksContext().getFrameSize(), inputRecordDesc);
+ try {
+ writer.init();
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ try {
+ writer.printTuple(tAccess, t);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (autoClose) {
+ printStream.close();
+ }
+ }
+
+ @Override
+ public void setInputRecordDescriptor(int index, RecordDescriptor recordDescriptor) {
+ this.inputRecordDesc = recordDescriptor;
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntimeFactory.java
new file mode 100644
index 0000000..4ac1d16
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/SinkWriterRuntimeFactory.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+
+public class SinkWriterRuntimeFactory implements IPushRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int[] fields;
+ private final IPrinterFactory[] printerFactories;
+ private final File outputFile;
+ private final RecordDescriptor inputRecordDesc;
+ private final IAWriterFactory writerFactory;
+
+ public SinkWriterRuntimeFactory(int[] fields, IPrinterFactory[] printerFactories, File outputFile,
+ IAWriterFactory writerFactory, RecordDescriptor inputRecordDesc) {
+ this.fields = fields;
+ this.printerFactories = printerFactories;
+ this.outputFile = outputFile;
+ this.writerFactory = writerFactory;
+ this.inputRecordDesc = inputRecordDesc;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder();
+ buf.append("sink-write " + "[");
+ for (int i = 0; i < fields.length; i++) {
+ if (i > 0) {
+ buf.append("; ");
+ }
+ buf.append(fields[i]);
+ }
+ buf.append("] outputFile");
+ return buf.toString();
+ }
+
+ @Override
+ public IPushRuntime createPushRuntime(RuntimeContext context) throws AlgebricksException {
+ PrintStream filePrintStream = null;
+ try {
+ filePrintStream = new PrintStream(new BufferedOutputStream(new FileOutputStream(outputFile)));
+ } catch (FileNotFoundException e) {
+ throw new AlgebricksException(e);
+ }
+ IAWriter w = writerFactory.createWriter(fields, filePrintStream, printerFactories, inputRecordDesc);
+ return new SinkWriterRuntime(w, context, filePrintStream, inputRecordDesc, true);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamDieRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamDieRuntimeFactory.java
new file mode 100644
index 0000000..86b9b3b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamDieRuntimeFactory.java
@@ -0,0 +1,98 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+
+public class StreamDieRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private IEvaluatorFactory aftterObjectsEvalFactory;
+ private IBinaryIntegerInspector binaryIntegerInspector;
+
+ public StreamDieRuntimeFactory(IEvaluatorFactory maxObjectsEvalFactory, int[] projectionList,
+ IBinaryIntegerInspector binaryIntegerInspector) {
+ super(projectionList);
+ this.aftterObjectsEvalFactory = maxObjectsEvalFactory;
+ this.binaryIntegerInspector = binaryIntegerInspector;
+ }
+
+ @Override
+ public String toString() {
+ String s = "stream-die " + aftterObjectsEvalFactory.toString();
+ return s;
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context) {
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ private IEvaluator evalAfterObjects;
+ private ArrayBackedValueStorage evalOutput;
+ private int toWrite = -1;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (evalAfterObjects == null) {
+ initAccessAppendRef(context);
+ evalOutput = new ArrayBackedValueStorage();
+ try {
+ evalAfterObjects = aftterObjectsEvalFactory.createEvaluator(evalOutput);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ if (toWrite < 0) {
+ toWrite = evaluateInteger(evalAfterObjects, 0);
+ }
+ for (int t = 0; t < nTuple; t++) {
+ if (toWrite > 0) {
+ toWrite--;
+ if (projectionList != null) {
+ appendProjectionToFrame(t, projectionList);
+ } else {
+ appendTupleToFrame(t);
+ }
+ } else {
+ throw new HyracksDataException("injected failure");
+ }
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ super.close();
+ }
+
+ private int evaluateInteger(IEvaluator eval, int tIdx) throws HyracksDataException {
+ tRef.reset(tAccess, tIdx);
+ evalOutput.reset();
+ try {
+ eval.evaluate(tRef);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ int lim = binaryIntegerInspector.getIntegerValue(evalOutput.getBytes(), 0, evalOutput.getLength());
+ return lim;
+ }
+
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamLimitRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamLimitRuntimeFactory.java
new file mode 100644
index 0000000..5caea6e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamLimitRuntimeFactory.java
@@ -0,0 +1,135 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryIntegerInspector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+
+public class StreamLimitRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private IEvaluatorFactory maxObjectsEvalFactory;
+ private IEvaluatorFactory offsetEvalFactory;
+ private IBinaryIntegerInspector binaryIntegerInspector;
+
+ public StreamLimitRuntimeFactory(IEvaluatorFactory maxObjectsEvalFactory, IEvaluatorFactory offsetEvalFactory,
+ int[] projectionList, IBinaryIntegerInspector binaryIntegerInspector) {
+ super(projectionList);
+ this.maxObjectsEvalFactory = maxObjectsEvalFactory;
+ this.offsetEvalFactory = offsetEvalFactory;
+ this.binaryIntegerInspector = binaryIntegerInspector;
+ }
+
+ @Override
+ public String toString() {
+ String s = "stream-limit " + maxObjectsEvalFactory.toString();
+ if (offsetEvalFactory != null) {
+ return s + ", " + offsetEvalFactory.toString();
+ } else {
+ return s;
+ }
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context) {
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ private IEvaluator evalMaxObjects;
+ private ArrayBackedValueStorage evalOutput;
+ private IEvaluator evalOffset = null;
+ private int toWrite = 0; // how many tuples still to write
+ private int toSkip = 0; // how many tuples still to skip
+ private boolean firstTuple = true;
+ private boolean afterLastTuple = false;
+
+ @Override
+ public void open() throws HyracksDataException {
+ // if (first) {
+ if (evalMaxObjects == null) {
+ initAccessAppendRef(context);
+ evalOutput = new ArrayBackedValueStorage();
+ try {
+ evalMaxObjects = maxObjectsEvalFactory.createEvaluator(evalOutput);
+ if (offsetEvalFactory != null) {
+ evalOffset = offsetEvalFactory.createEvaluator(evalOutput);
+ }
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ writer.open();
+ afterLastTuple = false;
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ if (afterLastTuple) {
+ // ignore the data
+ return;
+ }
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ int start = 0;
+ if (nTuple <= toSkip) {
+ toSkip -= nTuple;
+ return;
+ } else if (toSkip > 0) {
+ start = toSkip;
+ toSkip = 0;
+ }
+ for (int t = start; t < nTuple; t++) {
+ if (firstTuple) {
+ firstTuple = false;
+ toWrite = evaluateInteger(evalMaxObjects, t);
+ if (evalOffset != null) {
+ toSkip = evaluateInteger(evalOffset, t);
+ }
+ }
+ if (toSkip > 0) {
+ toSkip--;
+ } else if (toWrite > 0) {
+ toWrite--;
+ if (projectionList != null) {
+ appendProjectionToFrame(t, projectionList);
+ } else {
+ appendTupleToFrame(t);
+ }
+ } else {
+ // close();
+ afterLastTuple = true;
+ break;
+ }
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // if (!afterLastTuple) {
+ super.close();
+ // }
+ }
+
+ private int evaluateInteger(IEvaluator eval, int tIdx) throws HyracksDataException {
+ tRef.reset(tAccess, tIdx);
+ evalOutput.reset();
+ try {
+ eval.evaluate(tRef);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ int lim = binaryIntegerInspector.getIntegerValue(evalOutput.getBytes(), 0, evalOutput.getLength());
+ return lim;
+ }
+
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamProjectRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamProjectRuntimeFactory.java
new file mode 100644
index 0000000..fe30701
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamProjectRuntimeFactory.java
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public class StreamProjectRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public StreamProjectRuntimeFactory(int[] projectionList) {
+ super(projectionList);
+ }
+
+ @Override
+ public String toString() {
+ return "stream-project " + Arrays.toString(projectionList);
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ throws AlgebricksException {
+
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ private boolean first = true;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (first) {
+ first = false;
+ initAccessAppend(context);
+ }
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ appendProjectionToFrame(t, projectionList);
+ }
+
+ }
+
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamSelectRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamSelectRuntimeFactory.java
new file mode 100644
index 0000000..be579e5
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StreamSelectRuntimeFactory.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IBinaryBooleanInspector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+
+public class StreamSelectRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private IEvaluatorFactory cond;
+
+ private IBinaryBooleanInspector binaryBooleanInspector;
+
+ /**
+ *
+ * @param cond
+ * @param projectionList
+ * if projectionList is null, then no projection is performed
+ */
+ public StreamSelectRuntimeFactory(IEvaluatorFactory cond, int[] projectionList,
+ IBinaryBooleanInspector binaryBooleanInspector) {
+ super(projectionList);
+ this.cond = cond;
+ this.binaryBooleanInspector = binaryBooleanInspector;
+ }
+
+ @Override
+ public String toString() {
+ return "stream-select " + cond.toString();
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context) {
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ private IEvaluator eval;
+ private ArrayBackedValueStorage evalOutput;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (eval == null) {
+ initAccessAppendRef(context);
+ evalOutput = new ArrayBackedValueStorage();
+ try {
+ eval = cond.createEvaluator(evalOutput);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ tRef.reset(tAccess, t);
+ evalOutput.reset();
+ try {
+ eval.evaluate(tRef);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ if (binaryBooleanInspector.getBooleanValue(evalOutput.getBytes(), 0, evalOutput.getLength())) {
+ if (projectionList != null) {
+ appendProjectionToFrame(t, projectionList);
+ } else {
+ appendTupleToFrame(t);
+ }
+ }
+ }
+ }
+
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StringStreamingRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StringStreamingRuntimeFactory.java
new file mode 100644
index 0000000..f3d07b1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/StringStreamingRuntimeFactory.java
@@ -0,0 +1,189 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.std.file.ITupleParser;
+import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
+
+public class StringStreamingRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private String command;
+ private IPrinterFactory[] printerFactories;
+ private char fieldDelimiter;
+ private ITupleParserFactory parserFactory;
+
+ public StringStreamingRuntimeFactory(String command, IPrinterFactory[] printerFactories, char fieldDelimiter,
+ ITupleParserFactory parserFactory) {
+ super(null);
+ this.command = command;
+ this.printerFactories = printerFactories;
+ this.fieldDelimiter = fieldDelimiter;
+ this.parserFactory = parserFactory;
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ throws AlgebricksException {
+ final IPrinter[] printers = new IPrinter[printerFactories.length];
+ for (int i = 0; i < printerFactories.length; i++) {
+ printers[i] = printerFactories[i].createPrinter();
+ }
+
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ final class ForwardScriptOutput implements Runnable {
+
+ private InputStream inStream;
+ private ITupleParser parser;
+
+ public ForwardScriptOutput(ITupleParser parser, InputStream inStream) {
+ this.parser = parser;
+ this.inStream = inStream;
+ }
+
+ @Override
+ public void run() {
+ try {
+ parser.parse(inStream, writer);
+ } catch (HyracksDataException e) {
+ throw new RuntimeException(e);
+ } finally {
+ try {
+ inStream.close();
+ } catch (Exception e) {
+ }
+ }
+ }
+ }
+
+ final class DumpInStreamToPrintStream implements Runnable {
+
+ private BufferedReader reader;
+ private PrintStream printStream;
+
+ public DumpInStreamToPrintStream(InputStream inStream, PrintStream printStream) {
+ this.reader = new BufferedReader(new InputStreamReader(inStream));
+ this.printStream = printStream;
+ }
+
+ @Override
+ public void run() {
+ String s;
+ try {
+ while ((s = reader.readLine()) != null) {
+ printStream.println(s);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ } finally {
+ try {
+ reader.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ printStream.close();
+ }
+ }
+
+ }
+
+ private Process process;
+ private PrintStream ps;
+ private boolean first = true;
+ private Thread outputPipe;
+ private Thread dumpStderr;
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (first) {
+ first = false;
+ initAccessAppendRef(context);
+ }
+
+ try {
+ ITupleParser parser = parserFactory.createTupleParser(context.getHyracksContext());
+ process = Runtime.getRuntime().exec(command);
+ ps = new PrintStream(process.getOutputStream());
+ ForwardScriptOutput fso = new ForwardScriptOutput(parser, process.getInputStream());
+ outputPipe = new Thread(fso);
+ outputPipe.start();
+ DumpInStreamToPrintStream disps = new DumpInStreamToPrintStream(process.getErrorStream(),
+ System.err);
+ dumpStderr = new Thread(disps);
+ dumpStderr.start();
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ tRef.reset(tAccess, t);
+ for (int i = 0; i < printers.length; i++) {
+ try {
+ printers[i].print(buffer.array(), tRef.getFieldStart(i), tRef.getFieldLength(i), ps);
+ } catch (AlgebricksException e) {
+ throw new HyracksDataException(e);
+ }
+ ps.print(fieldDelimiter);
+ if (i == printers.length - 1) {
+ ps.print('\n');
+ }
+ }
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ // first close the printer printing to the process
+ ps.close();
+ int ret = 0;
+ // then wait for the process to finish
+
+ try {
+ ret = process.waitFor();
+ outputPipe.join();
+ dumpStderr.join();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ if (ret != 0) {
+ throw new HyracksDataException("Process exit value: " + ret);
+ }
+ // close the following operator in the chain
+ super.close();
+ }
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/UnnestRuntimeFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/UnnestRuntimeFactory.java
new file mode 100644
index 0000000..bc55b6a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/operators/std/UnnestRuntimeFactory.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunction;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.context.RuntimeContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputOneFramePushRuntime;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.base.AbstractOneInputOneOutputRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+
+public class UnnestRuntimeFactory extends AbstractOneInputOneOutputRuntimeFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int outCol;
+ private final IUnnestingFunctionFactory unnestingFactory;
+ private int outColPos;
+ private final boolean outColIsProjected;
+
+ // Each time step() is called on the aggregate, a new value is written in
+ // its output. One byte is written before that value and is neglected.
+ // By convention, if the aggregate function writes nothing, it means it
+ // produced the last value.
+
+ public UnnestRuntimeFactory(int outCol, IUnnestingFunctionFactory unnestingFactory, int[] projectionList) {
+ super(projectionList);
+ this.outCol = outCol;
+ this.unnestingFactory = unnestingFactory;
+ outColPos = -1;
+ for (int f = 0; f < projectionList.length; f++) {
+ if (projectionList[f] == outCol) {
+ outColPos = f;
+ }
+ }
+ outColIsProjected = outColPos >= 0;
+ }
+
+ @Override
+ public String toString() {
+ return "unnest " + outCol + " <- " + unnestingFactory;
+ }
+
+ @Override
+ public AbstractOneInputOneOutputOneFramePushRuntime createOneOutputPushRuntime(final RuntimeContext context)
+ throws AlgebricksException {
+
+ return new AbstractOneInputOneOutputOneFramePushRuntime() {
+
+ private ArrayBackedValueStorage evalOutput;
+ private IUnnestingFunction agg;
+ private ArrayTupleBuilder tupleBuilder;
+
+ @Override
+ public void open() throws HyracksDataException {
+ initAccessAppendRef(context);
+ evalOutput = new ArrayBackedValueStorage();
+ try {
+ agg = unnestingFactory.createUnnestingFunction(evalOutput);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ tupleBuilder = new ArrayTupleBuilder(projectionList.length);
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tAccess.reset(buffer);
+ int nTuple = tAccess.getTupleCount();
+ for (int t = 0; t < nTuple; t++) {
+ tRef.reset(tAccess, t);
+ try {
+ agg.init(tRef);
+ boolean goon = true;
+ do {
+ tupleBuilder.reset();
+ evalOutput.reset();
+ if (!agg.step()) {
+ goon = false;
+ } else {
+ if (!outColIsProjected) {
+ appendProjectionToFrame(t, projectionList);
+ } else {
+ for (int f = 0; f < outColPos; f++) {
+ tupleBuilder.addField(tAccess, t, f);
+ }
+ tupleBuilder.addField(evalOutput.getBytes(), evalOutput.getStartIndex(), evalOutput
+ .getLength());
+ for (int f = outColPos + 1; f < projectionList.length; f++) {
+ tupleBuilder.addField(tAccess, t, f);
+ }
+ }
+ appendToFrameFromTupleBuilder(tupleBuilder);
+ }
+ } while (goon);
+ } catch (AlgebricksException ae) {
+ throw new HyracksDataException(ae);
+ }
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/PrinterBasedWriterFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/PrinterBasedWriterFactory.java
new file mode 100644
index 0000000..36ef827
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/PrinterBasedWriterFactory.java
@@ -0,0 +1,54 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers;
+
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+
+public class PrinterBasedWriterFactory implements IAWriterFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final PrinterBasedWriterFactory INSTANCE = new PrinterBasedWriterFactory();
+
+ public PrinterBasedWriterFactory() {
+ }
+
+ @Override
+ public IAWriter createWriter(final int[] fields, final PrintStream printStream,
+ final IPrinterFactory[] printerFactories, RecordDescriptor inputRecordDescriptor) {
+ final IPrinter[] printers = new IPrinter[printerFactories.length];
+ for (int i = 0; i < printerFactories.length; i++) {
+ printers[i] = printerFactories[i].createPrinter();
+ }
+
+ return new IAWriter() {
+
+ @Override
+ public void init() throws AlgebricksException {
+ for (int i = 0; i < printers.length; i++) {
+ printers[i].init();
+ }
+ }
+
+ @Override
+ public void printTuple(FrameTupleAccessor tAccess, int tIdx) throws AlgebricksException {
+ for (int i = 0; i < fields.length; i++) {
+ int fldStart = tAccess.getTupleStartOffset(tIdx) + tAccess.getFieldSlotsLength()
+ + tAccess.getFieldStartOffset(tIdx, fields[i]);
+ int fldLen = tAccess.getFieldLength(tIdx, fields[i]);
+ if (i > 0) {
+ printStream.print("; ");
+ }
+ printers[i].print(tAccess.getBuffer().array(), fldStart, fldLen, printStream);
+ }
+ printStream.println();
+ }
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/SerializedDataWriterFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/SerializedDataWriterFactory.java
new file mode 100644
index 0000000..79d7a99
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/runtime/writers/SerializedDataWriterFactory.java
@@ -0,0 +1,49 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.ObjectOutputStream;
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IAWriterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+
+public class SerializedDataWriterFactory implements IAWriterFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IAWriter createWriter(final int[] fields, final PrintStream ps, IPrinterFactory[] printerFactories,
+ final RecordDescriptor inputRecordDescriptor) {
+ return new IAWriter() {
+
+ @Override
+ public void init() throws AlgebricksException {
+ // dump the SerializerDeserializers to disk
+ try {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(baos);
+ oos.writeObject(inputRecordDescriptor);
+ baos.writeTo(ps);
+ oos.close();
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ @Override
+ public void printTuple(FrameTupleAccessor tAccess, int tIdx) throws AlgebricksException {
+ for (int i = 0; i < fields.length; i++) {
+ int fldStart = tAccess.getTupleStartOffset(tIdx) + tAccess.getFieldSlotsLength()
+ + tAccess.getFieldStartOffset(tIdx, fields[i]);
+ int fldLen = tAccess.getFieldLength(tIdx, fields[i]);
+ ps.write(tAccess.getBuffer().array(), fldStart, fldLen);
+ }
+ }
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
new file mode 100644
index 0000000..f292dd7
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/IScriptDescription.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.scripting;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public interface IScriptDescription {
+ public enum ScriptKind {
+ STRING_STREAMING
+ }
+
+ public ScriptKind getKind();
+
+ public List<Pair<LogicalVariable, Object>> getVarTypePairs();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
new file mode 100644
index 0000000..64cea03
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/scripting/StringStreamingScriptDescription.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.scripting;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
+
+public class StringStreamingScriptDescription implements IScriptDescription {
+
+ private final String command;
+ private final IPrinterFactory[] printerFactories;
+ private final char fieldDelimiter;
+ private final ITupleParserFactory parserFactory;
+ private final List<Pair<LogicalVariable, Object>> varTypePairs;
+
+ public StringStreamingScriptDescription(String command, IPrinterFactory[] printerFactories, char fieldDelimiter,
+ ITupleParserFactory parserFactory, List<Pair<LogicalVariable, Object>> varTypePairs) {
+ this.command = command;
+ this.printerFactories = printerFactories;
+ this.fieldDelimiter = fieldDelimiter;
+ this.parserFactory = parserFactory;
+ this.varTypePairs = varTypePairs;
+ }
+
+ @Override
+ public ScriptKind getKind() {
+ return ScriptKind.STRING_STREAMING;
+ }
+
+ public String getCommand() {
+ return command;
+ }
+
+ public IPrinterFactory[] getPrinterFactories() {
+ return printerFactories;
+ }
+
+ public char getFieldDelimiter() {
+ return fieldDelimiter;
+ }
+
+ public ITupleParserFactory getParserFactory() {
+ return parserFactory;
+ }
+
+ @Override
+ public List<Pair<LogicalVariable, Object>> getVarTypePairs() {
+ return varTypePairs;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
new file mode 100644
index 0000000..d1ebbab
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/AbstractTypeEnvironment.java
@@ -0,0 +1,45 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public abstract class AbstractTypeEnvironment implements IVariableTypeEnvironment {
+
+ protected final Map<LogicalVariable, Object> varTypeMap = new HashMap<LogicalVariable, Object>();
+ protected final IExpressionTypeComputer expressionTypeComputer;
+ protected final IMetadataProvider<?, ?> metadataProvider;
+
+ public AbstractTypeEnvironment(IExpressionTypeComputer expressionTypeComputer,
+ IMetadataProvider<?, ?> metadataProvider) {
+ this.expressionTypeComputer = expressionTypeComputer;
+ this.metadataProvider = metadataProvider;
+ }
+
+ @Override
+ public Object getType(ILogicalExpression expr) throws AlgebricksException {
+ return expressionTypeComputer.getType(expr, metadataProvider, this);
+ }
+
+ @Override
+ public void setVarType(LogicalVariable var, Object type) {
+ varTypeMap.put(var, type);
+ }
+
+ @Override
+ public boolean substituteProducedVariable(LogicalVariable v1, LogicalVariable v2) throws AlgebricksException {
+ Object t = varTypeMap.get(v1);
+ if (t == null) {
+ return false;
+ }
+ varTypeMap.put(v1, null);
+ varTypeMap.put(v2, t);
+ return true;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java
new file mode 100644
index 0000000..3c8d674
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypeEnvPointer.java
@@ -0,0 +1,7 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public interface ITypeEnvPointer {
+ public IVariableTypeEnvironment getTypeEnv();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypingContext.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypingContext.java
new file mode 100644
index 0000000..4996b48
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/ITypingContext.java
@@ -0,0 +1,19 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+
+public interface ITypingContext {
+ public abstract IVariableTypeEnvironment getOutputTypeEnvironment(ILogicalOperator op);
+
+ public abstract void setOutputTypeEnvironment(ILogicalOperator op, IVariableTypeEnvironment env);
+
+ public abstract IExpressionTypeComputer getExpressionTypeComputer();
+
+ public abstract INullableTypeComputer getNullableTypeComputer();
+
+ public abstract IMetadataProvider<?, ?> getMetadataProvider();
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
new file mode 100644
index 0000000..e0f4e16
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/NonPropagatingTypeEnvironment.java
@@ -0,0 +1,27 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class NonPropagatingTypeEnvironment extends AbstractTypeEnvironment {
+
+ public NonPropagatingTypeEnvironment(IExpressionTypeComputer expressionTypeComputer,
+ IMetadataProvider<?, ?> metadataProvider) {
+ super(expressionTypeComputer, metadataProvider);
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var) throws AlgebricksException {
+ return varTypeMap.get(var);
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariables) throws AlgebricksException {
+ return getVarType(var);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
new file mode 100644
index 0000000..417b900
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/OpRefTypeEnvPointer.java
@@ -0,0 +1,28 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+
+public class OpRefTypeEnvPointer implements ITypeEnvPointer {
+
+ private final Mutable<ILogicalOperator> op;
+ private final ITypingContext ctx;
+
+ public OpRefTypeEnvPointer(Mutable<ILogicalOperator> op, ITypingContext ctx) {
+ this.op = op;
+ this.ctx = ctx;
+ }
+
+ @Override
+ public IVariableTypeEnvironment getTypeEnv() {
+ return ctx.getOutputTypeEnvironment(op.getValue());
+ }
+
+ @Override
+ public String toString() {
+ return this.getClass().getName() + ":" + op;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
new file mode 100644
index 0000000..15504cb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagateOperatorInputsTypeEnvironment.java
@@ -0,0 +1,60 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class PropagateOperatorInputsTypeEnvironment extends AbstractTypeEnvironment {
+
+ private final List<LogicalVariable> nonNullVariables = new ArrayList<LogicalVariable>();
+ private final ILogicalOperator op;
+ private final ITypingContext ctx;
+
+ public PropagateOperatorInputsTypeEnvironment(ILogicalOperator op, ITypingContext ctx,
+ IExpressionTypeComputer expressionTypeComputer, IMetadataProvider<?, ?> metadataProvider) {
+ super(expressionTypeComputer, metadataProvider);
+ this.op = op;
+ this.ctx = ctx;
+ }
+
+ public List<LogicalVariable> getNonNullVariables() {
+ return nonNullVariables;
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariableList) throws AlgebricksException {
+ nonNullVariableList.addAll(nonNullVariables);
+ return getVarTypeFullList(var, nonNullVariableList);
+ }
+
+ private Object getVarTypeFullList(LogicalVariable var, List<LogicalVariable> nonNullVariableList)
+ throws AlgebricksException {
+ Object t = varTypeMap.get(var);
+ if (t != null) {
+ return t;
+ }
+ for (Mutable<ILogicalOperator> r : op.getInputs()) {
+ ILogicalOperator c = r.getValue();
+ IVariableTypeEnvironment env = ctx.getOutputTypeEnvironment(c);
+ Object t2 = env.getVarType(var, nonNullVariableList);
+ if (t2 != null) {
+ return t2;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var) throws AlgebricksException {
+ return getVarTypeFullList(var, nonNullVariables);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
new file mode 100644
index 0000000..bb2db54
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/typing/PropagatingTypeEnvironment.java
@@ -0,0 +1,55 @@
+package edu.uci.ics.hyracks.algebricks.core.algebra.typing;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.TypePropagationPolicy;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class PropagatingTypeEnvironment extends AbstractTypeEnvironment {
+
+ private final TypePropagationPolicy policy;
+
+ private final INullableTypeComputer nullableTypeComputer;
+
+ private final ITypeEnvPointer[] envPointers;
+
+ private final List<LogicalVariable> nonNullVariables = new ArrayList<LogicalVariable>();
+
+ public PropagatingTypeEnvironment(IExpressionTypeComputer expressionTypeComputer,
+ INullableTypeComputer nullableTypeComputer, IMetadataProvider<?, ?> metadataProvider,
+ TypePropagationPolicy policy, ITypeEnvPointer[] envPointers) {
+ super(expressionTypeComputer, metadataProvider);
+ this.nullableTypeComputer = nullableTypeComputer;
+ this.policy = policy;
+ this.envPointers = envPointers;
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var) throws AlgebricksException {
+ return getVarTypeFullList(var, nonNullVariables);
+ }
+
+ public List<LogicalVariable> getNonNullVariables() {
+ return nonNullVariables;
+ }
+
+ @Override
+ public Object getVarType(LogicalVariable var, List<LogicalVariable> nonNullVariableList) throws AlgebricksException {
+ nonNullVariableList.addAll(nonNullVariables);
+ return getVarTypeFullList(var, nonNullVariableList);
+ }
+
+ private Object getVarTypeFullList(LogicalVariable var, List<LogicalVariable> nonNullVariableList)
+ throws AlgebricksException {
+ Object t = varTypeMap.get(var);
+ if (t != null) {
+ return t;
+ }
+ return policy.getVarType(var, nullableTypeComputer, nonNullVariableList, envPointers);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
new file mode 100644
index 0000000..4604198
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorManipulationUtil.java
@@ -0,0 +1,161 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.util;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.typing.ITypingContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class OperatorManipulationUtil {
+
+ // Transforms all NestedTupleSource operators to EmptyTupleSource operators
+ public static void ntsToEts(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
+ context.computeAndSetTypeEnvironmentForOperator(ets);
+ opRef.setValue(ets);
+ } else {
+ for (Mutable<ILogicalOperator> i : opRef.getValue().getInputs()) {
+ ntsToEts(i, context);
+ }
+ }
+ }
+
+ public static ILogicalOperator eliminateSingleSubplanOverEts(SubplanOperator subplan) {
+ if (subplan.getNestedPlans().size() > 1) {
+ // not a single subplan
+ List<Mutable<ILogicalOperator>> subInpList = subplan.getInputs();
+ subInpList.clear();
+ subInpList.add(new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator()));
+ return subplan;
+ }
+ ILogicalPlan plan = subplan.getNestedPlans().get(0);
+ if (plan.getRoots().size() > 1) {
+ // not a single subplan
+ List<Mutable<ILogicalOperator>> subInpList = subplan.getInputs();
+ subInpList.clear();
+ subInpList.add(new MutableObject<ILogicalOperator>(new EmptyTupleSourceOperator()));
+ return subplan;
+ }
+ return plan.getRoots().get(0).getValue();
+ }
+
+ public static boolean setOperatorMode(AbstractLogicalOperator op) {
+ boolean change = false;
+ switch (op.getOperatorTag()) {
+ case DATASOURCESCAN: {
+ // ILogicalExpression e = ((UnnestOperator) op).getExpression();
+ // if (AnalysisUtil.isDataSetCall(e)) {
+ op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+ AbstractLogicalOperator child = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ if (child.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+ child.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+ }
+ change = true;
+ // }
+ break;
+ }
+ case NESTEDTUPLESOURCE: {
+ NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
+ AbstractLogicalOperator prevOp = (AbstractLogicalOperator) nts.getDataSourceReference().getValue()
+ .getInputs().get(0).getValue();
+ if (prevOp.getExecutionMode() != AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
+ nts.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
+ change = true;
+ }
+ break;
+ }
+ default: {
+ if (op.getOperatorTag() == LogicalOperatorTag.LIMIT) {
+ LimitOperator opLim = (LimitOperator) op;
+ if (opLim.isTopmostLimitOp()) {
+ if (opLim.getExecutionMode() != AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) {
+ opLim.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED);
+ change = true;
+ }
+ break;
+ }
+ }
+
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ boolean exit = false;
+ AbstractLogicalOperator inputOp = (AbstractLogicalOperator) i.getValue();
+ switch (inputOp.getExecutionMode()) {
+ case PARTITIONED: {
+ op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+ change = true;
+ exit = true;
+ break;
+ }
+ case LOCAL: {
+ op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
+ change = true;
+ break;
+ }
+ }
+ if (exit) {
+ break;
+ }
+ }
+ break;
+ }
+ }
+ return change;
+ }
+
+ public static void substituteVarRec(AbstractLogicalOperator op, LogicalVariable v1, LogicalVariable v2,
+ boolean goThroughNts, ITypingContext ctx) throws AlgebricksException {
+ VariableUtilities.substituteVariables(op, v1, v2, goThroughNts, ctx);
+ for (Mutable<ILogicalOperator> opRef2 : op.getInputs()) {
+ substituteVarRec((AbstractLogicalOperator) opRef2.getValue(), v1, v2, goThroughNts, ctx);
+ }
+ if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE && goThroughNts) {
+ NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
+ if (nts.getDataSourceReference() != null) {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) nts.getDataSourceReference().getValue()
+ .getInputs().get(0).getValue();
+ substituteVarRec(op2, v1, v2, goThroughNts, ctx);
+ }
+ }
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans aonp = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : aonp.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> ref : p.getRoots()) {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) ref.getValue();
+ substituteVarRec(aop, v1, v2, goThroughNts, ctx);
+ }
+ }
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
new file mode 100644
index 0000000..97bd8ca
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/util/OperatorPropertiesUtil.java
@@ -0,0 +1,211 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.util;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class OperatorPropertiesUtil {
+
+ public static <T> boolean disjoint(Collection<T> c1, Collection<T> c2) {
+ for (T m : c1) {
+ if (c2.contains(m)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ // Obs: doesn't return expected result for op. with nested plans.
+ private static void getFreeVariablesInOp(ILogicalOperator op, Set<LogicalVariable> freeVars)
+ throws AlgebricksException {
+ VariableUtilities.getUsedVariables(op, freeVars);
+ HashSet<LogicalVariable> produced = new HashSet<LogicalVariable>();
+ VariableUtilities.getProducedVariables(op, produced);
+ for (LogicalVariable v : produced) {
+ freeVars.remove(v);
+ }
+ }
+
+ /**
+ * Adds the free variables of the plan rooted at that operator to the
+ * collection provided.
+ *
+ * @param op
+ * @param vars
+ * - The collection to which the free variables will be added.
+ */
+ public static void getFreeVariablesInSelfOrDesc(AbstractLogicalOperator op, Set<LogicalVariable> freeVars)
+ throws AlgebricksException {
+ HashSet<LogicalVariable> produced = new HashSet<LogicalVariable>();
+ VariableUtilities.getProducedVariables(op, produced);
+ for (LogicalVariable v : produced) {
+ freeVars.remove(v);
+ }
+
+ HashSet<LogicalVariable> used = new HashSet<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op, used);
+ for (LogicalVariable v : used) {
+ if (!freeVars.contains(v)) {
+ freeVars.add(v);
+ }
+ }
+
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans s = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : s.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), freeVars);
+ }
+ }
+ s.getUsedVariablesExceptNestedPlans(freeVars);
+ HashSet<LogicalVariable> produced2 = new HashSet<LogicalVariable>();
+ s.getProducedVariablesExceptNestedPlans(produced2);
+ freeVars.removeAll(produced);
+ }
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) i.getValue(), freeVars);
+ }
+ }
+
+ public static void getFreeVariablesInSubplans(AbstractOperatorWithNestedPlans op, Set<LogicalVariable> freeVars)
+ throws AlgebricksException {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue(), freeVars);
+ }
+ }
+ }
+
+ public static boolean hasFreeVariablesInSelfOrDesc(AbstractLogicalOperator op) throws AlgebricksException {
+ HashSet<LogicalVariable> free = new HashSet<LogicalVariable>();
+ getFreeVariablesInSelfOrDesc(op, free);
+ return !free.isEmpty();
+ }
+
+ public static boolean hasFreeVariables(ILogicalOperator op) throws AlgebricksException {
+ HashSet<LogicalVariable> free = new HashSet<LogicalVariable>();
+ getFreeVariablesInOp(op, free);
+ return !free.isEmpty();
+ }
+
+ public static void computeSchemaAndPropertiesRecIfNull(AbstractLogicalOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ if (op.getSchema() == null) {
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) i.getValue(), context);
+ }
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : a.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) r.getValue(), context);
+ }
+ }
+ }
+ op.recomputeSchema();
+ op.computeDeliveredPhysicalProperties(context);
+ }
+ }
+
+ public static void computeSchemaRecIfNull(AbstractLogicalOperator op) throws AlgebricksException {
+ if (op.getSchema() == null) {
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ computeSchemaRecIfNull((AbstractLogicalOperator) i.getValue());
+ }
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : a.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ computeSchemaRecIfNull((AbstractLogicalOperator) r.getValue());
+ }
+ }
+ }
+ op.recomputeSchema();
+ }
+ }
+
+ public static boolean isNullTest(AbstractLogicalOperator op) {
+ if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+ return false;
+ }
+ AbstractLogicalOperator doubleUnder = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ if (doubleUnder.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ return false;
+ }
+ ILogicalExpression eu = ((SelectOperator) op).getCondition().getValue();
+ if (eu.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression f1 = (AbstractFunctionCallExpression) eu;
+ if (f1.getFunctionIdentifier() != AlgebricksBuiltinFunctions.NOT) {
+ return false;
+ }
+ ILogicalExpression a1 = f1.getArguments().get(0).getValue();
+ if (a1.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression f2 = (AbstractFunctionCallExpression) a1;
+ if (f2.getFunctionIdentifier() != AlgebricksBuiltinFunctions.IS_NULL) {
+ return false;
+ }
+ return true;
+ }
+
+ public static void typePlan(ILogicalPlan p, IOptimizationContext context) throws AlgebricksException {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ typeOpRec(r, context);
+ }
+ }
+
+ public static void typeOpRec(Mutable<ILogicalOperator> r, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) r.getValue();
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ typeOpRec(i, context);
+ }
+ if (op.hasNestedPlans()) {
+ for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
+ typePlan(p, context);
+ }
+ }
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ }
+
+ public static boolean isAlwaysTrueCond(ILogicalExpression cond) {
+ if (cond.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+ return ((ConstantExpression) cond).getValue().isTrue();
+ }
+ return false;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
new file mode 100644
index 0000000..d4c5530
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/AbstractConstVarFunVisitor.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public abstract class AbstractConstVarFunVisitor<R, T> implements ILogicalExpressionVisitor<R, T> {
+
+ public abstract R visitFunctionCallExpression(AbstractFunctionCallExpression expr, T arg)
+ throws AlgebricksException;
+
+ public R visitAggregateFunctionCallExpression(AggregateFunctionCallExpression expr, T arg)
+ throws AlgebricksException {
+ return visitFunctionCallExpression(expr, arg);
+ }
+
+ public R visitScalarFunctionCallExpression(ScalarFunctionCallExpression expr, T arg) throws AlgebricksException {
+ return visitFunctionCallExpression(expr, arg);
+ }
+
+ public R visitStatefulFunctionCallExpression(StatefulFunctionCallExpression expr, T arg) throws AlgebricksException {
+ return visitFunctionCallExpression(expr, arg);
+ }
+
+ public R visitUnnestingFunctionCallExpression(UnnestingFunctionCallExpression expr, T arg)
+ throws AlgebricksException {
+ return visitFunctionCallExpression(expr, arg);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
new file mode 100644
index 0000000..2ad942a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionReferenceTransform.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface ILogicalExpressionReferenceTransform {
+ // returns true if any change is made to the expression
+ public boolean transform(Mutable<ILogicalExpression> expression) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
new file mode 100644
index 0000000..a299574
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalExpressionVisitor.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface ILogicalExpressionVisitor<R, T> {
+ public R visitConstantExpression(ConstantExpression expr, T arg) throws AlgebricksException;
+
+ public R visitVariableReferenceExpression(VariableReferenceExpression expr, T arg) throws AlgebricksException;
+
+ public R visitAggregateFunctionCallExpression(AggregateFunctionCallExpression expr, T arg)
+ throws AlgebricksException;
+
+ public R visitScalarFunctionCallExpression(ScalarFunctionCallExpression expr, T arg) throws AlgebricksException;
+
+ public R visitStatefulFunctionCallExpression(StatefulFunctionCallExpression expr, T arg) throws AlgebricksException;
+
+ public R visitUnnestingFunctionCallExpression(UnnestingFunctionCallExpression expr, T arg)
+ throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
new file mode 100644
index 0000000..bfc7acc
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/algebra/visitors/ILogicalOperatorVisitor.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.algebra.visitors;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DieOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.PartitioningSplitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.RunningAggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ScriptOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SinkOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface ILogicalOperatorVisitor<R, T> {
+
+ public R visitAggregateOperator(AggregateOperator op, T arg) throws AlgebricksException;
+
+ public R visitRunningAggregateOperator(RunningAggregateOperator op, T arg) throws AlgebricksException;
+
+ public R visitEmptyTupleSourceOperator(EmptyTupleSourceOperator op, T arg) throws AlgebricksException;
+
+ public R visitGroupByOperator(GroupByOperator op, T arg) throws AlgebricksException;
+
+ public R visitLimitOperator(LimitOperator op, T arg) throws AlgebricksException;
+
+ public R visitInnerJoinOperator(InnerJoinOperator op, T arg) throws AlgebricksException;
+
+ public R visitLeftOuterJoinOperator(LeftOuterJoinOperator op, T arg) throws AlgebricksException;
+
+ public R visitNestedTupleSourceOperator(NestedTupleSourceOperator op, T arg) throws AlgebricksException;
+
+ public R visitOrderOperator(OrderOperator op, T arg) throws AlgebricksException;
+
+ public R visitAssignOperator(AssignOperator op, T arg) throws AlgebricksException;
+
+ public R visitSelectOperator(SelectOperator op, T arg) throws AlgebricksException;
+
+ public R visitProjectOperator(ProjectOperator op, T arg) throws AlgebricksException;
+
+ public R visitPartitioningSplitOperator(PartitioningSplitOperator op, T arg) throws AlgebricksException;
+
+ public R visitReplicateOperator(ReplicateOperator op, T arg) throws AlgebricksException;
+
+ public R visitScriptOperator(ScriptOperator op, T arg) throws AlgebricksException;
+
+ public R visitSubplanOperator(SubplanOperator op, T arg) throws AlgebricksException;
+
+ public R visitSinkOperator(SinkOperator op, T arg) throws AlgebricksException;
+
+ public R visitUnionOperator(UnionAllOperator op, T arg) throws AlgebricksException;
+
+ public R visitUnnestOperator(UnnestOperator op, T arg) throws AlgebricksException;
+
+ public R visitUnnestMapOperator(UnnestMapOperator op, T arg) throws AlgebricksException;
+
+ public R visitDataScanOperator(DataSourceScanOperator op, T arg) throws AlgebricksException;
+
+ public R visitDistinctOperator(DistinctOperator op, T arg) throws AlgebricksException;
+
+ public R visitExchangeOperator(ExchangeOperator op, T arg) throws AlgebricksException;
+
+ public R visitWriteOperator(WriteOperator op, T arg) throws AlgebricksException;
+
+ public R visitWriteResultOperator(WriteResultOperator op, T arg) throws AlgebricksException;
+
+ public R visitInsertDeleteOperator(InsertDeleteOperator op, T tag) throws AlgebricksException;
+
+ public R visitIndexInsertDeleteOperator(IndexInsertDeleteOperator op, T tag) throws AlgebricksException;
+
+ public R visitDieOperator(DieOperator op, T arg) throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksAbsolutePartitionConstraint.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksAbsolutePartitionConstraint.java
new file mode 100644
index 0000000..de5f778
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksAbsolutePartitionConstraint.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.api.constraints;
+
+public class AlgebricksAbsolutePartitionConstraint extends AlgebricksPartitionConstraint {
+ private final String[] locations;
+
+ public AlgebricksAbsolutePartitionConstraint(String[] locations) {
+ this.locations = locations;
+ }
+
+ @Override
+ public PartitionConstraintType getPartitionConstraintType() {
+ return PartitionConstraintType.ABSOLUTE;
+ }
+
+ public String[] getLocations() {
+ return locations;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksCountPartitionConstraint.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksCountPartitionConstraint.java
new file mode 100644
index 0000000..9cd8079
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksCountPartitionConstraint.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.api.constraints;
+
+public class AlgebricksCountPartitionConstraint extends AlgebricksPartitionConstraint {
+
+ private final int count;
+
+ public AlgebricksCountPartitionConstraint(int count) {
+ this.count = count;
+ }
+
+ @Override
+ public PartitionConstraintType getPartitionConstraintType() {
+ return PartitionConstraintType.COUNT;
+ }
+
+ public int getCount() {
+ return count;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraint.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraint.java
new file mode 100644
index 0000000..0a63b80
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraint.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.api.constraints;
+
+public abstract class AlgebricksPartitionConstraint {
+ public enum PartitionConstraintType {
+ ABSOLUTE,
+ COUNT
+ }
+
+ public abstract PartitionConstraintType getPartitionConstraintType();
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraintHelper.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraintHelper.java
new file mode 100644
index 0000000..e64602b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/constraints/AlgebricksPartitionConstraintHelper.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.api.constraints;
+
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class AlgebricksPartitionConstraintHelper {
+
+ public static void setPartitionConstraintInJobSpec(JobSpecification jobSpec, IOperatorDescriptor opDesc,
+ AlgebricksPartitionConstraint apc) {
+ switch (apc.getPartitionConstraintType()) {
+ case ABSOLUTE: {
+ AlgebricksAbsolutePartitionConstraint absPc = (AlgebricksAbsolutePartitionConstraint) apc;
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(jobSpec, opDesc, absPc.getLocations());
+ break;
+ }
+ case COUNT: {
+ AlgebricksCountPartitionConstraint cntPc = (AlgebricksCountPartitionConstraint) apc;
+ PartitionConstraintHelper.addPartitionCountConstraint(jobSpec, opDesc, cntPc.getCount());
+ break;
+ }
+ default: {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ public static int getPartitionCount(AlgebricksPartitionConstraint partitionConstraint) {
+ switch (partitionConstraint.getPartitionConstraintType()) {
+ case COUNT: {
+ AlgebricksCountPartitionConstraint pcc = (AlgebricksCountPartitionConstraint) partitionConstraint;
+ return pcc.getCount();
+ }
+ case ABSOLUTE: {
+ AlgebricksAbsolutePartitionConstraint epc = (AlgebricksAbsolutePartitionConstraint) partitionConstraint;
+ return epc.getLocations().length;
+ }
+ default: {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/AlgebricksException.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/AlgebricksException.java
new file mode 100644
index 0000000..c852ecd
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/AlgebricksException.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.api.exceptions;
+
+public class AlgebricksException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public AlgebricksException() {
+ }
+
+ public AlgebricksException(String message) {
+ super(message);
+ }
+
+ public AlgebricksException(Throwable cause) {
+ super(cause);
+ }
+
+ public AlgebricksException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/NotImplementedException.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/NotImplementedException.java
new file mode 100644
index 0000000..f5343a8
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/api/exceptions/NotImplementedException.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.api.exceptions;
+
+public class NotImplementedException extends RuntimeException {
+ private static final long serialVersionUID = 2L;
+
+ public NotImplementedException() {
+ System.err.println("Not implemented.");
+ }
+
+ public NotImplementedException(String message) {
+ super(message);
+ }
+
+ public NotImplementedException(Throwable cause) {
+ super(cause);
+ }
+
+ public NotImplementedException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java
new file mode 100644
index 0000000..397f51b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.config;
+
+import java.util.logging.Logger;
+
+public class AlgebricksConfig {
+ public static final boolean DEBUG = true;
+ public static final String ALGEBRICKS_LOGGER_NAME = "edu.uci.ics.algebricks";
+ public static final Logger ALGEBRICKS_LOGGER = Logger.getLogger(ALGEBRICKS_LOGGER_NAME);
+ public static final String HYRACKS_APP_NAME = "algebricks";
+
+ // public static final Level ALGEBRICKS_LOG_LEVEL = Level.FINEST;
+ //
+ // static {
+ // Handler h;
+ // try {
+ // h = new ConsoleHandler();
+ // h.setFormatter(new SysoutFormatter());
+ // } catch (Exception e) {
+ // h = new ConsoleHandler();
+ // }
+ // h.setLevel(ALGEBRICKS_LOG_LEVEL);
+ // ALGEBRICKS_LOGGER.addHandler(h);
+ // ALGEBRICKS_LOGGER.setLevel(ALGEBRICKS_LOG_LEVEL);
+ // }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/SysoutFormatter.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/SysoutFormatter.java
new file mode 100644
index 0000000..872676b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/config/SysoutFormatter.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.config;
+
+import java.util.logging.Formatter;
+import java.util.logging.Handler;
+import java.util.logging.LogRecord;
+
+public class SysoutFormatter extends Formatter {
+ public String format(LogRecord rec) {
+ StringBuffer buf = new StringBuffer(5000);
+ buf.append(formatMessage(rec));
+ return buf.toString();
+ }
+
+ public String getHead(Handler h) {
+ return "";
+ }
+
+ public String getTail(Handler h) {
+ return "";
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
new file mode 100644
index 0000000..1293d5b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import java.util.Collection;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public abstract class AbstractRuleController {
+
+ protected IOptimizationContext context;
+ private LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
+
+ public AbstractRuleController() {
+ }
+
+ public void setContext(IOptimizationContext context) {
+ this.context = context;
+ }
+
+ /**
+ * Each rewriting strategy may differ in the
+ *
+ * @param root
+ * @param ruleClasses
+ * @return true iff one of the rules in the collection fired
+ */
+ public abstract boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root,
+ Collection<IAlgebraicRewriteRule> rules) throws AlgebricksException;
+
+ /**
+ * @param opRef
+ * @param rule
+ * @return true if any rewrite was fired, either on opRef or any operator
+ * under it.
+ */
+ protected boolean rewriteOperatorRef(Mutable<ILogicalOperator> opRef, IAlgebraicRewriteRule rule)
+ throws AlgebricksException {
+ return rewriteOperatorRef(opRef, rule, true, false);
+ }
+
+ private void printRuleApplication(IAlgebraicRewriteRule rule, Mutable<ILogicalOperator> opRef)
+ throws AlgebricksException {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Rule " + rule.getClass() + " fired.\n");
+ StringBuilder sb = new StringBuilder();
+ PlanPrettyPrinter.printOperator((AbstractLogicalOperator) opRef.getValue(), sb, pvisitor, 0);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(sb.toString());
+ }
+
+ protected boolean rewriteOperatorRef(Mutable<ILogicalOperator> opRef, IAlgebraicRewriteRule rule,
+ boolean enterNestedPlans, boolean fullDFS) throws AlgebricksException {
+
+ if (rule.rewritePre(opRef, context)) {
+ printRuleApplication(rule, opRef);
+ return true;
+ }
+ boolean rewritten = false;
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+ for (Mutable<ILogicalOperator> inp : op.getInputs()) {
+ if (rewriteOperatorRef(inp, rule, enterNestedPlans, fullDFS)) {
+ rewritten = true;
+ if (!fullDFS) {
+ break;
+ }
+ }
+ }
+
+ if (op.hasNestedPlans() && enterNestedPlans) {
+ AbstractOperatorWithNestedPlans o2 = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : o2.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ if (rewriteOperatorRef(r, rule, enterNestedPlans, fullDFS)) {
+ rewritten = true;
+ if (!fullDFS) {
+ break;
+ }
+ }
+ }
+ if (rewritten && !fullDFS) {
+ break;
+ }
+ }
+ }
+
+ if (rule.rewritePost(opRef, context)) {
+ printRuleApplication(rule, opRef);
+ return true;
+ }
+
+ return rewritten;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
new file mode 100644
index 0000000..ab436f4
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/AlgebricksOptimizationContext.java
@@ -0,0 +1,285 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableEvalSizeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class AlgebricksOptimizationContext implements IOptimizationContext {
+
+ private int varCounter;
+ private final IExpressionEvalSizeComputer expressionEvalSizeComputer;
+ private final IMergeAggregationExpressionFactory mergeAggregationExpressionFactory;
+ private final PhysicalOptimizationConfig physicalOptimizationConfig;
+ private final IVariableEvalSizeEnvironment varEvalSizeEnv = new IVariableEvalSizeEnvironment() {
+
+ Map<LogicalVariable, Integer> varSizeMap = new HashMap<LogicalVariable, Integer>();
+
+ @Override
+ public void setVariableEvalSize(LogicalVariable var, int size) {
+ varSizeMap.put(var, size);
+ }
+
+ @Override
+ public int getVariableEvalSize(LogicalVariable var) {
+ return varSizeMap.get(var);
+ }
+ };
+
+ private Map<ILogicalOperator, IVariableTypeEnvironment> typeEnvMap = new HashMap<ILogicalOperator, IVariableTypeEnvironment>();
+
+ private Map<ILogicalOperator, HashSet<ILogicalOperator>> alreadyCompared = new HashMap<ILogicalOperator, HashSet<ILogicalOperator>>();
+ private Map<IAlgebraicRewriteRule, HashSet<ILogicalOperator>> dontApply = new HashMap<IAlgebraicRewriteRule, HashSet<ILogicalOperator>>();
+ private Map<LogicalVariable, FunctionalDependency> recordToPrimaryKey = new HashMap<LogicalVariable, FunctionalDependency>();
+
+ @SuppressWarnings("unchecked")
+ private IMetadataProvider metadataProvider;
+ private HashSet<LogicalVariable> notToBeInlinedVars = new HashSet<LogicalVariable>();
+
+ protected final Map<ILogicalOperator, List<FunctionalDependency>> fdGlobalMap = new HashMap<ILogicalOperator, List<FunctionalDependency>>();
+ protected final Map<ILogicalOperator, Map<LogicalVariable, EquivalenceClass>> eqClassGlobalMap = new HashMap<ILogicalOperator, Map<LogicalVariable, EquivalenceClass>>();
+
+ protected final Map<ILogicalOperator, ILogicalPropertiesVector> logicalProps = new HashMap<ILogicalOperator, ILogicalPropertiesVector>();
+ private final int frameSize;
+ private final IExpressionTypeComputer expressionTypeComputer;
+ private final INullableTypeComputer nullableTypeComputer;
+
+ public AlgebricksOptimizationContext(int varCounter, int frameSize,
+ IExpressionEvalSizeComputer expressionEvalSizeComputer,
+ IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
+ IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
+ PhysicalOptimizationConfig physicalOptimizationConfig) {
+ this.varCounter = varCounter;
+ this.frameSize = frameSize;
+ this.expressionEvalSizeComputer = expressionEvalSizeComputer;
+ this.mergeAggregationExpressionFactory = mergeAggregationExpressionFactory;
+ this.expressionTypeComputer = expressionTypeComputer;
+ this.nullableTypeComputer = nullableTypeComputer;
+ this.physicalOptimizationConfig = physicalOptimizationConfig;
+ }
+
+ public int getVarCounter() {
+ return varCounter;
+ }
+
+ public void setVarCounter(int varCounter) {
+ this.varCounter = varCounter;
+ }
+
+ public LogicalVariable newVar() {
+ varCounter++;
+ LogicalVariable var = new LogicalVariable(varCounter);
+ return var;
+ }
+
+ @SuppressWarnings("unchecked")
+ public IMetadataProvider getMetadataProvider() {
+ return metadataProvider;
+ }
+
+ public void setMetadataDeclarations(IMetadataProvider<?, ?> metadataProvider) {
+ this.metadataProvider = metadataProvider;
+ }
+
+ public boolean checkIfInDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op) {
+ HashSet<ILogicalOperator> operators = dontApply.get(rule);
+ if (operators == null) {
+ return false;
+ } else {
+ return operators.contains(op);
+ }
+ }
+
+ public void addToDontApplySet(IAlgebraicRewriteRule rule, ILogicalOperator op) {
+ HashSet<ILogicalOperator> operators = dontApply.get(rule);
+ if (operators == null) {
+ HashSet<ILogicalOperator> os = new HashSet<ILogicalOperator>();
+ os.add(op);
+ dontApply.put(rule, os);
+ } else {
+ operators.add(op);
+ }
+
+ }
+
+ /*
+ * returns true if op1 and op2 have already been compared
+ */
+ public boolean checkAndAddToAlreadyCompared(ILogicalOperator op1, ILogicalOperator op2) {
+ HashSet<ILogicalOperator> ops = alreadyCompared.get(op1);
+ if (ops == null) {
+ HashSet<ILogicalOperator> newEntry = new HashSet<ILogicalOperator>();
+ newEntry.add(op2);
+ alreadyCompared.put(op1, newEntry);
+ return false;
+ } else {
+ if (ops.contains(op2)) {
+ return true;
+ } else {
+ ops.add(op2);
+ return false;
+ }
+ }
+ }
+
+ public void addNotToBeInlinedVar(LogicalVariable var) {
+ notToBeInlinedVars.add(var);
+ }
+
+ public boolean shouldNotBeInlined(LogicalVariable var) {
+ return notToBeInlinedVars.contains(var);
+ }
+
+ public void addPrimaryKey(FunctionalDependency pk) {
+ assert (pk.getTail().size() == 1);
+ LogicalVariable recordVar = pk.getTail().get(0);
+ recordToPrimaryKey.put(recordVar, pk);
+ }
+
+ public List<LogicalVariable> findPrimaryKey(LogicalVariable recordVar) {
+ FunctionalDependency fd = recordToPrimaryKey.get(recordVar);
+ if (fd == null) {
+ return null;
+ }
+ return fd.getHead();
+ }
+
+ @Override
+ public Map<LogicalVariable, EquivalenceClass> getEquivalenceClassMap(ILogicalOperator op) {
+ return eqClassGlobalMap.get(op);
+ }
+
+ @Override
+ public List<FunctionalDependency> getFDList(ILogicalOperator op) {
+ return fdGlobalMap.get(op);
+ }
+
+ @Override
+ public void putEquivalenceClassMap(ILogicalOperator op, Map<LogicalVariable, EquivalenceClass> eqClassMap) {
+ this.eqClassGlobalMap.put(op, eqClassMap);
+ }
+
+ @Override
+ public void putFDList(ILogicalOperator op, List<FunctionalDependency> fdList) {
+ this.fdGlobalMap.put(op, fdList);
+ }
+
+ @Override
+ public ILogicalPropertiesVector getLogicalPropertiesVector(ILogicalOperator op) {
+ return logicalProps.get(op);
+ }
+
+ @Override
+ public void putLogicalPropertiesVector(ILogicalOperator op, ILogicalPropertiesVector v) {
+ logicalProps.put(op, v);
+ }
+
+ @Override
+ public IExpressionEvalSizeComputer getExpressionEvalSizeComputer() {
+ return expressionEvalSizeComputer;
+ }
+
+ @Override
+ public IVariableEvalSizeEnvironment getVariableEvalSizeEnvironment() {
+ return varEvalSizeEnv;
+ }
+
+ @Override
+ public int getFrameSize() {
+ return frameSize;
+ }
+
+ public IMergeAggregationExpressionFactory getMergeAggregationExpressionFactory() {
+ return mergeAggregationExpressionFactory;
+ }
+
+ public PhysicalOptimizationConfig getPhysicalOptimizationConfig() {
+ return physicalOptimizationConfig;
+ }
+
+ @Override
+ public IVariableTypeEnvironment getOutputTypeEnvironment(ILogicalOperator op) {
+ return typeEnvMap.get(op);
+ }
+
+ @Override
+ public void setOutputTypeEnvironment(ILogicalOperator op, IVariableTypeEnvironment env) {
+ typeEnvMap.put(op, env);
+ }
+
+ @Override
+ public IExpressionTypeComputer getExpressionTypeComputer() {
+ return expressionTypeComputer;
+ }
+
+ @Override
+ public INullableTypeComputer getNullableTypeComputer() {
+ return nullableTypeComputer;
+ }
+
+ @Override
+ public void invalidateTypeEnvironmentForOperator(ILogicalOperator op) {
+ typeEnvMap.put(op, null);
+ }
+
+ @Override
+ public void computeAndSetTypeEnvironmentForOperator(ILogicalOperator op) throws AlgebricksException {
+ setOutputTypeEnvironment(op, op.computeOutputTypeEnvironment(this));
+ }
+
+ @Override
+ public void updatePrimaryKeys(Map<LogicalVariable, LogicalVariable> mappedVars) {
+ for (Map.Entry<LogicalVariable, FunctionalDependency> me : recordToPrimaryKey.entrySet()) {
+ FunctionalDependency fd = me.getValue();
+ List<LogicalVariable> hd = new ArrayList<LogicalVariable>();
+ for (LogicalVariable v : fd.getHead()) {
+ LogicalVariable v2 = mappedVars.get(v);
+ if (v2 == null) {
+ hd.add(v);
+ } else {
+ hd.add(v2);
+ }
+ }
+ List<LogicalVariable> tl = new ArrayList<LogicalVariable>();
+ for (LogicalVariable v : fd.getTail()) {
+ LogicalVariable v2 = mappedVars.get(v);
+ if (v2 == null) {
+ tl.add(v);
+ } else {
+ tl.add(v2);
+ }
+ }
+ me.setValue(new FunctionalDependency(hd, tl));
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
new file mode 100644
index 0000000..381f70c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
@@ -0,0 +1,113 @@
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class HeuristicOptimizer {
+
+ public static PhysicalOperatorTag[] hyraxOperators = new PhysicalOperatorTag[] {
+ PhysicalOperatorTag.DATASOURCE_SCAN, PhysicalOperatorTag.BTREE_SEARCH,
+ PhysicalOperatorTag.EXTERNAL_GROUP_BY, PhysicalOperatorTag.HASH_GROUP_BY, PhysicalOperatorTag.HDFS_READER,
+ PhysicalOperatorTag.HYBRID_HASH_JOIN, PhysicalOperatorTag.IN_MEMORY_HASH_JOIN,
+ PhysicalOperatorTag.NESTED_LOOP, PhysicalOperatorTag.PRE_SORTED_DISTINCT_BY,
+ PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY, PhysicalOperatorTag.SPLIT, PhysicalOperatorTag.STABLE_SORT,
+ PhysicalOperatorTag.UNION_ALL };
+ public static PhysicalOperatorTag[] hyraxOperatorsBelowWhichJobGenIsDisabled = new PhysicalOperatorTag[] {};
+
+ public static boolean isHyraxOp(PhysicalOperatorTag opTag) {
+ for (PhysicalOperatorTag t : hyraxOperators) {
+ if (t == opTag) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private IOptimizationContext context;
+ private List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites;
+ private List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites;
+ private ILogicalPlan plan;
+ private LogicalOperatorPrettyPrintVisitor ppvisitor = new LogicalOperatorPrettyPrintVisitor();
+
+ public HeuristicOptimizer(ILogicalPlan plan,
+ List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> logicalRewrites,
+ List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites,
+ IOptimizationContext context) {
+ this.plan = plan;
+ this.context = context;
+ this.logicalRewrites = logicalRewrites;
+ this.physicalRewrites = physicalRewrites;
+ }
+
+ public void optimize() throws AlgebricksException {
+ if (plan == null) {
+ return;
+ }
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting logical optimizations.\n");
+ }
+
+ StringBuilder sb = new StringBuilder();
+ PlanPrettyPrinter.printPlan(plan, sb, ppvisitor, 0);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Logical Plan:\n" + sb.toString());
+ runOptimizationSets(plan, logicalRewrites);
+ computeSchemaBottomUpForPlan(plan);
+ runPhysicalOptimizations(plan, physicalRewrites);
+ StringBuilder sb2 = new StringBuilder();
+ PlanPrettyPrinter.printPlan(plan, sb2, ppvisitor, 0);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Optimized Plan:\n" + sb2.toString());
+ }
+
+ private void runOptimizationSets(ILogicalPlan plan,
+ List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> optimSet) throws AlgebricksException {
+ for (Pair<AbstractRuleController, List<IAlgebraicRewriteRule>> ruleList : optimSet) {
+ for (Mutable<ILogicalOperator> r : plan.getRoots()) {
+ ruleList.first.setContext(context);
+ ruleList.first.rewriteWithRuleCollection(r, ruleList.second);
+ }
+ }
+ }
+
+ private static void computeSchemaBottomUpForPlan(ILogicalPlan p) throws AlgebricksException {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ computeSchemaBottomUpForOp((AbstractLogicalOperator) r.getValue());
+ }
+ }
+
+ private static void computeSchemaBottomUpForOp(AbstractLogicalOperator op) throws AlgebricksException {
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ computeSchemaBottomUpForOp((AbstractLogicalOperator) i.getValue());
+ }
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : a.getNestedPlans()) {
+ computeSchemaBottomUpForPlan(p);
+ }
+ }
+ op.recomputeSchema();
+ }
+
+ private void runPhysicalOptimizations(ILogicalPlan plan,
+ List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> physicalRewrites)
+ throws AlgebricksException {
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Starting physical optimizations.\n");
+ }
+ // PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(plan);
+ runOptimizationSets(plan, physicalRewrites);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
new file mode 100644
index 0000000..bfbd636
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IAlgebraicRewriteRule.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public interface IAlgebraicRewriteRule {
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException;
+
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException;
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java
new file mode 100644
index 0000000..1373fa7
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/IOptimizationContextFactory.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.INullableTypeComputer;
+
+public interface IOptimizationContextFactory {
+ public IOptimizationContext createOptimizationContext(int varCounter, int frameSize,
+ IExpressionEvalSizeComputer expressionEvalSizeComputer,
+ IMergeAggregationExpressionFactory mergeAggregationExpressionFactory,
+ IExpressionTypeComputer expressionTypeComputer, INullableTypeComputer nullableTypeComputer,
+ PhysicalOptimizationConfig physicalOptimizationConfig);
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
new file mode 100644
index 0000000..9ce910b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java
@@ -0,0 +1,91 @@
+package edu.uci.ics.hyracks.algebricks.core.rewriter.base;
+
+import java.util.Properties;
+
+public class PhysicalOptimizationConfig {
+ private static final int MB = 1048576;
+ private static final String FRAMESIZE = "FRAMESIZE";
+ private static final String MAX_FRAMES_EXTERNAL_SORT = "MAX_FRAMES_EXTERNAL_SORT";
+ private static final String MAX_FRAMES_EXTERNAL_GROUP_BY = "MAX_FRAMES_EXTERNAL_GROUP_BY";
+
+ private static final String DEFAULT_HASH_GROUP_TABLE_SIZE = "DEFAULT_HASH_GROUP_TABLE_SIZE";
+ private static final String DEFAULT_EXTERNAL_GROUP_TABLE_SIZE = "DEFAULT_EXTERNAL_GROUP_TABLE_SIZE";
+ private static final String DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE = "DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE";
+
+ private Properties properties = new Properties();
+
+ public PhysicalOptimizationConfig() {
+ int frameSize = 32768;
+ setInt(FRAMESIZE, frameSize);
+ setInt(MAX_FRAMES_EXTERNAL_SORT, (int) (((long) 512 * MB) / frameSize));
+ setInt(MAX_FRAMES_EXTERNAL_GROUP_BY, (int) (((long) 256 * MB) / frameSize));
+
+ // use http://www.rsok.com/~jrm/printprimes.html to find prime numbers
+ setInt(DEFAULT_HASH_GROUP_TABLE_SIZE, 10485767);
+ setInt(DEFAULT_EXTERNAL_GROUP_TABLE_SIZE, 10485767);
+ setInt(DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE, 10485767);
+ }
+
+ public int getFrameSize() {
+ return getInt(FRAMESIZE, 32768);
+ }
+
+ public void setFrameSize(int frameSize) {
+ setInt(FRAMESIZE, frameSize);
+ }
+
+ public int getMaxFramesExternalSort() {
+ int frameSize = getFrameSize();
+ return getInt(MAX_FRAMES_EXTERNAL_SORT, (int) (((long) 512 * MB) / frameSize));
+ }
+
+ public void setMaxFramesExternalSort(int frameLimit) {
+ setInt(MAX_FRAMES_EXTERNAL_SORT, frameLimit);
+ }
+
+ public int getMaxFramesExternalGroupBy() {
+ int frameSize = getFrameSize();
+ return getInt(MAX_FRAMES_EXTERNAL_GROUP_BY, (int) (((long) 256 * MB) / frameSize));
+ }
+
+ public void setMaxFramesExternalGroupBy(int frameLimit) {
+ setInt(MAX_FRAMES_EXTERNAL_GROUP_BY, frameLimit);
+ }
+
+ public int getHashGroupByTableSize() {
+ return getInt(DEFAULT_HASH_GROUP_TABLE_SIZE, 10485767);
+ }
+
+ public void setHashGroupByTableSize(int tableSize) {
+ setInt(DEFAULT_HASH_GROUP_TABLE_SIZE, tableSize);
+ }
+
+ public int getExternalGroupByTableSize() {
+ return getInt(DEFAULT_EXTERNAL_GROUP_TABLE_SIZE, 10485767);
+ }
+
+ public void setExternalGroupByTableSize(int tableSize) {
+ setInt(DEFAULT_EXTERNAL_GROUP_TABLE_SIZE, tableSize);
+ }
+
+ public int getInMemHashJoinTableSize() {
+ return getInt(DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE, 10485767);
+ }
+
+ public void setInMemHashJoinTableSize(int tableSize) {
+ setInt(DEFAULT_IN_MEM_HASH_JOIN_TABLE_SIZE, tableSize);
+ }
+
+ private void setInt(String property, int value) {
+ properties.setProperty(property, Integer.toString(value));
+ }
+
+ private int getInt(String property, int defaultValue) {
+ String value = properties.getProperty(property);
+ if (value == null)
+ return defaultValue;
+ else
+ return Integer.parseInt(value);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Pair.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Pair.java
new file mode 100644
index 0000000..d3aff9f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Pair.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.utils;
+
+import java.io.Serializable;
+
+public class Pair<T1, T2> implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+ public T1 first;
+ public T2 second;
+
+ public Pair(T1 first, T2 second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ @Override
+ public String toString() {
+ return first + "," + second;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (!(obj instanceof Pair<?, ?>)) {
+ return false;
+ } else {
+ Pair<?, ?> p = (Pair<?, ?>) obj;
+ return this.first.equals(p.first) && this.second.equals(p.second);
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return first.hashCode() * 31 + second.hashCode();
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Substitution.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Substitution.java
new file mode 100644
index 0000000..c52cdff
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Substitution.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.utils;
+
+public class Substitution<T> {
+ public T substituted;
+ public T substitutedWith;
+
+ public Substitution(T substituted, T substitutedWith) {
+ this.substituted = substituted;
+ this.substitutedWith = substitutedWith;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Triple.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Triple.java
new file mode 100644
index 0000000..ad76ef1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/Triple.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.core.utils;
+
+public class Triple<T1, T2, T3> {
+ public T1 first;
+ public T2 second;
+ public T3 third;
+
+ public Triple(T1 first, T2 second, T3 third) {
+ this.first = first;
+ this.second = second;
+ this.third = third;
+ }
+
+ @Override
+ public String toString() {
+ return first + "," + second + ", " + third;
+ }
+
+ @Override
+ public int hashCode() {
+ return first.hashCode() * 31 + second.hashCode() * 15 + third.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof Triple<?, ?, ?>))
+ return false;
+ Triple<?, ?, ?> triple = (Triple<?, ?, ?>) o;
+ return first.equals(triple.first) && second.equals(triple.second) && third.equals(triple.third);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/WriteValueTools.java b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/WriteValueTools.java
new file mode 100644
index 0000000..8ecd16b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-core/src/main/java/edu/uci/ics/hyracks/algebricks/core/utils/WriteValueTools.java
@@ -0,0 +1,106 @@
+package edu.uci.ics.hyracks.algebricks.core.utils;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+
+public final class WriteValueTools {
+
+ private final static int[] INT_INTERVALS = { 9, 99, 999, 9999, 99999, 999999, 9999999, 99999999, 999999999,
+ Integer.MAX_VALUE };
+ private final static int[] INT_DIVIDERS = { 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000 };
+ private final static int[] DIGITS = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9' };
+
+ public static void writeInt(int i, OutputStream os) throws IOException {
+ if (i < 0) {
+ if (i == Integer.MIN_VALUE) {
+ os.write("-2147483648".getBytes());
+ return;
+ }
+ os.write('-');
+ i = -i;
+ }
+ int k = 0;
+ for (; k < INT_INTERVALS.length; k++) {
+ if (i <= INT_INTERVALS[k]) {
+ break;
+ }
+ }
+ while (k > 0) {
+ int q = i / INT_DIVIDERS[k - 1];
+ os.write(DIGITS[q % 10]);
+ k--;
+ }
+ // now, print the units
+ os.write(DIGITS[i % 10]);
+ }
+
+ public static void writeLong(long d, OutputStream os) throws IOException {
+ // now, print the units
+ if (d < 0) {
+ if (d == Long.MIN_VALUE) {
+ os.write("-9223372036854775808".getBytes());
+ return;
+ }
+ os.write('-');
+ d = -d;
+ }
+ long divisor = 1000000000000000000L;
+ while (divisor > d) {
+ divisor = divisor / 10;
+ }
+ while (divisor > 1) {
+ os.write(DIGITS[(int) ((d / divisor) % 10)]);
+ divisor = divisor / 10;
+ }
+ os.write(DIGITS[(int) (d % 10)]);
+ }
+
+ public static void writeUTF8String(byte[] b, int s, int l, OutputStream os) throws IOException {
+ int stringLength = UTF8StringPointable.getUTFLen(b, s);
+ int position = s + 2;
+ int maxPosition = position + stringLength;
+ os.write('\"');
+ while (position < maxPosition) {
+ char c = UTF8StringPointable.charAt(b, position);
+ switch (c) {
+ // escape
+ case '\\':
+ case '"':
+ os.write('\\');
+ break;
+ }
+ int sz = UTF8StringPointable.charSize(b, position);
+ while (sz > 0) {
+ os.write(b[position]);
+ position++;
+ sz--;
+ }
+ }
+ os.write('\"');
+ }
+
+ public static void writeUTF8StringNoQuotes(byte[] b, int s, int l, OutputStream os) throws IOException {
+ int stringLength = UTF8StringPointable.getUTFLen(b, s);
+ int position = s + 2;
+ int maxPosition = position + stringLength;
+ while (position < maxPosition) {
+ char c = UTF8StringPointable.charAt(b, position);
+ switch (c) {
+ // escape
+ case '\\':
+ case '"':
+ os.write('\\');
+ break;
+ }
+ int sz = UTF8StringPointable.charSize(b, position);
+ while (sz > 0) {
+ os.write(b[position]);
+ position++;
+ sz--;
+ }
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/pom.xml b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/pom.xml
new file mode 100644
index 0000000..92c4f5f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/pom.xml
@@ -0,0 +1,65 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>piglet-example</artifactId>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-examples</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>javacc-maven-plugin</artifactId>
+ <version>2.6</version>
+ <executions>
+ <execution>
+ <id>javacc</id>
+ <goals>
+ <goal>javacc</goal>
+ </goals>
+ <configuration>
+ <isStatic>false</isStatic>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.7.2</version>
+ <configuration>
+ <includes>
+ <include>**/*TestSuite.java</include>
+ <include>**/*Test.java</include>
+ </includes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-compiler</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.2</version>
+ <type>jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ASTNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ASTNode.java
new file mode 100644
index 0000000..d94fb9e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ASTNode.java
@@ -0,0 +1,16 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public abstract class ASTNode {
+ public enum Tag {
+ ASSIGNMENT,
+ DUMP,
+ LOAD,
+ FILTER,
+
+ SCALAR_FUNCTION,
+ LITERAL,
+ FIELD_ACCESS,
+ }
+
+ public abstract Tag getTag();
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java
new file mode 100644
index 0000000..7fa8c6c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/AssignmentNode.java
@@ -0,0 +1,25 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public class AssignmentNode extends ASTNode {
+ private String alias;
+
+ private RelationNode relation;
+
+ public AssignmentNode(String alias, RelationNode relation) {
+ this.alias = alias;
+ this.relation = relation;
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.ASSIGNMENT;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public RelationNode getRelation() {
+ return relation;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/DumpNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/DumpNode.java
new file mode 100644
index 0000000..5e8bc76
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/DumpNode.java
@@ -0,0 +1,24 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public class DumpNode extends RelationNode {
+ private final String file;
+ private final String alias;
+
+ public DumpNode(String file, String alias) {
+ this.file = file;
+ this.alias = alias;
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.DUMP;
+ }
+
+ public String getFile() {
+ return file;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java
new file mode 100644
index 0000000..3f68ae6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ExpressionNode.java
@@ -0,0 +1,4 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public abstract class ExpressionNode extends ASTNode {
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java
new file mode 100644
index 0000000..69c9262
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FieldAccessExpressionNode.java
@@ -0,0 +1,25 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public class FieldAccessExpressionNode extends ExpressionNode {
+ private String relationName;
+
+ private String fieldName;
+
+ public FieldAccessExpressionNode(String relationName, String fieldName) {
+ this.relationName = relationName;
+ this.fieldName = fieldName;
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.FIELD_ACCESS;
+ }
+
+ public String getRelationName() {
+ return relationName;
+ }
+
+ public String getFieldName() {
+ return fieldName;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FilterNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FilterNode.java
new file mode 100644
index 0000000..bc90043
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FilterNode.java
@@ -0,0 +1,25 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public class FilterNode extends RelationNode {
+ private String alias;
+
+ private ExpressionNode expression;
+
+ public FilterNode(String alias, ExpressionNode expression) {
+ this.alias = alias;
+ this.expression = expression;
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.FILTER;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public ExpressionNode getExpression() {
+ return expression;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FunctionTag.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FunctionTag.java
new file mode 100644
index 0000000..2ab397a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/FunctionTag.java
@@ -0,0 +1,20 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public enum FunctionTag {
+ BOOLEAN_AND,
+ BOOLEAN_OR,
+ BOOLEAN_NOT,
+
+ EQ,
+ NEQ,
+ LT,
+ LTE,
+ GT,
+ GTE,
+
+ ADD,
+ SUBTRACT,
+ MULTIPLY,
+ DIVIDE,
+ MOD,
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java
new file mode 100644
index 0000000..547d487
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LiteralExpressionNode.java
@@ -0,0 +1,27 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+
+public class LiteralExpressionNode extends ExpressionNode {
+ private String image;
+
+ private Type type;
+
+ public LiteralExpressionNode(String image, Type type) {
+ this.image = image;
+ this.type = type;
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.LITERAL;
+ }
+
+ public String getImage() {
+ return image;
+ }
+
+ public Type getType() {
+ return type;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LoadNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LoadNode.java
new file mode 100644
index 0000000..e5da760
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/LoadNode.java
@@ -0,0 +1,27 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Schema;
+
+public class LoadNode extends RelationNode {
+ private String dataFile;
+
+ private Schema schema;
+
+ public LoadNode(String dataFile, Schema schema) {
+ this.dataFile = dataFile;
+ this.schema = schema;
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.LOAD;
+ }
+
+ public String getDataFile() {
+ return dataFile;
+ }
+
+ public Schema getSchema() {
+ return schema;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/RelationNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/RelationNode.java
new file mode 100644
index 0000000..51e275f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/RelationNode.java
@@ -0,0 +1,4 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+public abstract class RelationNode extends ASTNode {
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java
new file mode 100644
index 0000000..9b55ae9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/ast/ScalarFunctionExpressionNode.java
@@ -0,0 +1,34 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.ast;
+
+import java.util.List;
+
+public class ScalarFunctionExpressionNode extends ExpressionNode {
+ private FunctionTag fTag;
+
+ private String fName;
+
+ private List<ASTNode> arguments;
+
+ public ScalarFunctionExpressionNode(FunctionTag fTag, String fName, List<ASTNode> arguments) {
+ this.fTag = fTag;
+ this.fName = fName;
+ this.arguments = arguments;
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.SCALAR_FUNCTION;
+ }
+
+ public FunctionTag getFunctionTag() {
+ return fTag;
+ }
+
+ public String getFunctionName() {
+ return fName;
+ }
+
+ public List<ASTNode> getArguments() {
+ return arguments;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java
new file mode 100644
index 0000000..e6226f4
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/ConstantValue.java
@@ -0,0 +1,38 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.compiler;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+
+public final class ConstantValue implements IAlgebricksConstantValue {
+ private final Type type;
+
+ private final String image;
+
+ public ConstantValue(Type type, String image) {
+ this.type = type;
+ this.image = image;
+ }
+
+ public Type getType() {
+ return type;
+ }
+
+ public String getImage() {
+ return image;
+ }
+
+ @Override
+ public boolean isFalse() {
+ return false;
+ }
+
+ @Override
+ public boolean isNull() {
+ return false;
+ }
+
+ @Override
+ public boolean isTrue() {
+ return false;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
new file mode 100644
index 0000000..07f6f5d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletCompiler.java
@@ -0,0 +1,353 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.compiler;
+
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.compiler.api.HeuristicCompilerFactoryBuilder;
+import edu.uci.ics.hyracks.algebricks.compiler.api.ICompiler;
+import edu.uci.ics.hyracks.algebricks.compiler.api.ICompilerFactory;
+import edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialFixpointRuleController;
+import edu.uci.ics.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialOnceRuleController;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.ISerializerDeserializerProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionTypeComputer;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.AbstractRuleController;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ASTNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.AssignmentNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.DumpNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ExpressionNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.FieldAccessExpressionNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.FilterNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.FunctionTag;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.LiteralExpressionNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.LoadNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.RelationNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ScalarFunctionExpressionNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.metadata.PigletFileDataSink;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.metadata.PigletFileDataSource;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.metadata.PigletMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.parser.ParseException;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.parser.PigletParser;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.rewriter.PigletRewriteRuleset;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.PigletExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Schema;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class PigletCompiler {
+ private static final Logger LOGGER = Logger.getLogger(PigletCompiler.class.getName());
+
+ private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultLogicalRewrites() {
+ List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>();
+ SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false);
+ SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true);
+ SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true);
+ defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrl,
+ PigletRewriteRuleset.buildTypeInferenceRuleCollection()));
+ defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlFullDfs,
+ PigletRewriteRuleset.buildNormalizationRuleCollection()));
+ defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
+ PigletRewriteRuleset.buildCondPushDownRuleCollection()));
+ defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
+ PigletRewriteRuleset.buildJoinInferenceRuleCollection()));
+ defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
+ PigletRewriteRuleset.buildOpPushDownRuleCollection()));
+ defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrl,
+ PigletRewriteRuleset.buildDataExchangeRuleCollection()));
+ defaultLogicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqCtrlNoDfs,
+ PigletRewriteRuleset.buildConsolidationRuleCollection()));
+ return defaultLogicalRewrites;
+ }
+
+ private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultPhysicalRewrites() {
+ List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultPhysicalRewrites = new ArrayList<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>>();
+ SequentialOnceRuleController seqOnceCtrlAllLevels = new SequentialOnceRuleController(true);
+ SequentialOnceRuleController seqOnceCtrlTopLevel = new SequentialOnceRuleController(false);
+ defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlAllLevels,
+ PigletRewriteRuleset.buildPhysicalRewritesAllLevelsRuleCollection()));
+ defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlTopLevel,
+ PigletRewriteRuleset.buildPhysicalRewritesTopLevelRuleCollection()));
+ defaultPhysicalRewrites.add(new Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>(seqOnceCtrlAllLevels,
+ PigletRewriteRuleset.prepareForJobGenRuleCollection()));
+ return defaultPhysicalRewrites;
+ }
+
+ private final ICompilerFactory cFactory;
+
+ private final PigletMetadataProvider metadataProvider;
+
+ private int varCounter;
+
+ private ILogicalOperator previousOp;
+
+ public PigletCompiler() {
+ HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder();
+ builder.setLogicalRewrites(buildDefaultLogicalRewrites());
+ builder.setPhysicalRewrites(buildDefaultPhysicalRewrites());
+ builder.setSerializerDeserializerProvider(new ISerializerDeserializerProvider() {
+ @SuppressWarnings("unchecked")
+ @Override
+ public ISerializerDeserializer getSerializerDeserializer(Object type) throws AlgebricksException {
+ return null;
+ }
+ });
+ builder.setPrinterProvider(PigletPrinterFactoryProvider.INSTANCE);
+ builder.setExprJobGen(new PigletExpressionJobGen());
+ builder.setExpressionTypeComputer(new IExpressionTypeComputer() {
+ @Override
+ public Object getType(ILogicalExpression expr, IMetadataProvider<?, ?> metadataProvider,
+ IVariableTypeEnvironment env) throws AlgebricksException {
+ return null;
+ }
+ });
+ cFactory = builder.create();
+ metadataProvider = new PigletMetadataProvider();
+ }
+
+ public List<ASTNode> parse(Reader in) throws ParseException {
+ PigletParser parser = new PigletParser(in);
+ List<ASTNode> statements = parser.Statements();
+ return statements;
+ }
+
+ public JobSpecification compile(List<ASTNode> ast) throws AlgebricksException, PigletException {
+ ILogicalPlan plan = translate(ast);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Translated Plan:");
+ LOGGER.info(getPrettyPrintedPlan(plan));
+ }
+ ICompiler compiler = cFactory.createCompiler(plan, metadataProvider, varCounter);
+ compiler.optimize();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Optimized Plan:");
+ LOGGER.info(getPrettyPrintedPlan(plan));
+ }
+ return compiler.createJob(null);
+ }
+
+ private ILogicalPlan translate(List<ASTNode> ast) throws PigletException {
+ Map<String, Relation> symMap = new HashMap<String, Relation>();
+ List<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>();
+ previousOp = null;
+ for (ASTNode an : ast) {
+ switch (an.getTag()) {
+ case DUMP: {
+ DumpNode dn = (DumpNode) an;
+ Relation input = symMap.get(dn.getAlias());
+ List<Mutable<ILogicalExpression>> expressions = new ArrayList<Mutable<ILogicalExpression>>();
+ for (LogicalVariable v : input.schema.values()) {
+ expressions.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(v)));
+ }
+ PigletFileDataSink dataSink = new PigletFileDataSink(dn.getFile());
+ ILogicalOperator op = new WriteOperator(expressions, dataSink);
+ op.getInputs().add(new MutableObject<ILogicalOperator>(input.op));
+ roots.add(new MutableObject<ILogicalOperator>(op));
+ }
+ break;
+
+ case ASSIGNMENT: {
+ AssignmentNode asn = (AssignmentNode) an;
+ String alias = asn.getAlias();
+ RelationNode rn = asn.getRelation();
+ Relation rel = translate(rn, symMap);
+ previousOp = rel.op;
+ rel.alias = alias;
+ symMap.put(alias, rel);
+ }
+ break;
+ }
+ }
+ return new ALogicalPlanImpl(roots);
+ }
+
+ private Relation translate(RelationNode rn, Map<String, Relation> symMap) throws PigletException {
+ switch (rn.getTag()) {
+ case LOAD: {
+ LoadNode ln = (LoadNode) rn;
+ String file = ln.getDataFile();
+ Schema schema = ln.getSchema();
+ List<Pair<String, Type>> fieldsSchema = schema.getSchema();
+ List<LogicalVariable> variables = new ArrayList<LogicalVariable>();
+ List<Object> types = new ArrayList<Object>();
+ Relation rel = new Relation();
+ for (Pair<String, Type> p : fieldsSchema) {
+ LogicalVariable v = newVariable();
+ rel.schema.put(p.first, v);
+ variables.add(v);
+ types.add(p.second);
+ }
+ PigletFileDataSource ds = new PigletFileDataSource(file, types.toArray());
+ rel.op = new DataSourceScanOperator(variables, ds);
+ rel.op.getInputs().add(
+ new MutableObject<ILogicalOperator>(previousOp == null ? new EmptyTupleSourceOperator()
+ : previousOp));
+ return rel;
+ }
+
+ case FILTER: {
+ FilterNode fn = (FilterNode) rn;
+ String alias = fn.getAlias();
+ ExpressionNode conditionNode = fn.getExpression();
+ Relation inputRel = findInputRelation(alias, symMap);
+ Pair<Relation, LogicalVariable> tempInput = translateScalarExpression(inputRel, conditionNode);
+ Relation rel = new Relation();
+ rel.op = new SelectOperator(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(
+ tempInput.second)));
+ rel.op.getInputs().add(new MutableObject<ILogicalOperator>(tempInput.first.op));
+ rel.schema.putAll(tempInput.first.schema);
+ return rel;
+ }
+ }
+ throw new IllegalArgumentException("Unknown node: " + rn.getTag() + " encountered");
+ }
+
+ private Pair<Relation, LogicalVariable> translateScalarExpression(Relation inputRel, ExpressionNode expressionNode)
+ throws PigletException {
+ switch (expressionNode.getTag()) {
+ case FIELD_ACCESS: {
+ FieldAccessExpressionNode faen = (FieldAccessExpressionNode) expressionNode;
+ String fieldName = faen.getFieldName();
+ LogicalVariable lVar = findField(fieldName, inputRel.schema);
+ return new Pair<Relation, LogicalVariable>(inputRel, lVar);
+ }
+
+ case LITERAL: {
+ LiteralExpressionNode len = (LiteralExpressionNode) expressionNode;
+ String image = len.getImage();
+ Type type = len.getType();
+ ConstantExpression ce = new ConstantExpression(new ConstantValue(type, image));
+ Relation rel = new Relation();
+ LogicalVariable var = newVariable();
+ List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+ vars.add(var);
+
+ List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
+ exprs.add(new MutableObject<ILogicalExpression>(ce));
+
+ rel.op = new AssignOperator(vars, exprs);
+ rel.op.getInputs().add(new MutableObject<ILogicalOperator>(inputRel.op));
+ rel.schema.putAll(inputRel.schema);
+
+ return new Pair<Relation, LogicalVariable>(rel, var);
+ }
+
+ case SCALAR_FUNCTION: {
+ ScalarFunctionExpressionNode sfen = (ScalarFunctionExpressionNode) expressionNode;
+ List<Mutable<ILogicalExpression>> argExprs = new ArrayList<Mutable<ILogicalExpression>>();
+ List<ASTNode> arguments = sfen.getArguments();
+ Relation rel = inputRel;
+ for (ASTNode a : arguments) {
+ Pair<Relation, LogicalVariable> argPair = translateScalarExpression(rel, (ExpressionNode) a);
+ rel = argPair.first;
+ argExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(argPair.second)));
+ }
+ Relation outRel = new Relation();
+ outRel.schema.putAll(rel.schema);
+ LogicalVariable var = newVariable();
+ List<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+ vars.add(var);
+
+ IFunctionInfo fInfo = lookupFunction(sfen.getFunctionTag(), sfen.getFunctionName());
+
+ List<Mutable<ILogicalExpression>> exprs = new ArrayList<Mutable<ILogicalExpression>>();
+ exprs.add(new MutableObject<ILogicalExpression>(new ScalarFunctionCallExpression(fInfo, argExprs)));
+ outRel.op = new AssignOperator(vars, exprs);
+ outRel.op.getInputs().add(new MutableObject<ILogicalOperator>(rel.op));
+ return new Pair<Relation, LogicalVariable>(outRel, var);
+ }
+ }
+ return null;
+ }
+
+ private IFunctionInfo lookupFunction(FunctionTag functionTag, String functionName) throws PigletException {
+ switch (functionTag) {
+ case EQ:
+ return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.EQ);
+
+ case NEQ:
+ return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.NEQ);
+
+ case LT:
+ return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.LT);
+
+ case LTE:
+ return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.LE);
+
+ case GT:
+ return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.GT);
+
+ case GTE:
+ return AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.GE);
+ }
+ throw new PigletException("Unsupported function: " + functionTag);
+ }
+
+ private LogicalVariable newVariable() {
+ return new LogicalVariable(varCounter++);
+ }
+
+ private LogicalVariable findField(String fieldName, Map<String, LogicalVariable> schema) throws PigletException {
+ LogicalVariable var = schema.get(fieldName);
+ if (var == null) {
+ throw new PigletException("Unable to find field named: " + fieldName);
+ }
+ return var;
+ }
+
+ private Relation findInputRelation(String alias, Map<String, Relation> symMap) throws PigletException {
+ Relation rel = symMap.get(alias);
+ if (rel == null) {
+ throw new PigletException("Unknown alias " + alias + "referenced");
+ }
+ return rel;
+ }
+
+ private static class Relation {
+ String alias;
+ ILogicalOperator op;
+ final Map<String, LogicalVariable> schema;
+
+ public Relation() {
+ schema = new LinkedHashMap<String, LogicalVariable>();
+ }
+ }
+
+ private String getPrettyPrintedPlan(ILogicalPlan plan) throws AlgebricksException {
+ LogicalOperatorPrettyPrintVisitor v = new LogicalOperatorPrettyPrintVisitor();
+ StringBuilder buffer = new StringBuilder();
+ PlanPrettyPrinter.printPlan(plan, buffer, v, 0);
+ return buffer.toString();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
new file mode 100644
index 0000000..cb12b7d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/compiler/PigletPrinterFactoryProvider.java
@@ -0,0 +1,90 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.compiler;
+
+import java.io.IOException;
+import java.io.PrintStream;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactoryProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.IntegerPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.WriteValueTools;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+
+public class PigletPrinterFactoryProvider implements IPrinterFactoryProvider {
+
+ public static final PigletPrinterFactoryProvider INSTANCE = new PigletPrinterFactoryProvider();
+
+ private PigletPrinterFactoryProvider() {
+ }
+
+ @Override
+ public IPrinterFactory getPrinterFactory(Object type) throws AlgebricksException {
+ Type t = (Type) type;
+ switch (t.getTag()) {
+ case INTEGER:
+ return IntegerPrinterFactory.INSTANCE;
+ case CHAR_ARRAY:
+ return CharArrayPrinterFactory.INSTANCE;
+ case FLOAT:
+ return FloatPrinterFactory.INSTANCE;
+ default:
+ throw new UnsupportedOperationException();
+
+ }
+ }
+
+ public static class CharArrayPrinterFactory implements IPrinterFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final CharArrayPrinterFactory INSTANCE = new CharArrayPrinterFactory();
+
+ private CharArrayPrinterFactory() {
+ }
+
+ @Override
+ public IPrinter createPrinter() {
+ return new IPrinter() {
+ @Override
+ public void init() throws AlgebricksException {
+ }
+
+ @Override
+ public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+ try {
+ WriteValueTools.writeUTF8String(b, s, l, ps);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+ };
+ }
+ }
+
+ public static class FloatPrinterFactory implements IPrinterFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final FloatPrinterFactory INSTANCE = new FloatPrinterFactory();
+
+ private FloatPrinterFactory() {
+ }
+
+ @Override
+ public IPrinter createPrinter() {
+ return new IPrinter() {
+ @Override
+ public void init() throws AlgebricksException {
+ }
+
+ @Override
+ public void print(byte[] b, int s, int l, PrintStream ps) throws AlgebricksException {
+ ps.print(FloatSerializerDeserializer.getFloat(b, s));
+ }
+ };
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/exceptions/PigletException.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/exceptions/PigletException.java
new file mode 100644
index 0000000..d196b70
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/exceptions/PigletException.java
@@ -0,0 +1,9 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions;
+
+public class PigletException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public PigletException(String message) {
+ super(message);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
new file mode 100644
index 0000000..d0f9b44
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/FileSplitUtils.java
@@ -0,0 +1,22 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+
+import java.io.File;
+
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+
+public class FileSplitUtils {
+ public static FileSplit[] parseFileSplits(String fileSplits) {
+ String[] splits = fileSplits.split(",");
+ FileSplit[] fSplits = new FileSplit[splits.length];
+ for (int i = 0; i < splits.length; ++i) {
+ String s = splits[i].trim();
+ int idx = s.indexOf(':');
+ if (idx < 0) {
+ throw new IllegalArgumentException("File split " + s + " not well formed");
+ }
+ fSplits[i] = new FileSplit(s.substring(0, idx), new FileReference(new File(s.substring(idx + 1))));
+ }
+ return fSplits;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
new file mode 100644
index 0000000..93f9cae
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSink.java
@@ -0,0 +1,40 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FileSplitDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+
+public class PigletFileDataSink implements IDataSink {
+ private String file;
+
+ private FileSplit[] fileSplits;
+
+ private IPartitioningProperty partProp;
+
+ public PigletFileDataSink(String file) {
+ this.file = file;
+ fileSplits = FileSplitUtils.parseFileSplits(file);
+ partProp = new RandomPartitioningProperty(new FileSplitDomain(fileSplits));
+ }
+
+ @Override
+ public Object getId() {
+ return file;
+ }
+
+ public FileSplit[] getFileSplits() {
+ return fileSplits;
+ }
+
+ @Override
+ public Object[] getSchemaTypes() {
+ return null;
+ }
+
+ @Override
+ public IPartitioningProperty getPartitioningProperty() {
+ return partProp;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
new file mode 100644
index 0000000..6763d45
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletFileDataSource.java
@@ -0,0 +1,62 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourcePropertiesProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FileSplitDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+
+public class PigletFileDataSource implements IDataSource<String> {
+ private final String file;
+
+ private final Object[] types;
+
+ private final FileSplit[] fileSplits;
+
+ private IDataSourcePropertiesProvider propProvider;
+
+ public PigletFileDataSource(String file, Object[] types) {
+ this.file = file;
+ this.types = types;
+ fileSplits = FileSplitUtils.parseFileSplits(file);
+ final IPhysicalPropertiesVector vec = new StructuralPropertiesVector(new RandomPartitioningProperty(
+ new FileSplitDomain(fileSplits)), new ArrayList<ILocalStructuralProperty>());
+ propProvider = new IDataSourcePropertiesProvider() {
+ @Override
+ public IPhysicalPropertiesVector computePropertiesVector(List<LogicalVariable> scanVariables) {
+ return vec;
+ }
+ };
+ }
+
+ @Override
+ public String getId() {
+ return file;
+ }
+
+ @Override
+ public Object[] getSchemaTypes() {
+ return types;
+ }
+
+ public FileSplit[] getFileSplits() {
+ return fileSplits;
+ }
+
+ @Override
+ public IDataSourcePropertiesProvider getPropertiesProvider() {
+ return propProvider;
+ }
+
+ @Override
+ public void computeFDs(List<LogicalVariable> scanVariables, List<FunctionalDependency> fdList) {
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
new file mode 100644
index 0000000..668d7a2
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/metadata/PigletMetadataProvider.java
@@ -0,0 +1,167 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.metadata;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSink;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSourceIndex;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.SinkWriterRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers.PrinterBasedWriterFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksAbsolutePartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.FloatParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
+
+public class PigletMetadataProvider implements IMetadataProvider<String, String> {
+ @Override
+ public IDataSource<String> findDataSource(String id) throws AlgebricksException {
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getScannerRuntime(IDataSource<String> dataSource,
+ List<LogicalVariable> scanVariables, List<LogicalVariable> projectVariables, boolean projectPushed,
+ JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ PigletFileDataSource ds = (PigletFileDataSource) dataSource;
+
+ FileSplit[] fileSplits = ds.getFileSplits();
+ String[] locations = new String[fileSplits.length];
+ for (int i = 0; i < fileSplits.length; ++i) {
+ locations[i] = fileSplits[i].getNodeName();
+ }
+ IFileSplitProvider fsp = new ConstantFileSplitProvider(fileSplits);
+
+ Object[] colTypes = ds.getSchemaTypes();
+ IValueParserFactory[] vpfs = new IValueParserFactory[colTypes.length];
+ ISerializerDeserializer[] serDesers = new ISerializerDeserializer[colTypes.length];
+
+ for (int i = 0; i < colTypes.length; ++i) {
+ Type colType = (Type) colTypes[i];
+ IValueParserFactory vpf;
+ ISerializerDeserializer serDeser;
+ switch (colType.getTag()) {
+ case INTEGER:
+ vpf = IntegerParserFactory.INSTANCE;
+ serDeser = IntegerSerializerDeserializer.INSTANCE;
+ break;
+
+ case CHAR_ARRAY:
+ vpf = UTF8StringParserFactory.INSTANCE;
+ serDeser = UTF8StringSerializerDeserializer.INSTANCE;
+ break;
+
+ case FLOAT:
+ vpf = FloatParserFactory.INSTANCE;
+ serDeser = FloatSerializerDeserializer.INSTANCE;
+ break;
+
+ default:
+ throw new UnsupportedOperationException();
+ }
+ vpfs[i] = vpf;
+ serDesers[i] = serDeser;
+ }
+
+ ITupleParserFactory tpf = new DelimitedDataTupleParserFactory(vpfs, ',');
+ RecordDescriptor rDesc = new RecordDescriptor(serDesers);
+
+ IOperatorDescriptor scanner = new FileScanOperatorDescriptor(jobSpec, fsp, tpf, rDesc);
+ AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
+ return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(scanner, constraint);
+ }
+
+ @Override
+ public boolean scannerOperatorIsLeaf(IDataSource<String> dataSource) {
+ return true;
+ }
+
+ @Override
+ public Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint> getWriteFileRuntime(IDataSink sink,
+ int[] printColumns, IPrinterFactory[] printerFactories, RecordDescriptor inputDesc)
+ throws AlgebricksException {
+ PigletFileDataSink ds = (PigletFileDataSink) sink;
+ FileSplit[] fileSplits = ds.getFileSplits();
+ String[] locations = new String[fileSplits.length];
+ for (int i = 0; i < fileSplits.length; ++i) {
+ locations[i] = fileSplits[i].getNodeName();
+ }
+ IPushRuntimeFactory prf = new SinkWriterRuntimeFactory(printColumns, printerFactories, fileSplits[0]
+ .getLocalFile().getFile(), PrinterBasedWriterFactory.INSTANCE, inputDesc);
+ AlgebricksAbsolutePartitionConstraint constraint = new AlgebricksAbsolutePartitionConstraint(locations);
+ return new Pair<IPushRuntimeFactory, AlgebricksPartitionConstraint>(prf, constraint);
+ }
+
+ @Override
+ public IDataSourceIndex<String, String> findDataSourceIndex(String indexId, String dataSourceId)
+ throws AlgebricksException {
+ return null;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getWriteResultRuntime(
+ IDataSource<String> dataSource, IOperatorSchema propagatedSchema, List<LogicalVariable> keys,
+ LogicalVariable payLoadVar, JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getInsertRuntime(IDataSource<String> dataSource,
+ IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
+ RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<String> dataSource,
+ IOperatorSchema propagatedSchema, List<LogicalVariable> keys, LogicalVariable payLoadVar,
+ RecordDescriptor recordDesc, JobGenContext context, JobSpecification jobSpec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexInsertRuntime(
+ IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema,
+ List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification spec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getIndexDeleteRuntime(
+ IDataSourceIndex<String, String> dataSource, IOperatorSchema propagatedSchema,
+ List<LogicalVariable> primaryKeys, List<LogicalVariable> secondaryKeys, RecordDescriptor recordDesc,
+ JobGenContext context, JobSpecification spec) throws AlgebricksException {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
new file mode 100644
index 0000000..90ee5bb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/rewriter/PigletRewriteRuleset.java
@@ -0,0 +1,116 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.rewriter;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.HeuristicOptimizer;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.BreakSelectIntoConjunctsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ComplexJoinInferenceRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ConsolidateAssignsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ConsolidateSelectsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.EliminateSubplanRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.EnforceStructuralPropertiesRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ExtractCommonOperatorsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ExtractGbyExpressionsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.FactorRedundantGroupAndDecorVarsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.InferTypesRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.InlineVariablesRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.IntroduceGroupByForStandaloneAggregRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.IsolateHyracksOperatorsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PullSelectOutOfEqJoin;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushLimitDownRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushProjectDownRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushProjectIntoDataSourceScanRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushSelectDownRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.PushSelectIntoJoinRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.ReinferAllTypesRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.RemoveUnusedAssignAndAggregateRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.SetAlgebricksPhysicalOperatorsRule;
+import edu.uci.ics.hyracks.algebricks.rewriter.rules.SetExecutionModeRule;
+
+public class PigletRewriteRuleset {
+
+ public final static List<IAlgebraicRewriteRule> buildTypeInferenceRuleCollection() {
+ List<IAlgebraicRewriteRule> typeInfer = new LinkedList<IAlgebraicRewriteRule>();
+ typeInfer.add(new InferTypesRule());
+ return typeInfer;
+ }
+
+ public final static List<IAlgebraicRewriteRule> buildNormalizationRuleCollection() {
+ List<IAlgebraicRewriteRule> normalization = new LinkedList<IAlgebraicRewriteRule>();
+ normalization.add(new EliminateSubplanRule());
+ normalization.add(new IntroduceGroupByForStandaloneAggregRule());
+ normalization.add(new BreakSelectIntoConjunctsRule());
+ normalization.add(new PushSelectIntoJoinRule());
+ normalization.add(new ExtractGbyExpressionsRule());
+ return normalization;
+ }
+
+ public final static List<IAlgebraicRewriteRule> buildCondPushDownRuleCollection() {
+ List<IAlgebraicRewriteRule> condPushDown = new LinkedList<IAlgebraicRewriteRule>();
+ condPushDown.add(new PushSelectDownRule());
+ condPushDown.add(new InlineVariablesRule());
+ condPushDown.add(new FactorRedundantGroupAndDecorVarsRule());
+ condPushDown.add(new EliminateSubplanRule());
+ return condPushDown;
+ }
+
+ public final static List<IAlgebraicRewriteRule> buildJoinInferenceRuleCollection() {
+ List<IAlgebraicRewriteRule> joinInference = new LinkedList<IAlgebraicRewriteRule>();
+ joinInference.add(new InlineVariablesRule());
+ joinInference.add(new ComplexJoinInferenceRule());
+ return joinInference;
+ }
+
+ public final static List<IAlgebraicRewriteRule> buildOpPushDownRuleCollection() {
+ List<IAlgebraicRewriteRule> opPushDown = new LinkedList<IAlgebraicRewriteRule>();
+ opPushDown.add(new PushProjectDownRule());
+ opPushDown.add(new PushSelectDownRule());
+ return opPushDown;
+ }
+
+ public final static List<IAlgebraicRewriteRule> buildDataExchangeRuleCollection() {
+ List<IAlgebraicRewriteRule> dataExchange = new LinkedList<IAlgebraicRewriteRule>();
+ dataExchange.add(new SetExecutionModeRule());
+ return dataExchange;
+ }
+
+ public final static List<IAlgebraicRewriteRule> buildConsolidationRuleCollection() {
+ List<IAlgebraicRewriteRule> consolidation = new LinkedList<IAlgebraicRewriteRule>();
+ consolidation.add(new ConsolidateSelectsRule());
+ consolidation.add(new ConsolidateAssignsRule());
+ consolidation.add(new RemoveUnusedAssignAndAggregateRule());
+ return consolidation;
+ }
+
+ public final static List<IAlgebraicRewriteRule> buildPhysicalRewritesAllLevelsRuleCollection() {
+ List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<IAlgebraicRewriteRule>();
+ physicalPlanRewrites.add(new PullSelectOutOfEqJoin());
+ physicalPlanRewrites.add(new SetAlgebricksPhysicalOperatorsRule());
+ physicalPlanRewrites.add(new EnforceStructuralPropertiesRule());
+ physicalPlanRewrites.add(new PushProjectDownRule());
+ physicalPlanRewrites.add(new PushLimitDownRule());
+ return physicalPlanRewrites;
+ }
+
+ public final static List<IAlgebraicRewriteRule> buildPhysicalRewritesTopLevelRuleCollection() {
+ List<IAlgebraicRewriteRule> physicalPlanRewrites = new LinkedList<IAlgebraicRewriteRule>();
+ physicalPlanRewrites.add(new PushLimitDownRule());
+ return physicalPlanRewrites;
+ }
+
+
+ public final static List<IAlgebraicRewriteRule> prepareForJobGenRuleCollection() {
+ List<IAlgebraicRewriteRule> prepareForJobGenRewrites = new LinkedList<IAlgebraicRewriteRule>();
+ prepareForJobGenRewrites.add(new IsolateHyracksOperatorsRule(
+ HeuristicOptimizer.hyraxOperatorsBelowWhichJobGenIsDisabled));
+ prepareForJobGenRewrites.add(new ExtractCommonOperatorsRule());
+ // Re-infer all types, so that, e.g., the effect of not-is-null is
+ // propagated.
+ prepareForJobGenRewrites.add(new PushProjectIntoDataSourceScanRule());
+ prepareForJobGenRewrites.add(new ReinferAllTypesRule());
+ return prepareForJobGenRewrites;
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
new file mode 100644
index 0000000..63e6f53
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/PigletExpressionJobGen.java
@@ -0,0 +1,126 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime;
+
+import java.io.DataOutput;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ILogicalExpressionJobGen;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.ISerializableAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators.ColumnAccessEvalFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators.ConstantEvalFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.impl.JobGenContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.compiler.ConstantValue;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions.PigletFunctionRegistry;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.Type;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+
+public class PigletExpressionJobGen implements ILogicalExpressionJobGen {
+ @Override
+ public IEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env,
+ IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException {
+ switch (expr.getExpressionTag()) {
+ case CONSTANT: {
+ ConstantValue cv = (ConstantValue) ((ConstantExpression) expr).getValue();
+ Type type = cv.getType();
+ String image = cv.getImage();
+ ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ DataOutput dos = abvs.getDataOutput();
+ switch (type.getTag()) {
+ case INTEGER:
+ try {
+ IntegerSerializerDeserializer.INSTANCE.serialize(Integer.valueOf(image), dos);
+ } catch (Exception e) {
+ throw new AlgebricksException(e);
+ }
+ break;
+
+ case CHAR_ARRAY:
+ try {
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(image, dos);
+ } catch (Exception e) {
+ throw new AlgebricksException(e);
+ }
+ break;
+
+ default:
+ throw new UnsupportedOperationException("Unsupported constant type: " + type.getTag());
+ }
+ return new ConstantEvalFactory(Arrays.copyOf(abvs.getBytes(), abvs.getLength()));
+ }
+
+ case FUNCTION_CALL: {
+ ScalarFunctionCallExpression sfce = (ScalarFunctionCallExpression) expr;
+
+ List<Mutable<ILogicalExpression>> argExprs = sfce.getArguments();
+ IEvaluatorFactory argEvalFactories[] = new IEvaluatorFactory[argExprs.size()];
+ for (int i = 0; i < argEvalFactories.length; ++i) {
+ Mutable<ILogicalExpression> er = argExprs.get(i);
+ argEvalFactories[i] = createEvaluatorFactory(er.getValue(), env, inputSchemas, context);
+ }
+ IEvaluatorFactory funcEvalFactory;
+ try {
+ funcEvalFactory = PigletFunctionRegistry.createFunctionEvaluatorFactory(
+ sfce.getFunctionIdentifier(), argEvalFactories);
+ } catch (PigletException e) {
+ throw new AlgebricksException(e);
+ }
+ return funcEvalFactory;
+ }
+
+ case VARIABLE: {
+ LogicalVariable var = ((VariableReferenceExpression) expr).getVariableReference();
+ int index = inputSchemas[0].findVariable(var);
+ return new ColumnAccessEvalFactory(index);
+ }
+ }
+ throw new IllegalArgumentException("Unknown expression type: " + expr.getExpressionTag());
+ }
+
+ @Override
+ public IAggregateFunctionFactory createAggregateFunctionFactory(AggregateFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ISerializableAggregateFunctionFactory createSerializableAggregateFunctionFactory(
+ AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas,
+ JobGenContext context) throws AlgebricksException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IRunningAggregateFunctionFactory createRunningAggregateFunctionFactory(StatefulFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public IUnnestingFunctionFactory createUnnestingFunctionFactory(UnnestingFunctionCallExpression expr,
+ IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context)
+ throws AlgebricksException {
+ throw new UnsupportedOperationException();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
new file mode 100644
index 0000000..38e7615
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IPigletFunctionEvaluatorFactoryBuilder.java
@@ -0,0 +1,8 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+
+public interface IPigletFunctionEvaluatorFactoryBuilder {
+ public IEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, IEvaluatorFactory[] arguments);
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
new file mode 100644
index 0000000..d654d77
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/IntegerEqFunctionEvaluatorFactory.java
@@ -0,0 +1,52 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+public class IntegerEqFunctionEvaluatorFactory implements IEvaluatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final IEvaluatorFactory arg1Factory;
+
+ private final IEvaluatorFactory arg2Factory;
+
+ public IntegerEqFunctionEvaluatorFactory(IEvaluatorFactory arg1Factory, IEvaluatorFactory arg2Factory) {
+ this.arg1Factory = arg1Factory;
+ this.arg2Factory = arg2Factory;
+ }
+
+ @Override
+ public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new IEvaluator() {
+ private DataOutput dataout = output.getDataOutput();
+ private ArrayBackedValueStorage out1 = new ArrayBackedValueStorage();
+ private ArrayBackedValueStorage out2 = new ArrayBackedValueStorage();
+ private IEvaluator eval1 = arg1Factory.createEvaluator(out1);
+ private IEvaluator eval2 = arg2Factory.createEvaluator(out2);
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ out1.reset();
+ eval1.evaluate(tuple);
+ out2.reset();
+ eval2.evaluate(tuple);
+ int v1 = IntegerSerializerDeserializer.getInt(out1.getBytes(), 0);
+ int v2 = IntegerSerializerDeserializer.getInt(out2.getBytes(), 0);
+ boolean r = v1 == v2;
+ try {
+ dataout.writeBoolean(r);
+ } catch (IOException ioe) {
+ throw new AlgebricksException(ioe);
+ }
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
new file mode 100644
index 0000000..5acded6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/runtime/functions/PigletFunctionRegistry.java
@@ -0,0 +1,36 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.runtime.functions;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.exceptions.PigletException;
+
+public class PigletFunctionRegistry {
+ private static final Map<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder> builderMap;
+
+ static {
+ Map<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder> temp = new HashMap<FunctionIdentifier, IPigletFunctionEvaluatorFactoryBuilder>();
+
+ temp.put(AlgebricksBuiltinFunctions.EQ, new IPigletFunctionEvaluatorFactoryBuilder() {
+ @Override
+ public IEvaluatorFactory buildEvaluatorFactory(FunctionIdentifier fid, IEvaluatorFactory[] arguments) {
+ return new IntegerEqFunctionEvaluatorFactory(arguments[0], arguments[1]);
+ }
+ });
+
+ builderMap = Collections.unmodifiableMap(temp);
+ }
+
+ public static IEvaluatorFactory createFunctionEvaluatorFactory(FunctionIdentifier fid, IEvaluatorFactory[] args)
+ throws PigletException {
+ IPigletFunctionEvaluatorFactoryBuilder builder = builderMap.get(fid);
+ if (builder == null) {
+ throw new PigletException("Unknown function: " + fid);
+ }
+ return builder.buildEvaluatorFactory(fid, args);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/BagType.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/BagType.java
new file mode 100644
index 0000000..0d964b0
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/BagType.java
@@ -0,0 +1,8 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class BagType extends Type {
+ @Override
+ public Tag getTag() {
+ return Tag.BAG;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/CharArrayType.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/CharArrayType.java
new file mode 100644
index 0000000..ddb8e2b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/CharArrayType.java
@@ -0,0 +1,13 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class CharArrayType extends Type {
+ public static final Type INSTANCE = new CharArrayType();
+
+ private CharArrayType() {
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.CHAR_ARRAY;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/DoubleType.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/DoubleType.java
new file mode 100644
index 0000000..f3a3a1b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/DoubleType.java
@@ -0,0 +1,13 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class DoubleType extends Type {
+ public static final Type INSTANCE = new DoubleType();
+
+ private DoubleType() {
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.DOUBLE;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/FloatType.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/FloatType.java
new file mode 100644
index 0000000..4e6a9d9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/FloatType.java
@@ -0,0 +1,13 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class FloatType extends Type {
+ public static final Type INSTANCE = new FloatType();
+
+ private FloatType() {
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.FLOAT;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/IntegerType.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/IntegerType.java
new file mode 100644
index 0000000..3e755c6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/IntegerType.java
@@ -0,0 +1,13 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class IntegerType extends Type {
+ public static final Type INSTANCE = new IntegerType();
+
+ private IntegerType() {
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.INTEGER;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/LongType.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/LongType.java
new file mode 100644
index 0000000..e3af89a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/LongType.java
@@ -0,0 +1,13 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class LongType extends Type {
+ public static final Type INSTANCE = new LongType();
+
+ private LongType() {
+ }
+
+ @Override
+ public Tag getTag() {
+ return Tag.LONG;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/MapType.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/MapType.java
new file mode 100644
index 0000000..74cf717
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/MapType.java
@@ -0,0 +1,8 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class MapType extends Type {
+ @Override
+ public Tag getTag() {
+ return Tag.MAP;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java
new file mode 100644
index 0000000..ba2401c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Schema.java
@@ -0,0 +1,17 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class Schema {
+ private List<Pair<String, Type>> schema;
+
+ public Schema(List<Pair<String, Type>> schema) {
+ this.schema = schema;
+ }
+
+ public List<Pair<String, Type>> getSchema() {
+ return schema;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/TupleType.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/TupleType.java
new file mode 100644
index 0000000..4c68c2d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/TupleType.java
@@ -0,0 +1,8 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public class TupleType extends Type {
+ @Override
+ public Tag getTag() {
+ return Tag.TUPLE;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Type.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Type.java
new file mode 100644
index 0000000..b8286e5
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/java/edu/uci/ics/hyracks/algebricks/examples/piglet/types/Type.java
@@ -0,0 +1,16 @@
+package edu.uci.ics.hyracks.algebricks.examples.piglet.types;
+
+public abstract class Type {
+ public enum Tag {
+ INTEGER,
+ LONG,
+ FLOAT,
+ DOUBLE,
+ CHAR_ARRAY,
+ TUPLE,
+ BAG,
+ MAP
+ }
+
+ public abstract Tag getTag();
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/javacc/PigletParser.jj b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/javacc/PigletParser.jj
new file mode 100644
index 0000000..9e339e4
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/main/javacc/PigletParser.jj
@@ -0,0 +1,386 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+options {
+ STATIC = false;
+ IGNORE_CASE = true;
+ LOOKAHEAD = 2;
+}
+
+PARSER_BEGIN(PigletParser)
+
+package edu.uci.ics.hyracks.algebricks.examples.piglet.parser;
+
+import java.util.*;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.*;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.types.*;
+
+public class PigletParser {
+ private ExpressionNode createFunction(FunctionTag fTag, String fName, ExpressionNode... arguments) {
+ List<ASTNode> args = new ArrayList<ASTNode>();
+ for(ExpressionNode e : arguments) {
+ args.add(e);
+ }
+ return new ScalarFunctionExpressionNode(fTag, fName, args);
+ }
+
+ private String stripQuotes(String s) {
+ s = s.substring(1);
+ s = s.substring(0, s.length() - 1);
+ return s;
+ }
+}
+
+PARSER_END(PigletParser)
+
+List<ASTNode> Statements(): {
+ List<ASTNode> statememts = new ArrayList<ASTNode>();
+ ASTNode s;
+} {
+ (
+ (
+ s = AssignmentStatement() {
+ statememts.add(s);
+ }
+ | s = DumpStatement() {
+ statememts.add(s);
+ }
+ ) ";"
+ )* <EOF> {
+ return statememts;
+ }
+}
+
+ASTNode AssignmentStatement(): {
+ String a;
+ RelationNode r;
+} {
+ a = Alias() "=" r = RelationalStatement() {
+ return new AssignmentNode(a, r);
+ }
+}
+
+String Alias(): {
+ Token t;
+} {
+ t = <IDENTIFIER> {
+ return t.image;
+ }
+}
+
+String ColumnName(): {
+ Token t;
+} {
+ t = <IDENTIFIER> {
+ return t.image;
+ }
+}
+
+Type TypeName(): {
+ Token t;
+} {
+ "int" {
+ return IntegerType.INSTANCE;
+ }
+ | "long" {
+ return LongType.INSTANCE;
+ }
+ | "float" {
+ return FloatType.INSTANCE;
+ }
+ | "double" {
+ return DoubleType.INSTANCE;
+ }
+ | "chararray" {
+ return CharArrayType.INSTANCE;
+ }
+ | "tuple" {
+ return new TupleType();
+ }
+ | "bag" {
+ return new BagType();
+ }
+ | "map" {
+ return new MapType();
+ }
+}
+
+Pair<String, Type> ColumnSchema(): {
+ String c;
+ Type t;
+} {
+ c = ColumnName() ":" t = TypeName() {
+ return new Pair<String, Type>(c, t);
+ }
+}
+
+Schema Schema(): {
+ List<Pair<String, Type>> schema = new ArrayList<Pair<String, Type>>();
+ Pair<String, Type> cSchema;
+} {
+ "(" (
+ cSchema = ColumnSchema() {
+ schema.add(cSchema);
+ } (
+ "," cSchema = ColumnSchema() {
+ schema.add(cSchema);
+ }
+ )*
+ )? ")" {
+ return new Schema(schema);
+ }
+}
+
+RelationNode RelationalStatement(): {
+ RelationNode r;
+} {
+ r = LoadStatement() {
+ return r;
+ }
+ | r = FilterStatement() {
+ return r;
+ }
+}
+
+RelationNode LoadStatement(): {
+ Token t;
+ Schema s;
+} {
+ "load" t = <STRING_LITERAL> "as" s = Schema() {
+ return new LoadNode(t.image, s);
+ }
+}
+
+RelationNode FilterStatement(): {
+ String a;
+ ExpressionNode e;
+} {
+ "filter" a = Alias() "by" e = Expression() {
+ return new FilterNode(a, e);
+ }
+}
+
+ASTNode DumpStatement(): {
+ String a;
+ Token t;
+} {
+ "dump" a = Alias() "into" t = <STRING_LITERAL> {
+ return new DumpNode(t.image, a);
+ }
+}
+
+ExpressionNode Expression(): {
+ ExpressionNode e;
+} {
+ e = OrExpression() {
+ return e;
+ }
+}
+
+ExpressionNode OrExpression(): {
+ ExpressionNode e1;
+ ExpressionNode e2;
+} {
+ e1 = AndExpression() (
+ "or" e2 = AndExpression() {
+ e1 = createFunction(FunctionTag.BOOLEAN_OR, null, e1, e2);
+ }
+ )* {
+ return e1;
+ }
+}
+
+ExpressionNode AndExpression(): {
+ ExpressionNode e1;
+ ExpressionNode e2;
+} {
+ e1 = ComparisonExpression() (
+ "and" e2 = ComparisonExpression() {
+ e1 = createFunction(FunctionTag.BOOLEAN_AND, null, e1, e2);
+ }
+ )* {
+ return e1;
+ }
+}
+
+ExpressionNode ComparisonExpression(): {
+ ExpressionNode e1;
+ ExpressionNode e2;
+ FunctionTag fTag;
+} {
+ e1 = AdditiveExpression() (
+ fTag = ComparisonOperator() e2 = AdditiveExpression() {
+ e1 = createFunction(fTag, null, e1, e2);
+ }
+ )? {
+ return e1;
+ }
+}
+
+FunctionTag ComparisonOperator(): {
+} {
+ "==" {
+ return FunctionTag.EQ;
+ }
+ | "!=" {
+ return FunctionTag.NEQ;
+ }
+ | "<" {
+ return FunctionTag.LT;
+ }
+ | "<=" {
+ return FunctionTag.LTE;
+ }
+ | ">" {
+ return FunctionTag.GT;
+ }
+ | ">=" {
+ return FunctionTag.GTE;
+ }
+}
+
+ExpressionNode AdditiveExpression(): {
+ ExpressionNode e1;
+ ExpressionNode e2;
+ FunctionTag fTag;
+} {
+ e1 = MultiplicativeExpression() (
+ fTag = AdditiveOperator() e2 = MultiplicativeExpression() {
+ e1 = createFunction(fTag, null, e1, e2);
+ }
+ )* {
+ return e1;
+ }
+}
+
+FunctionTag AdditiveOperator(): {
+} {
+ "+" {
+ return FunctionTag.ADD;
+ }
+ | "-" {
+ return FunctionTag.SUBTRACT;
+ }
+}
+
+ExpressionNode MultiplicativeExpression(): {
+ ExpressionNode e1;
+ ExpressionNode e2;
+ FunctionTag fTag;
+} {
+ e1 = PrimaryExpression() (
+ fTag = MultiplicativeOperator() e2 = PrimaryExpression() {
+ e1 = createFunction(fTag, null, e1, e2);
+ }
+ )* {
+ return e1;
+ }
+}
+
+FunctionTag MultiplicativeOperator(): {
+} {
+ "*" {
+ return FunctionTag.MULTIPLY;
+ }
+ | "/" {
+ return FunctionTag.DIVIDE;
+ }
+ | "%" {
+ return FunctionTag.MOD;
+ }
+}
+
+ExpressionNode PrimaryExpression(): {
+ ExpressionNode e;
+} {
+ e = Literal() {
+ return e;
+ }
+ | e = FieldAccess() {
+ return e;
+ }
+}
+
+ExpressionNode Literal(): {
+ Token t;
+} {
+ t = <STRING_LITERAL> {
+ return new LiteralExpressionNode(stripQuotes(t.image), CharArrayType.INSTANCE);
+ }
+ | t = <INTEGER_LITERAL> {
+ return new LiteralExpressionNode(t.image, IntegerType.INSTANCE);
+ }
+ | t = <DOUBLE_LITERAL> {
+ return new LiteralExpressionNode(t.image, DoubleType.INSTANCE);
+ }
+}
+
+ExpressionNode FieldAccess(): {
+ String relName = null;
+ Token fieldName;
+} {
+ (relName = Alias() ".")? fieldName = <IDENTIFIER> {
+ return new FieldAccessExpressionNode(relName, fieldName.image);
+ }
+}
+
+<DEFAULT>
+TOKEN : {
+ <STRING_LITERAL: (("\"" (~["\"", "\n"])* "\"") | ("'" (~["'", "\n"])* "'"))>
+ | <IDENTIFIER: <Letter> (<Letter> | <Digit> | <Extender>)*>
+ | <INTEGER_LITERAL: (<Digit>)+>
+ | <DOUBLE_LITERAL: (((<Digit>)* "." (<Digit>)+ (<Exponent>)?) | (<INTEGER_LITERAL> <Exponent>))>
+ | <INDEXED_FIELD: ("$" <INTEGER_LITERAL>)>
+}
+
+TOKEN :
+{
+ < #Exponent : ((["+", "-"])? ["E", "e"] <INTEGER_LITERAL>)>
+}
+
+SPECIAL_TOKEN :
+{
+ < WhitespaceChar : ["\t", "\r", "\n", " "] >
+}
+
+TOKEN :
+{
+ < #Letter : (<BaseChar> | <Ideographic>) >
+}
+
+TOKEN :
+{
+ < #BaseChar : ["\u0041" - "\u005a", "\u0061" - "\u007a", "\u00c0" - "\u00d6", "\u00d8" - "\u00f6", "\u00f8" - "\u00ff", "\u0100" - "\u0131", "\u0134" - "\u013e", "\u0141" - "\u0148", "\u014a" - "\u017e", "\u0180" - "\u01c3", "\u01cd" - "\u01f0", "\u01f4" - "\u01f5", "\u01fa" - "\u0217", "\u0250" - "\u02a8", "\u02bb" - "\u02c1", "\u0386", "\u0388" - "\u038a", "\u038c", "\u038e" - "\u03a1", "\u03a3" - "\u03ce", "\u03d0" - "\u03d6", "\u03da", "\u03dc", "\u03de", "\u03e0", "\u03e2" - "\u03f3", "\u0401" - "\u040c", "\u040e" - "\u044f", "\u0451" - "\u045c", "\u045e" - "\u0481", "\u0490" - "\u04c4", "\u04c7" - "\u04c8", "\u04cb" - "\u04cc", "\u04d0" - "\u04eb", "\u04ee" - "\u04f5", "\u04f8" - "\u04f9", "\u0531" - "\u0556", "\u0559", "\u0561" - "\u0586", "\u05d0" - "\u05ea", "\u05f0" - "\u05f2", "\u0621" - "\u063a", "\u0641" - "\u064a", "\u0671" - "\u06b7", "\u06ba" - "\u06be", "\u06c0" - "\u06ce", "\u06d0" - "\u06d3", "\u06d5", "\u06e5" - "\u06e6", "\u0905" - "\u0939", "\u093d", "\u0958" - "\u0961", "\u0985" - "\u098c", "\u098f" - "\u0990", "\u0993" - "\u09a8", "\u09aa" - "\u09b0", "\u09b2", "\u09b6" - "\u09b9", "\u09dc" - "\u09dd", "\u09df" - "\u09e1", "\u09f0" - "\u09f1", "\u0a05" - "\u0a0a", "\u0a0f" - "\u0a10", "\u0a13" - "\u0a28", "\u0a2a" - "\u0a30", "\u0a32" - "\u0a33", "\u0a35" - "\u0a36", "\u0a38" - "\u0a39", "\u0a59" - "\u0a5c", "\u0a5e", "\u0a72" - "\u0a74", "\u0a85" - "\u0a8b", "\u0a8d", "\u0a8f" - "\u0a91", "\u0a93" - "\u0aa8", "\u0aaa" - "\u0ab0", "\u0ab2" - "\u0ab3", "\u0ab5" - "\u0ab9", "\u0abd", "\u0ae0", "\u0b05" - "\u0b0c", "\u0b0f" - "\u0b10", "\u0b13" - "\u0b28", "\u0b2a" - "\u0b30", "\u0b32" - "\u0b33", "\u0b36" - "\u0b39", "\u0b3d", "\u0b5c" - "\u0b5d", "\u0b5f" - "\u0b61", "\u0b85" - "\u0b8a", "\u0b8e" - "\u0b90", "\u0b92" - "\u0b95", "\u0b99" - "\u0b9a", "\u0b9c", "\u0b9e" - "\u0b9f", "\u0ba3" - "\u0ba4", "\u0ba8" - "\u0baa", "\u0bae" - "\u0bb5", "\u0bb7" - "\u0bb9", "\u0c05" - "\u0c0c", "\u0c0e" - "\u0c10", "\u0c12" - "\u0c28", "\u0c2a" - "\u0c33", "\u0c35" - "\u0c39", "\u0c60" - "\u0c61", "\u0c85" - "\u0c8c", "\u0c8e" - "\u0c90", "\u0c92" - "\u0ca8", "\u0caa" - "\u0cb3", "\u0cb5" - "\u0cb9", "\u0cde", "\u0ce0" - "\u0ce1", "\u0d05" - "\u0d0c", "\u0d0e" - "\u0d10", "\u0d12" - "\u0d28", "\u0d2a" - "\u0d39", "\u0d60" - "\u0d61", "\u0e01" - "\u0e2e", "\u0e30", "\u0e32" - "\u0e33", "\u0e40" - "\u0e45", "\u0e81" - "\u0e82", "\u0e84", "\u0e87" - "\u0e88", "\u0e8a", "\u0e8d", "\u0e94" - "\u0e97", "\u0e99" - "\u0e9f", "\u0ea1" - "\u0ea3", "\u0ea5", "\u0ea7", "\u0eaa" - "\u0eab", "\u0ead" - "\u0eae", "\u0eb0", "\u0eb2" - "\u0eb3", "\u0ebd", "\u0ec0" - "\u0ec4", "\u0f40" - "\u0f47", "\u0f49" - "\u0f69", "\u10a0" - "\u10c5", "\u10d0" - "\u10f6", "\u1100", "\u1102" - "\u1103", "\u1105" - "\u1107", "\u1109", "\u110b" - "\u110c", "\u110e" - "\u1112", "\u113c", "\u113e", "\u1140", "\u114c", "\u114e", "\u1150", "\u1154" - "\u1155", "\u1159", "\u115f" - "\u1161", "\u1163", "\u1165", "\u1167", "\u1169", "\u116d" - "\u116e", "\u1172" - "\u1173", "\u1175", "\u119e", "\u11a8", "\u11ab", "\u11ae" - "\u11af", "\u11b7" - "\u11b8", "\u11ba", "\u11bc" - "\u11c2", "\u11eb", "\u11f0", "\u11f9", "\u1e00" - "\u1e9b", "\u1ea0" - "\u1ef9", "\u1f00" - "\u1f15", "\u1f18" - "\u1f1d", "\u1f20" - "\u1f45", "\u1f48" - "\u1f4d", "\u1f50" - "\u1f57", "\u1f59", "\u1f5b", "\u1f5d", "\u1f5f" - "\u1f7d", "\u1f80" - "\u1fb4", "\u1fb6" - "\u1fbc", "\u1fbe", "\u1fc2" - "\u1fc4", "\u1fc6" - "\u1fcc", "\u1fd0" - "\u1fd3", "\u1fd6" - "\u1fdb", "\u1fe0" - "\u1fec", "\u1ff2" - "\u1ff4", "\u1ff6" - "\u1ffc", "\u2126", "\u212a" - "\u212b", "\u212e", "\u2180" - "\u2182", "\u3041" - "\u3094", "\u30a1" - "\u30fa", "\u3105" - "\u312c", "\uac00" - "\ud7a3"] >
+}
+
+TOKEN :
+{
+ < #Ideographic : ["\u4e00" - "\u9fa5", "\u3007", "\u3021" - "\u3029"] >
+}
+
+TOKEN :
+{
+ < #CombiningChar : ["\u0300" - "\u0345", "\u0360" - "\u0361", "\u0483" - "\u0486", "\u0591" - "\u05a1", "\u05a3" - "\u05b9", "\u05bb" - "\u05bd", "\u05bf", "\u05c1" - "\u05c2", "\u05c4", "\u064b" - "\u0652", "\u0670", "\u06d6" - "\u06dc", "\u06dd" - "\u06df", "\u06e0" - "\u06e4", "\u06e7" - "\u06e8", "\u06ea" - "\u06ed", "\u0901" - "\u0903", "\u093c", "\u093e" - "\u094c", "\u094d", "\u0951" - "\u0954", "\u0962" - "\u0963", "\u0981" - "\u0983", "\u09bc", "\u09be", "\u09bf", "\u09c0" - "\u09c4", "\u09c7" - "\u09c8", "\u09cb" - "\u09cd", "\u09d7", "\u09e2" - "\u09e3", "\u0a02", "\u0a3c", "\u0a3e", "\u0a3f", "\u0a40" - "\u0a42", "\u0a47" - "\u0a48", "\u0a4b" - "\u0a4d", "\u0a70" - "\u0a71", "\u0a81" - "\u0a83", "\u0abc", "\u0abe" - "\u0ac5", "\u0ac7" - "\u0ac9", "\u0acb" - "\u0acd", "\u0b01" - "\u0b03", "\u0b3c", "\u0b3e" - "\u0b43", "\u0b47" - "\u0b48", "\u0b4b" - "\u0b4d", "\u0b56" - "\u0b57", "\u0b82" - "\u0b83", "\u0bbe" - "\u0bc2", "\u0bc6" - "\u0bc8", "\u0bca" - "\u0bcd", "\u0bd7", "\u0c01" - "\u0c03", "\u0c3e" - "\u0c44", "\u0c46" - "\u0c48", "\u0c4a" - "\u0c4d", "\u0c55" - "\u0c56", "\u0c82" - "\u0c83", "\u0cbe" - "\u0cc4", "\u0cc6" - "\u0cc8", "\u0cca" - "\u0ccd", "\u0cd5" - "\u0cd6", "\u0d02" - "\u0d03", "\u0d3e" - "\u0d43", "\u0d46" - "\u0d48", "\u0d4a" - "\u0d4d", "\u0d57", "\u0e31", "\u0e34" - "\u0e3a", "\u0e47" - "\u0e4e", "\u0eb1", "\u0eb4" - "\u0eb9", "\u0ebb" - "\u0ebc", "\u0ec8" - "\u0ecd", "\u0f18" - "\u0f19", "\u0f35", "\u0f37", "\u0f39", "\u0f3e", "\u0f3f", "\u0f71" - "\u0f84", "\u0f86" - "\u0f8b", "\u0f90" - "\u0f95", "\u0f97", "\u0f99" - "\u0fad", "\u0fb1" - "\u0fb7", "\u0fb9", "\u20d0" - "\u20dc", "\u20e1", "\u302a" - "\u302f", "\u3099", "\u309a"] >
+}
+
+TOKEN :
+{
+ < #Digit : ["\u0030" - "\u0039", "\u0660" - "\u0669", "\u06f0" - "\u06f9", "\u0966" - "\u096f", "\u09e6" - "\u09ef", "\u0a66" - "\u0a6f", "\u0ae6" - "\u0aef", "\u0b66" - "\u0b6f", "\u0be7" - "\u0bef", "\u0c66" - "\u0c6f", "\u0ce6" - "\u0cef", "\u0d66" - "\u0d6f", "\u0e50" - "\u0e59", "\u0ed0" - "\u0ed9", "\u0f20" - "\u0f29"] >
+}
+
+TOKEN :
+{
+ < #Extender : ["\u00b7", "\u02d0", "\u02d1", "\u0387", "\u0640", "\u0e46", "\u0ec6", "\u3005", "\u3031" - "\u3035", "\u309d" - "\u309e", "\u30fc" - "\u30fe"] >
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTest.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTest.java
new file mode 100644
index 0000000..7e027fa
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTest.java
@@ -0,0 +1,29 @@
+package edu.uci.ics.algebricks.examples.piglet.test;
+
+import java.io.File;
+
+import junit.framework.Test;
+import junit.framework.TestSuite;
+
+public class PigletTest {
+ public static Test suite() {
+ TestSuite suite = new TestSuite();
+ File dir = new File("testcases");
+ findAndAddTests(suite, dir);
+
+ return suite;
+ }
+
+ private static void findAndAddTests(TestSuite suite, File dir) {
+ for (final File f : dir.listFiles()) {
+ if (f.getName().startsWith(".")) {
+ continue;
+ }
+ if (f.isDirectory()) {
+ findAndAddTests(suite, f);
+ } else if (f.getName().endsWith(".piglet")) {
+ suite.addTest(new PigletTestCase(f));
+ }
+ }
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTestCase.java b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTestCase.java
new file mode 100644
index 0000000..1a329f6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/src/test/java/edu/uci/ics/algebricks/examples/piglet/test/PigletTestCase.java
@@ -0,0 +1,42 @@
+package edu.uci.ics.algebricks.examples.piglet.test;
+
+import java.io.File;
+import java.io.FileReader;
+import java.util.List;
+
+import junit.framework.TestCase;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.algebricks.examples.piglet.ast.ASTNode;
+import edu.uci.ics.hyracks.algebricks.examples.piglet.compiler.PigletCompiler;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+
+public class PigletTestCase extends TestCase {
+
+ private final File file;
+
+ PigletTestCase(File file) {
+ super("testPiglet");
+ this.file = file;
+ }
+
+ @Test
+ public void testPiglet() {
+ try {
+ FileReader in = new FileReader(file);
+ try {
+ PigletCompiler c = new PigletCompiler();
+
+ List<ASTNode> ast = c.parse(in);
+ JobSpecification jobSpec = c.compile(ast);
+
+ System.err.println(jobSpec.toJSON());
+ } finally {
+ in.close();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/testcases/q1.piglet b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/testcases/q1.piglet
new file mode 100644
index 0000000..88111ff
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/testcases/q1.piglet
@@ -0,0 +1,2 @@
+R = load "nc1:data/file1.txt,nc2:data/file2.txt" as (id : int, name : chararray);
+dump R into "nc1:output";
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/testcases/q2.piglet b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/testcases/q2.piglet
new file mode 100644
index 0000000..5c45ac1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/piglet-example/testcases/q2.piglet
@@ -0,0 +1,3 @@
+R = load "nc1:data/file1.txt,nc2:data/file2.txt" as (id : int, name : chararray);
+S = filter R by id == 5;
+dump S into "nc1:output";
diff --git a/hyracks-algebricks/hyracks-algebricks-examples/pom.xml b/hyracks-algebricks/hyracks-algebricks-examples/pom.xml
new file mode 100644
index 0000000..05dd6df
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-examples/pom.xml
@@ -0,0 +1,15 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-algebricks-examples</artifactId>
+ <packaging>pom</packaging>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <modules>
+ <module>piglet-example</module>
+ </modules>
+</project>
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/pom.xml b/hyracks-algebricks/hyracks-algebricks-rewriter/pom.xml
new file mode 100644
index 0000000..fd3b2d1
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/pom.xml
@@ -0,0 +1,31 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-algebricks-rewriter</artifactId>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-core</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
new file mode 100644
index 0000000..fe33d09
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractDecorrelationRule.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
+
+public abstract class AbstractDecorrelationRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ protected boolean descOrSelfIsScanOrJoin(AbstractLogicalOperator op2) {
+ LogicalOperatorTag t = op2.getOperatorTag();
+ if (t == LogicalOperatorTag.DATASOURCESCAN || t == LogicalOperatorTag.INNERJOIN
+ || t == LogicalOperatorTag.LEFTOUTERJOIN) {
+ return true;
+ }
+ if (op2.getInputs().size() != 1) {
+ return false;
+ }
+ AbstractLogicalOperator alo = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
+ if (descOrSelfIsScanOrJoin(alo)) {
+ return true;
+ }
+ return false;
+ }
+
+ protected Set<LogicalVariable> computeGbyVarsUsingPksOnly(Set<LogicalVariable> varSet, AbstractLogicalOperator op,
+ IOptimizationContext context) throws AlgebricksException {
+ PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(op, context);
+ List<FunctionalDependency> fdList = context.getFDList(op);
+ if (fdList == null) {
+ return null;
+ }
+ // check if any of the FDs is a key
+ for (FunctionalDependency fd : fdList) {
+ if (fd.getTail().containsAll(varSet)) {
+ return new HashSet<LogicalVariable>(fd.getHead());
+ }
+ }
+ return null;
+ }
+
+ protected void buildVarExprList(Collection<LogicalVariable> vars, IOptimizationContext context, GroupByOperator g,
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
+ for (LogicalVariable ov : vars) {
+ LogicalVariable newVar = context.newVar();
+ ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
+ outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov,
+ new MutableObject<ILogicalExpression>(varExpr)));
+ for (ILogicalPlan p : g.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
+ context);
+ }
+ }
+ // g.substituteVarInNestedPlans(ov, newVar);
+ // OperatorManipulationUtil.substituteVarRec(lojoin, ov, newVar);
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
new file mode 100644
index 0000000..fbacede
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/AbstractExtractExprRule.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public abstract class AbstractExtractExprRule implements IAlgebraicRewriteRule {
+
+ protected LogicalVariable extractExprIntoAssignOpRef(ILogicalExpression gExpr, Mutable<ILogicalOperator> opRef2,
+ IOptimizationContext context) throws AlgebricksException {
+ LogicalVariable v = context.newVar();
+ AssignOperator a = new AssignOperator(v, new MutableObject<ILogicalExpression>(gExpr));
+ a.getInputs().add(new MutableObject<ILogicalOperator>(opRef2.getValue()));
+ opRef2.setValue(a);
+ if (gExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+ context.addNotToBeInlinedVar(v);
+ }
+ context.computeAndSetTypeEnvironmentForOperator(a);
+ return v;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
new file mode 100644
index 0000000..55eef21
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/BreakSelectIntoConjunctsRule.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class BreakSelectIntoConjunctsRule implements IAlgebraicRewriteRule {
+
+ private List<Mutable<ILogicalExpression>> conjs = new ArrayList<Mutable<ILogicalExpression>>();
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+ return false;
+ }
+ SelectOperator select = (SelectOperator) op;
+
+ ILogicalExpression cond = select.getCondition().getValue();
+
+ conjs.clear();
+ if (!cond.splitIntoConjuncts(conjs)) {
+ return false;
+ }
+
+ Mutable<ILogicalOperator> childOfSelect = select.getInputs().get(0);
+ boolean fst = true;
+ ILogicalOperator botOp = select;
+ ILogicalExpression firstExpr = null;
+ for (Mutable<ILogicalExpression> eRef : conjs) {
+ ILogicalExpression e = eRef.getValue();
+ if (fst) {
+ fst = false;
+ firstExpr = e;
+ } else {
+ SelectOperator newSelect = new SelectOperator(new MutableObject<ILogicalExpression>(e));
+ List<Mutable<ILogicalOperator>> botInpList = botOp.getInputs();
+ botInpList.clear();
+ botInpList.add(new MutableObject<ILogicalOperator>(newSelect));
+ context.computeAndSetTypeEnvironmentForOperator(botOp);
+ botOp = newSelect;
+ }
+ }
+ botOp.getInputs().add(childOfSelect);
+ select.getCondition().setValue(firstExpr);
+ context.computeAndSetTypeEnvironmentForOperator(botOp);
+ context.computeAndSetTypeEnvironmentForOperator(select);
+
+ return true;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
new file mode 100644
index 0000000..2ac387d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ComplexJoinInferenceRule.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ComplexJoinInferenceRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator op = opRef.getValue();
+ if (!(op instanceof AbstractScanOperator)) {
+ return false;
+ }
+
+ Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+ return false;
+ }
+ SubplanOperator subplan = (SubplanOperator) op2;
+
+ Mutable<ILogicalOperator> opRef3 = subplan.getInputs().get(0);
+ AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
+
+ if (op3.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
+ || op3.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ return false;
+ }
+
+ if (subplanHasFreeVariables(subplan)) {
+ return false;
+ }
+
+ HashSet<LogicalVariable> varsUsedInUnnest = new HashSet<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op, varsUsedInUnnest);
+
+ HashSet<LogicalVariable> producedInSubplan = new HashSet<LogicalVariable>();
+ VariableUtilities.getLiveVariables(subplan, producedInSubplan);
+
+ if (!producedInSubplan.containsAll(varsUsedInUnnest)) {
+ return false;
+ }
+
+ ntsToEtsInSubplan(subplan, context);
+ InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
+ join.getInputs().add(opRef3);
+ opRef2.setValue(OperatorManipulationUtil.eliminateSingleSubplanOverEts(subplan));
+ join.getInputs().add(new MutableObject<ILogicalOperator>(op));
+ opRef.setValue(join);
+ context.computeAndSetTypeEnvironmentForOperator(join);
+ return true;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ private static void ntsToEtsInSubplan(SubplanOperator s, IOptimizationContext context) throws AlgebricksException {
+ for (ILogicalPlan p : s.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ OperatorManipulationUtil.ntsToEts(r, context);
+ }
+ }
+ }
+
+ private static boolean subplanHasFreeVariables(SubplanOperator s) throws AlgebricksException {
+ for (ILogicalPlan p : s.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ if (OperatorPropertiesUtil.hasFreeVariablesInSelfOrDesc((AbstractLogicalOperator) r.getValue())) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
new file mode 100644
index 0000000..67e7cf0
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateAssignsRule.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ConsolidateAssignsRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ return false;
+ }
+ AssignOperator assign1 = (AssignOperator) op;
+
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) assign1.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ return false;
+ }
+
+ AssignOperator assign2 = (AssignOperator) op2;
+
+ HashSet<LogicalVariable> used1 = new HashSet<LogicalVariable>();
+ VariableUtilities.getUsedVariables(assign1, used1);
+ for (LogicalVariable v2 : assign2.getVariables()) {
+ if (used1.contains(v2)) {
+ return false;
+ }
+ }
+
+ assign1.getVariables().addAll(assign2.getVariables());
+ assign1.getExpressions().addAll(assign2.getExpressions());
+
+ Mutable<ILogicalOperator> botOpRef = assign2.getInputs().get(0);
+ List<Mutable<ILogicalOperator>> asgnInpList = assign1.getInputs();
+ asgnInpList.clear();
+ asgnInpList.add(botOpRef);
+ context.computeAndSetTypeEnvironmentForOperator(assign1);
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
new file mode 100644
index 0000000..c2bb13d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ConsolidateSelectsRule.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ConsolidateSelectsRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+ return false;
+ }
+ SelectOperator select = (SelectOperator) op;
+
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) select.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.SELECT) {
+ return false;
+ }
+
+ AbstractFunctionCallExpression conj = new ScalarFunctionCallExpression(
+ AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND));
+ conj.getArguments().add(new MutableObject<ILogicalExpression>(select.getCondition().getValue()));
+ conj.getArguments().add(((SelectOperator) op2).getCondition());
+
+ Mutable<ILogicalOperator> botOpRef = select.getInputs().get(0);
+ boolean more = true;
+ while (more) {
+ botOpRef = botOpRef.getValue().getInputs().get(0);
+ AbstractLogicalOperator botOp = (AbstractLogicalOperator) botOpRef.getValue();
+ if (botOp.getOperatorTag() == LogicalOperatorTag.SELECT) {
+ conj.getArguments().add(((SelectOperator) botOp).getCondition());
+ } else {
+ more = false;
+ }
+ }
+ select.getCondition().setValue(conj);
+ List<Mutable<ILogicalOperator>> selInptList = select.getInputs();
+ selInptList.clear();
+ selInptList.add(botOpRef);
+ context.computeAndSetTypeEnvironmentForOperator(select);
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
new file mode 100644
index 0000000..5c5356c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EliminateSubplanRule.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.LinkedList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class EliminateSubplanRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ /**
+ * Eliminate Subplan above ETS
+ *
+ * and Subplan that has only ops. with one input and no free vars. (could we
+ * modify it to consider free vars which are sources of Unnest or Assign, if
+ * there are no aggregates?)
+ */
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+ return false;
+ }
+ SubplanOperator subplan = (SubplanOperator) op;
+
+ Mutable<ILogicalOperator> outerRef = subplan.getInputs().get(0);
+ AbstractLogicalOperator outerRefOp = (AbstractLogicalOperator) outerRef.getValue();
+ if (outerRefOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+ elimSubplanOverEts(opRef, context);
+ return true;
+ }
+ if (subplan.getNestedPlans().size() == 1 && subplan.getNestedPlans().get(0).getRoots().size() == 1
+ && !OperatorPropertiesUtil.hasFreeVariables(subplan)) {
+ if (elimOneSubplanWithNoFreeVars(opRef)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private boolean elimOneSubplanWithNoFreeVars(Mutable<ILogicalOperator> opRef) {
+ SubplanOperator subplan = (SubplanOperator) opRef.getValue();
+ AbstractLogicalOperator rootOp = (AbstractLogicalOperator) subplan.getNestedPlans().get(0).getRoots().get(0)
+ .getValue();
+ if (rootOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+ opRef.setValue(subplan.getInputs().get(0).getValue());
+ return true;
+ } else {
+ AbstractLogicalOperator botOp = rootOp;
+ if (botOp.getInputs().size() != 1) {
+ return false;
+ }
+ do {
+ Mutable<ILogicalOperator> botRef = botOp.getInputs().get(0);
+ botOp = (AbstractLogicalOperator) botRef.getValue();
+ if (botOp.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+ botRef.setValue(subplan.getInputs().get(0).getValue());
+ opRef.setValue(rootOp);
+ return true;
+ }
+ } while (botOp.getInputs().size() == 1);
+ return false;
+ }
+ }
+
+ private void elimSubplanOverEts(Mutable<ILogicalOperator> opRef, IOptimizationContext ctx)
+ throws AlgebricksException {
+ SubplanOperator subplan = (SubplanOperator) opRef.getValue();
+ for (ILogicalPlan p : subplan.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ OperatorManipulationUtil.ntsToEts(r, ctx);
+ }
+ }
+ LinkedList<Mutable<ILogicalOperator>> allRoots = subplan.allRootsInReverseOrder();
+ if (allRoots.size() == 1) {
+ opRef.setValue(allRoots.get(0).getValue());
+ } else {
+ ILogicalOperator topOp = null;
+ for (Mutable<ILogicalOperator> r : allRoots) {
+ if (topOp == null) {
+ topOp = r.getValue();
+ } else {
+ LeftOuterJoinOperator j = new LeftOuterJoinOperator(new MutableObject<ILogicalExpression>(
+ ConstantExpression.TRUE));
+ j.getInputs().add(new MutableObject<ILogicalOperator>(topOp));
+ j.getInputs().add(r);
+ ctx.setOutputTypeEnvironment(j, j.computeOutputTypeEnvironment(ctx));
+ topOp = j;
+ }
+ }
+ opRef.setValue(topOp);
+ }
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
new file mode 100644
index 0000000..5ebaad7
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
@@ -0,0 +1,583 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder.OrderKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractStableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.BroadcastPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ExternalGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryStableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreSortedDistinctByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreclusteredGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RandomMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RangePartitionPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.AsterixNodeGroupDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILocalStructuralProperty.PropertyType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.INodeDomain;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningProperty.PartitioningType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPartitioningRequirementsCoordinator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.IPhysicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.LocalOrderProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PhysicalRequirements;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.PropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.RandomPartitioningProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.StructuralPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.UnorderedPartitionedProperty;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
+
+public class EnforceStructuralPropertiesRule implements IAlgebraicRewriteRule {
+
+ private static final INodeDomain DEFAULT_DOMAIN = new AsterixNodeGroupDomain("__DEFAULT");
+
+ private PhysicalOptimizationConfig physicalOptimizationConfig;
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ // wait for the physical operators to be set first
+ if (op.getPhysicalOperator() == null) {
+ return false;
+ }
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+
+ List<FunctionalDependency> fds = context.getFDList(op);
+ if (fds != null && !fds.isEmpty()) {
+ return false;
+ }
+ // These are actually logical constraints, so they could be pre-computed
+ // somewhere else, too.
+
+ physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Optimizing operator " + op.getPhysicalOperator() + ".\n");
+
+ PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(op, context);
+
+ StructuralPropertiesVector pvector = new StructuralPropertiesVector(new RandomPartitioningProperty(null),
+ new LinkedList<ILocalStructuralProperty>());
+ boolean changed = physOptimizeOp(opRef, pvector, false, context);
+ op.computeDeliveredPhysicalProperties(context);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + op.getPhysicalOperator() + ": "
+ + op.getDeliveredPhysicalProperties() + "\n");
+
+ context.addToDontApplySet(this, opRef.getValue());
+
+ return changed;
+ }
+
+ private boolean physOptimizePlan(ILogicalPlan plan, IPhysicalPropertiesVector pvector, boolean nestedPlan,
+ IOptimizationContext context) throws AlgebricksException {
+ boolean changed = false;
+ for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+ if (physOptimizeOp(root, pvector, nestedPlan, context)) {
+ changed = true;
+ }
+ AbstractLogicalOperator op = (AbstractLogicalOperator) root.getValue();
+ op.computeDeliveredPhysicalProperties(context);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + op.getPhysicalOperator()
+ + ": " + op.getDeliveredPhysicalProperties() + "\n");
+ }
+ return changed;
+ }
+
+ private boolean physOptimizeOp(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector required,
+ boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
+
+ boolean changed = false;
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ optimizeUsingConstraintsAndEquivClasses(op);
+ PhysicalRequirements pr = op.getRequiredPhysicalPropertiesForChildren(required);
+ IPhysicalPropertiesVector[] reqdProperties = null;
+ if (pr != null) {
+ reqdProperties = pr.getRequiredProperties();
+ }
+ boolean opIsRedundantSort = false;
+
+ // compute properties and figure out the domain
+ INodeDomain childrenDomain = null;
+ {
+ int j = 0;
+ for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+ AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
+ // recursive call
+ if (physOptimizeOp(childRef, reqdProperties[j], nestedPlan, context)) {
+ changed = true;
+ }
+ child.computeDeliveredPhysicalProperties(context);
+ IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
+ if (childrenDomain == null) {
+ childrenDomain = delivered.getPartitioningProperty().getNodeDomain();
+ } else {
+ INodeDomain dom2 = delivered.getPartitioningProperty().getNodeDomain();
+ if (!childrenDomain.sameAs(dom2)) {
+ childrenDomain = DEFAULT_DOMAIN;
+ }
+ }
+ j++;
+ }
+ }
+
+ if (reqdProperties != null) {
+ for (int k = 0; k < reqdProperties.length; k++) {
+ IPhysicalPropertiesVector pv = reqdProperties[k];
+ IPartitioningProperty pp = pv.getPartitioningProperty();
+ if (pp != null && pp.getNodeDomain() == null) {
+ pp.setNodeDomain(childrenDomain);
+ }
+ }
+ }
+
+ IPartitioningProperty firstDeliveredPartitioning = null;
+ int i = 0;
+ for (Mutable<ILogicalOperator> childRef : op.getInputs()) {
+ AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue();
+ IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties();
+
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Properties delivered by " + child.getPhysicalOperator()
+ + ": " + delivered + "\n");
+ IPartitioningRequirementsCoordinator prc = pr.getPartitioningCoordinator();
+ Pair<Boolean, IPartitioningProperty> pbpp = prc.coordinateRequirements(
+ reqdProperties[i].getPartitioningProperty(), firstDeliveredPartitioning, op, context);
+ boolean mayExpandPartitioningProperties = pbpp.first;
+ IPhysicalPropertiesVector rqd = new StructuralPropertiesVector(pbpp.second,
+ reqdProperties[i].getLocalProperties());
+
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Required properties for " + child.getPhysicalOperator()
+ + ": " + rqd + "\n");
+ IPhysicalPropertiesVector diff = delivered.getUnsatisfiedPropertiesFrom(rqd,
+ mayExpandPartitioningProperties, context.getEquivalenceClassMap(child), context.getFDList(child));
+
+ if (isRedundantSort(opRef, delivered, diff, context)) {
+ opIsRedundantSort = true;
+ }
+
+ if (diff != null) {
+ changed = true;
+ addEnforcers(op, i, diff, rqd, delivered, childrenDomain, nestedPlan, context);
+
+ AbstractLogicalOperator newChild = ((AbstractLogicalOperator) op.getInputs().get(i).getValue());
+
+ if (newChild != child) {
+ delivered = newChild.getDeliveredPhysicalProperties();
+ IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, rqd,
+ mayExpandPartitioningProperties, context);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> New properties diff: " + newDiff + "\n");
+
+ if (isRedundantSort(opRef, delivered, newDiff, context)) {
+ opIsRedundantSort = true;
+ break;
+ }
+ }
+
+ }
+ if (firstDeliveredPartitioning == null) {
+ IPartitioningProperty dpp = delivered.getPartitioningProperty();
+ if (dpp.getPartitioningType() == PartitioningType.ORDERED_PARTITIONED
+ || dpp.getPartitioningType() == PartitioningType.UNORDERED_PARTITIONED) {
+ firstDeliveredPartitioning = dpp;
+ }
+ }
+
+ i++;
+ }
+
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : nested.getNestedPlans()) {
+ if (physOptimizePlan(p, required, true, context)) {
+ changed = true;
+ }
+ }
+ }
+
+ if (opIsRedundantSort) {
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Removing redundant SORT operator "
+ + op.getPhysicalOperator() + "\n");
+ printOp((AbstractLogicalOperator) op);
+ }
+ changed = true;
+ AbstractLogicalOperator nextOp = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ if (nextOp.getOperatorTag() == LogicalOperatorTag.PROJECT) {
+ nextOp = (AbstractLogicalOperator) nextOp.getInputs().get(0).getValue();
+ }
+ opRef.setValue(nextOp);
+ // Now, transfer annotations from the original sort op. to this one.
+ AbstractLogicalOperator transferTo = nextOp;
+ if (transferTo.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+ transferTo = (AbstractLogicalOperator) transferTo.getInputs().get(0).getValue();
+ }
+ transferTo.getAnnotations().putAll(op.getAnnotations());
+ physOptimizeOp(opRef, required, nestedPlan, context);
+ }
+ return changed;
+ }
+
+ private IPhysicalPropertiesVector newPropertiesDiff(AbstractLogicalOperator newChild,
+ IPhysicalPropertiesVector required, boolean mayExpandPartitioningProperties, IOptimizationContext context)
+ throws AlgebricksException {
+ IPhysicalPropertiesVector newDelivered = newChild.getDeliveredPhysicalProperties();
+
+ Map<LogicalVariable, EquivalenceClass> newChildEqClasses = context.getEquivalenceClassMap(newChild);
+ List<FunctionalDependency> newChildFDs = context.getFDList(newChild);
+ if (newChildEqClasses == null || newChildFDs == null) {
+ FDsAndEquivClassesVisitor fdsVisitor = new FDsAndEquivClassesVisitor();
+ newChild.accept(fdsVisitor, context);
+ newChildEqClasses = context.getEquivalenceClassMap(newChild);
+ newChildFDs = context.getFDList(newChild);
+ }
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Required properties for new op. "
+ + newChild.getPhysicalOperator() + ": " + required + "\n");
+
+ return newDelivered.getUnsatisfiedPropertiesFrom(required, mayExpandPartitioningProperties, newChildEqClasses,
+ newChildFDs);
+ }
+
+ private void optimizeUsingConstraintsAndEquivClasses(AbstractLogicalOperator op) {
+ IPhysicalOperator pOp = op.getPhysicalOperator();
+ switch (pOp.getOperatorTag()) {
+ case HASH_GROUP_BY:
+ case EXTERNAL_GROUP_BY: {
+ GroupByOperator gby = (GroupByOperator) op;
+ ExternalGroupByPOperator hgbyOp = (ExternalGroupByPOperator) pOp;
+ hgbyOp.computeColumnSet(gby.getGroupByList());
+ break;
+ }
+ case PRE_CLUSTERED_GROUP_BY: {
+ GroupByOperator gby = (GroupByOperator) op;
+ PreclusteredGroupByPOperator preSortedGby = (PreclusteredGroupByPOperator) pOp;
+ preSortedGby.setGbyColumns(gby.getGbyVarList());
+ break;
+ }
+ case PRE_SORTED_DISTINCT_BY: {
+ DistinctOperator d = (DistinctOperator) op;
+ PreSortedDistinctByPOperator preSortedDistinct = (PreSortedDistinctByPOperator) pOp;
+ preSortedDistinct.setDistinctByColumns(d.getDistinctByVarList());
+ break;
+ }
+ }
+ }
+
+ private List<OrderColumn> getOrderColumnsFromGroupingProperties(List<ILocalStructuralProperty> reqd,
+ List<ILocalStructuralProperty> dlvd) {
+ List<OrderColumn> returnedProperties = new ArrayList<OrderColumn>();
+ List<LogicalVariable> rqdCols = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> dlvdCols = new ArrayList<LogicalVariable>();
+ for (ILocalStructuralProperty r : reqd) {
+ r.getVariables(rqdCols);
+ }
+ for (ILocalStructuralProperty d : dlvd) {
+ d.getVariables(dlvdCols);
+ }
+
+ int prefix = dlvdCols.size() - 1;
+ for (; prefix >= 0;)
+ if (!rqdCols.contains(dlvdCols.get(prefix)))
+ prefix--;
+ else
+ break;
+ for (int j = 0; j <= prefix; j++) {
+ LocalOrderProperty orderProp = (LocalOrderProperty) dlvd.get(j);
+ returnedProperties.add(new OrderColumn(orderProp.getColumn(), orderProp.getOrder()));
+ }
+ return returnedProperties;
+ }
+
+ /*
+ * We assume delivered to be already normalized.
+ */
+ private boolean isRedundantSort(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector delivered,
+ IPhysicalPropertiesVector diffOfProperties, IOptimizationContext context) {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.ORDER
+ || (op.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.STABLE_SORT && op
+ .getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT)
+ || delivered.getLocalProperties() == null) {
+ return false;
+ }
+ AbstractStableSortPOperator sortOp = (AbstractStableSortPOperator) op.getPhysicalOperator();
+ sortOp.computeLocalProperties(op);
+ List<ILocalStructuralProperty> orderProps = sortOp.getOrderProperties();
+ return PropertiesUtil.matchLocalProperties(orderProps, delivered.getLocalProperties(),
+ context.getEquivalenceClassMap(op), context.getFDList(op));
+ }
+
+ private void addEnforcers(AbstractLogicalOperator op, int childIndex,
+ IPhysicalPropertiesVector diffPropertiesVector, IPhysicalPropertiesVector required,
+ IPhysicalPropertiesVector deliveredByChild, INodeDomain domain, boolean nestedPlan,
+ IOptimizationContext context) throws AlgebricksException {
+
+ IPartitioningProperty pp = diffPropertiesVector.getPartitioningProperty();
+ if (pp == null || pp.getPartitioningType() == PartitioningType.UNPARTITIONED) {
+ addLocalEnforcers(op, childIndex, diffPropertiesVector.getLocalProperties(), nestedPlan, context);
+ IPhysicalPropertiesVector deliveredByNewChild = ((AbstractLogicalOperator) op.getInputs().get(0).getValue())
+ .getDeliveredPhysicalProperties();
+ addPartitioningEnforcers(op, childIndex, pp, required, deliveredByNewChild, domain, context);
+ } else {
+ addPartitioningEnforcers(op, childIndex, pp, required, deliveredByChild, domain, context);
+ AbstractLogicalOperator newChild = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue();
+ IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, required, true, context);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> New properties diff: " + newDiff + "\n");
+ if (newDiff != null) {
+ addLocalEnforcers(op, childIndex, newDiff.getLocalProperties(), nestedPlan, context);
+ }
+ }
+ }
+
+ private void addLocalEnforcers(AbstractLogicalOperator op, int i, List<ILocalStructuralProperty> localProperties,
+ boolean nestedPlan, IOptimizationContext context) throws AlgebricksException {
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Adding local enforcers for local props = " + localProperties
+ + "\n");
+ }
+
+ if (localProperties == null || localProperties.isEmpty()) {
+ return;
+ }
+
+ Mutable<ILogicalOperator> topOp = new MutableObject<ILogicalOperator>();
+ topOp.setValue(op.getInputs().get(i).getValue());
+ LinkedList<LocalOrderProperty> oList = new LinkedList<LocalOrderProperty>();
+
+ for (ILocalStructuralProperty prop : localProperties) {
+ switch (prop.getPropertyType()) {
+ case LOCAL_ORDER_PROPERTY: {
+ oList.add((LocalOrderProperty) prop);
+ break;
+ }
+ case LOCAL_GROUPING_PROPERTY: {
+ LocalGroupingProperty g = (LocalGroupingProperty) prop;
+ Collection<LogicalVariable> vars = (g.getPreferredOrderEnforcer() != null) ? g
+ .getPreferredOrderEnforcer() : g.getColumnSet();
+ for (LogicalVariable v : vars) {
+ OrderColumn oc = new OrderColumn(v, OrderKind.ASC);
+ LocalOrderProperty lop = new LocalOrderProperty(oc);
+ oList.add(lop);
+ }
+ break;
+ }
+ default: {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ if (!oList.isEmpty()) {
+ topOp = enforceOrderProperties(oList, topOp, nestedPlan, context);
+ }
+
+ op.getInputs().set(i, topOp);
+ OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) topOp.getValue(), context);
+ printOp((AbstractLogicalOperator) topOp.getValue());
+ }
+
+ private Mutable<ILogicalOperator> enforceOrderProperties(List<LocalOrderProperty> oList,
+ Mutable<ILogicalOperator> topOp, boolean isMicroOp, IOptimizationContext context)
+ throws AlgebricksException {
+ List<Pair<IOrder, Mutable<ILogicalExpression>>> oe = new LinkedList<Pair<IOrder, Mutable<ILogicalExpression>>>();
+ for (LocalOrderProperty o : oList) {
+ IOrder ordType = (o.getOrder() == OrderKind.ASC) ? OrderOperator.ASC_ORDER : OrderOperator.DESC_ORDER;
+ Pair<IOrder, Mutable<ILogicalExpression>> pair = new Pair<IOrder, Mutable<ILogicalExpression>>(ordType,
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(o.getColumn())));
+ oe.add(pair);
+ }
+ OrderOperator oo = new OrderOperator(oe);
+ oo.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL);
+ if (isMicroOp) {
+ oo.setPhysicalOperator(new InMemoryStableSortPOperator());
+ } else {
+ oo.setPhysicalOperator(new StableSortPOperator(physicalOptimizationConfig.getMaxFramesExternalSort()));
+ }
+ oo.getInputs().add(topOp);
+ context.computeAndSetTypeEnvironmentForOperator(oo);
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Added sort enforcer " + oo.getPhysicalOperator() + ".\n");
+ }
+ return new MutableObject<ILogicalOperator>(oo);
+ }
+
+ private void addPartitioningEnforcers(ILogicalOperator op, int i, IPartitioningProperty pp,
+ IPhysicalPropertiesVector required, IPhysicalPropertiesVector deliveredByChild, INodeDomain domain,
+ IOptimizationContext context) throws AlgebricksException {
+ if (pp != null) {
+ IPhysicalOperator pop;
+ switch (pp.getPartitioningType()) {
+ case UNPARTITIONED: {
+ List<OrderColumn> ordCols = computeOrderColumns(deliveredByChild);
+ if (ordCols == null || ordCols.size() == 0) {
+ pop = new RandomMergeExchangePOperator();
+ } else {
+ OrderColumn[] sortColumns = new OrderColumn[ordCols.size()];
+ sortColumns = ordCols.toArray(sortColumns);
+ pop = new SortMergeExchangePOperator(sortColumns);
+ }
+ break;
+ }
+ case UNORDERED_PARTITIONED: {
+ List<LogicalVariable> varList = new ArrayList<LogicalVariable>(
+ ((UnorderedPartitionedProperty) pp).getColumnSet());
+ List<ILocalStructuralProperty> cldLocals = deliveredByChild.getLocalProperties();
+ List<ILocalStructuralProperty> reqdLocals = required.getLocalProperties();
+ boolean propWasSet = false;
+ pop = null;
+ if (reqdLocals != null && cldLocals != null && allAreOrderProps(cldLocals)) {
+ AbstractLogicalOperator c = (AbstractLogicalOperator) op.getInputs().get(i).getValue();
+ Map<LogicalVariable, EquivalenceClass> ecs = context.getEquivalenceClassMap(c);
+ List<FunctionalDependency> fds = context.getFDList(c);
+ if (PropertiesUtil.matchLocalProperties(reqdLocals, cldLocals, ecs, fds)) {
+ List<OrderColumn> orderColumns = getOrderColumnsFromGroupingProperties(reqdLocals,
+ cldLocals);
+ pop = new HashPartitionMergeExchangePOperator(orderColumns, varList, domain);
+ propWasSet = true;
+ }
+ }
+ if (!propWasSet) {
+ pop = new HashPartitionExchangePOperator(varList, domain);
+ }
+ break;
+ }
+ case ORDERED_PARTITIONED: {
+ pop = new RangePartitionPOperator(((OrderedPartitionedProperty) pp).getOrderColumns(), domain);
+ break;
+ }
+ case BROADCAST: {
+ pop = new BroadcastPOperator(domain);
+ break;
+ }
+ case RANDOM: {
+ RandomPartitioningProperty rpp = (RandomPartitioningProperty) pp;
+ INodeDomain nd = rpp.getNodeDomain();
+ if (nd == null) {
+ throw new AlgebricksException("Unknown node domain for " + rpp);
+ }
+ if (nd.cardinality() == null) {
+ throw new AlgebricksException("Unknown cardinality for node domain " + nd);
+ }
+ if (nd.cardinality() != 1) {
+ throw new NotImplementedException(
+ "Random repartitioning is only implemented for target domains of"
+ + "cardinality equal to 1.");
+ }
+ pop = new BroadcastPOperator(nd);
+ break;
+ }
+ default: {
+ throw new NotImplementedException("Enforcer for " + pp.getPartitioningType()
+ + " partitioning type has not been implemented.");
+ }
+ }
+ Mutable<ILogicalOperator> ci = op.getInputs().get(i);
+ ExchangeOperator exchg = new ExchangeOperator();
+ exchg.setPhysicalOperator(pop);
+ setNewOp(ci, exchg, context);
+ exchg.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED);
+ OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(exchg, context);
+ context.computeAndSetTypeEnvironmentForOperator(exchg);
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(">>>> Added partitioning enforcer "
+ + exchg.getPhysicalOperator() + ".\n");
+ printOp((AbstractLogicalOperator) op);
+ }
+ }
+ }
+
+ private boolean allAreOrderProps(List<ILocalStructuralProperty> cldLocals) {
+ for (ILocalStructuralProperty lsp : cldLocals) {
+ if (lsp.getPropertyType() != PropertyType.LOCAL_ORDER_PROPERTY) {
+ return false;
+ }
+ }
+ return !cldLocals.isEmpty();
+ }
+
+ private void printOp(AbstractLogicalOperator op) throws AlgebricksException {
+ StringBuilder sb = new StringBuilder();
+ LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
+ PlanPrettyPrinter.printOperator(op, sb, pvisitor, 0);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine(sb.toString());
+ }
+
+ private List<OrderColumn> computeOrderColumns(IPhysicalPropertiesVector pv) {
+ List<OrderColumn> ordCols = new ArrayList<OrderColumn>();
+ List<ILocalStructuralProperty> localProps = pv.getLocalProperties();
+ if (localProps == null || localProps.size() == 0) {
+ return null;
+ } else {
+ for (ILocalStructuralProperty p : localProps) {
+ if (p.getPropertyType() == PropertyType.LOCAL_ORDER_PROPERTY) {
+ LocalOrderProperty lop = (LocalOrderProperty) p;
+ ordCols.add(lop.getOrderColumn());
+ } else {
+ return null;
+ }
+ }
+ return ordCols;
+ }
+
+ }
+
+ private void setNewOp(Mutable<ILogicalOperator> opRef, AbstractLogicalOperator newOp, IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator oldOp = opRef.getValue();
+ opRef.setValue(newOp);
+ newOp.getInputs().add(new MutableObject<ILogicalOperator>(oldOp));
+ newOp.recomputeSchema();
+ newOp.computeDeliveredPhysicalProperties(context);
+ context.computeAndSetTypeEnvironmentForOperator(newOp);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest(">>>> Structural properties for " + newOp.getPhysicalOperator()
+ + ": " + newOp.getDeliveredPhysicalProperties() + "\n");
+
+ PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(newOp, context);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
new file mode 100644
index 0000000..3501467
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractCommonOperatorsRule.java
@@ -0,0 +1,391 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ReplicateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.IsomorphismUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AssignPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.OneToOneExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ReplicatePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ExtractCommonOperatorsRule implements IAlgebraicRewriteRule {
+
+ private HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>> childrenToParents = new HashMap<Mutable<ILogicalOperator>, List<Mutable<ILogicalOperator>>>();
+ private List<Mutable<ILogicalOperator>> roots = new ArrayList<Mutable<ILogicalOperator>>();
+ private List<Mutable<ILogicalOperator>> joins = new ArrayList<Mutable<ILogicalOperator>>();
+ private List<List<Mutable<ILogicalOperator>>> equivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT) {
+ return false;
+ }
+ if (!roots.contains(op))
+ roots.add(new MutableObject<ILogicalOperator>(op));
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.WRITE && op.getOperatorTag() != LogicalOperatorTag.WRITE_RESULT) {
+ return false;
+ }
+ boolean rewritten = false;
+ boolean changed = false;
+ if (roots.size() > 0) {
+ do {
+ changed = false;
+ // applying the rewriting until fixpoint
+ topDownMaterialization(roots);
+ removeNonJoinBuildBranchCandidates();
+ genCandidates(context);
+ removeTrivialShare();
+ removeNonJoinBuildBranchCandidates();
+ if (equivalenceClasses.size() > 0)
+ changed = rewrite(context);
+ if (!rewritten)
+ rewritten = changed;
+ equivalenceClasses.clear();
+ childrenToParents.clear();
+ joins.clear();
+ } while (changed);
+ roots.clear();
+ }
+ return rewritten;
+ }
+
+ private void removeTrivialShare() {
+ for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+ for (int i = candidates.size() - 1; i >= 0; i--) {
+ Mutable<ILogicalOperator> opRef = candidates.get(i);
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) opRef.getValue();
+ if (aop.getOperatorTag() == LogicalOperatorTag.EXCHANGE)
+ aop = (AbstractLogicalOperator) aop.getInputs().get(0).getValue();
+ if (aop.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE)
+ candidates.remove(i);
+ }
+ }
+ for (int i = equivalenceClasses.size() - 1; i >= 0; i--)
+ if (equivalenceClasses.get(i).size() < 2)
+ equivalenceClasses.remove(i);
+ }
+
+ private void removeNonJoinBuildBranchCandidates() {
+ for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+ for (int i = candidates.size() - 1; i >= 0; i--) {
+ Mutable<ILogicalOperator> opRef = candidates.get(i);
+ boolean reserve = false;
+ for (Mutable<ILogicalOperator> join : joins)
+ if (isInJoinBuildBranch(join, opRef)) {
+ reserve = true;
+ }
+ if (!reserve)
+ candidates.remove(i);
+ }
+ }
+ for (int i = equivalenceClasses.size() - 1; i >= 0; i--)
+ if (equivalenceClasses.get(i).size() < 2)
+ equivalenceClasses.remove(i);
+ }
+
+ private boolean isInJoinBuildBranch(Mutable<ILogicalOperator> joinRef, Mutable<ILogicalOperator> opRef) {
+ Mutable<ILogicalOperator> buildBranch = joinRef.getValue().getInputs().get(1);
+ do {
+ if (buildBranch.equals(opRef)) {
+ return true;
+ } else {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) buildBranch.getValue();
+ if (aop.getOperatorTag() == LogicalOperatorTag.INNERJOIN
+ || aop.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN
+ || buildBranch.getValue().getInputs().size() == 0)
+ return false;
+ else
+ buildBranch = buildBranch.getValue().getInputs().get(0);
+ }
+ } while (true);
+ }
+
+ private boolean rewrite(IOptimizationContext context) throws AlgebricksException {
+ boolean changed = false;
+ for (List<Mutable<ILogicalOperator>> members : equivalenceClasses) {
+ if (rewriteForOneEquivalentClass(members, context))
+ changed = true;
+ }
+ return changed;
+ }
+
+ private boolean rewriteForOneEquivalentClass(List<Mutable<ILogicalOperator>> members, IOptimizationContext context)
+ throws AlgebricksException {
+ List<Mutable<ILogicalOperator>> group = new ArrayList<Mutable<ILogicalOperator>>();
+ boolean rewritten = false;
+ while (members.size() > 0) {
+ group.clear();
+ Mutable<ILogicalOperator> candidate = members.remove(members.size() - 1);
+ group.add(candidate);
+ for (int i = members.size() - 1; i >= 0; i--) {
+ Mutable<ILogicalOperator> peer = members.get(i);
+ if (IsomorphismUtilities.isOperatorIsomorphic(candidate.getValue(), peer.getValue())) {
+ group.add(peer);
+ members.remove(i);
+ }
+ }
+ AbstractLogicalOperator rop = new ReplicateOperator(group.size());
+ rop.setPhysicalOperator(new ReplicatePOperator());
+ rop.setExecutionMode(ExecutionMode.PARTITIONED);
+ Mutable<ILogicalOperator> ropRef = new MutableObject<ILogicalOperator>(rop);
+ AbstractLogicalOperator aopCandidate = (AbstractLogicalOperator) candidate.getValue();
+
+ if (aopCandidate.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+ rop.getInputs().add(candidate);
+ } else {
+ AbstractLogicalOperator beforeExchange = new ExchangeOperator();
+ beforeExchange.setPhysicalOperator(new OneToOneExchangePOperator());
+ beforeExchange.getInputs().add(candidate);
+ context.computeAndSetTypeEnvironmentForOperator(beforeExchange);
+ rop.getInputs().add(new MutableObject<ILogicalOperator>(beforeExchange));
+ }
+ context.computeAndSetTypeEnvironmentForOperator(rop);
+
+ List<Mutable<ILogicalOperator>> parents = childrenToParents.get(candidate);
+ for (Mutable<ILogicalOperator> parentRef : parents) {
+ AbstractLogicalOperator parent = (AbstractLogicalOperator) parentRef.getValue();
+ int index = parent.getInputs().indexOf(candidate);
+ if (parent.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+ parent.getInputs().set(index, ropRef);
+ } else {
+ AbstractLogicalOperator exchange = new ExchangeOperator();
+ exchange.setPhysicalOperator(new OneToOneExchangePOperator());
+ exchange.getInputs().add(ropRef);
+ context.computeAndSetTypeEnvironmentForOperator(exchange);
+ // parent.getInputs().get(index).setValue(exchange);
+ parent.getInputs().set(index, new MutableObject<ILogicalOperator>(exchange));
+ context.computeAndSetTypeEnvironmentForOperator(parent);
+ }
+ }
+
+ List<LogicalVariable> liveVarsNew = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(candidate.getValue(), liveVarsNew);
+ ArrayList<Mutable<ILogicalExpression>> assignExprs = new ArrayList<Mutable<ILogicalExpression>>();
+ for (LogicalVariable liveVar : liveVarsNew)
+ assignExprs.add(new MutableObject<ILogicalExpression>(new VariableReferenceExpression(liveVar)));
+ for (Mutable<ILogicalOperator> ref : group) {
+ if (ref.equals(candidate))
+ continue;
+ ArrayList<LogicalVariable> liveVars = new ArrayList<LogicalVariable>();
+ Map<LogicalVariable, LogicalVariable> variableMappingBack = new HashMap<LogicalVariable, LogicalVariable>();
+ IsomorphismUtilities.mapVariablesTopDown(ref.getValue(), candidate.getValue(), variableMappingBack);
+ for (int i = 0; i < liveVarsNew.size(); i++) {
+ liveVars.add(variableMappingBack.get(liveVarsNew.get(i)));
+ }
+
+ AbstractLogicalOperator assignOperator = new AssignOperator(liveVars, assignExprs);
+ assignOperator.setPhysicalOperator(new AssignPOperator());
+ AbstractLogicalOperator projectOperator = new ProjectOperator(liveVars);
+ projectOperator.setPhysicalOperator(new StreamProjectPOperator());
+ AbstractLogicalOperator exchOp = new ExchangeOperator();
+ exchOp.setPhysicalOperator(new OneToOneExchangePOperator());
+ exchOp.getInputs().add(ropRef);
+
+ assignOperator.getInputs().add(new MutableObject<ILogicalOperator>(exchOp));
+ projectOperator.getInputs().add(new MutableObject<ILogicalOperator>(assignOperator));
+ // set the types
+ context.computeAndSetTypeEnvironmentForOperator(exchOp);
+ context.computeAndSetTypeEnvironmentForOperator(assignOperator);
+ context.computeAndSetTypeEnvironmentForOperator(projectOperator);
+
+ List<Mutable<ILogicalOperator>> parentOpList = childrenToParents.get(ref);
+ for (Mutable<ILogicalOperator> parentOpRef : parentOpList) {
+ AbstractLogicalOperator parentOp = (AbstractLogicalOperator) parentOpRef.getValue();
+ int index = parentOp.getInputs().indexOf(ref);
+ if (parentOp.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+ AbstractLogicalOperator parentOpNext = (AbstractLogicalOperator) childrenToParents
+ .get(parentOpRef).get(0).getValue();
+ if (parentOpNext.isMap()) {
+ index = parentOpNext.getInputs().indexOf(parentOpRef);
+ parentOp = parentOpNext;
+ }
+ }
+
+ AbstractLogicalOperator exchg = new ExchangeOperator();
+ exchg.setPhysicalOperator(new OneToOneExchangePOperator());
+
+ ILogicalOperator childOp = parentOp.getOperatorTag() == LogicalOperatorTag.PROJECT ? assignOperator
+ : projectOperator;
+ if (parentOp.isMap()) {
+ parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(childOp));
+ } else {
+ exchg.getInputs().add(new MutableObject<ILogicalOperator>(childOp));
+ parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(exchg));
+ }
+ context.computeAndSetTypeEnvironmentForOperator(exchg);
+ }
+ }
+ rewritten = true;
+ }
+ return rewritten;
+ }
+
+ private void genCandidates(IOptimizationContext context) throws AlgebricksException {
+ List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
+ while (equivalenceClasses.size() > 0) {
+ previousEquivalenceClasses.clear();
+ for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+ List<Mutable<ILogicalOperator>> candidatesCopy = new ArrayList<Mutable<ILogicalOperator>>();
+ candidatesCopy.addAll(candidates);
+ previousEquivalenceClasses.add(candidatesCopy);
+ }
+ List<Mutable<ILogicalOperator>> currentLevelOpRefs = new ArrayList<Mutable<ILogicalOperator>>();
+ for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+ if (candidates.size() > 0) {
+ for (Mutable<ILogicalOperator> opRef : candidates) {
+ List<Mutable<ILogicalOperator>> refs = childrenToParents.get(opRef);
+ if (refs != null)
+ currentLevelOpRefs.addAll(refs);
+ }
+ }
+ if (currentLevelOpRefs.size() == 0)
+ continue;
+ candidatesGrow(currentLevelOpRefs, candidates);
+ }
+ if (currentLevelOpRefs.size() == 0)
+ break;
+ prune(context);
+ }
+ if (equivalenceClasses.size() < 1 && previousEquivalenceClasses.size() > 0) {
+ equivalenceClasses.addAll(previousEquivalenceClasses);
+ prune(context);
+ }
+ }
+
+ private void topDownMaterialization(List<Mutable<ILogicalOperator>> tops) {
+ List<Mutable<ILogicalOperator>> candidates = new ArrayList<Mutable<ILogicalOperator>>();
+ List<Mutable<ILogicalOperator>> nextLevel = new ArrayList<Mutable<ILogicalOperator>>();
+ for (Mutable<ILogicalOperator> op : tops) {
+ AbstractLogicalOperator aop = (AbstractLogicalOperator) op.getValue();
+ if ((aop.getOperatorTag() == LogicalOperatorTag.INNERJOIN || aop.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN)
+ && !joins.contains(op)) {
+ joins.add(op);
+ }
+ for (Mutable<ILogicalOperator> opRef : op.getValue().getInputs()) {
+ List<Mutable<ILogicalOperator>> opRefList = childrenToParents.get(opRef);
+ if (opRefList == null) {
+ opRefList = new ArrayList<Mutable<ILogicalOperator>>();
+ childrenToParents.put(opRef, opRefList);
+ nextLevel.add(opRef);
+ }
+ opRefList.add(op);
+ }
+ if (op.getValue().getInputs().size() == 0)
+ candidates.add(op);
+ }
+ if (equivalenceClasses.size() > 0) {
+ equivalenceClasses.get(0).addAll(candidates);
+ } else {
+ equivalenceClasses.add(candidates);
+ }
+ if (nextLevel.size() > 0) {
+ topDownMaterialization(nextLevel);
+ }
+ }
+
+ private void candidatesGrow(List<Mutable<ILogicalOperator>> opList, List<Mutable<ILogicalOperator>> candidates) {
+ List<Mutable<ILogicalOperator>> previousCandidates = new ArrayList<Mutable<ILogicalOperator>>();
+ previousCandidates.addAll(candidates);
+ candidates.clear();
+ boolean validCandidate = false;
+ for (Mutable<ILogicalOperator> op : opList) {
+ for (Mutable<ILogicalOperator> inputRef : op.getValue().getInputs()) {
+ validCandidate = false;
+ // if current input is in candidates
+ for (Mutable<ILogicalOperator> candidate : previousCandidates)
+ if (inputRef.getValue().equals(candidate.getValue()))
+ validCandidate = true;
+ // if one input is not in candidates
+ if (!validCandidate)
+ break;
+ }
+ if (!validCandidate)
+ continue;
+ candidates.add(op);
+ }
+ }
+
+ private void prune(IOptimizationContext context) throws AlgebricksException {
+ List<List<Mutable<ILogicalOperator>>> previousEquivalenceClasses = new ArrayList<List<Mutable<ILogicalOperator>>>();
+ for (List<Mutable<ILogicalOperator>> candidates : equivalenceClasses) {
+ List<Mutable<ILogicalOperator>> candidatesCopy = new ArrayList<Mutable<ILogicalOperator>>();
+ candidatesCopy.addAll(candidates);
+ previousEquivalenceClasses.add(candidatesCopy);
+ }
+ equivalenceClasses.clear();
+ for (List<Mutable<ILogicalOperator>> candidates : previousEquivalenceClasses) {
+ boolean[] reserved = new boolean[candidates.size()];
+ for (int i = 0; i < reserved.length; i++)
+ reserved[i] = false;
+ for (int i = candidates.size() - 1; i >= 0; i--) {
+ if (reserved[i] == false) {
+ List<Mutable<ILogicalOperator>> equivalentClass = new ArrayList<Mutable<ILogicalOperator>>();
+ ILogicalOperator candidate = candidates.get(i).getValue();
+ equivalentClass.add(candidates.get(i));
+ for (int j = i - 1; j >= 0; j--) {
+ ILogicalOperator peer = candidates.get(j).getValue();
+ if (IsomorphismUtilities.isOperatorIsomorphic(candidate, peer)) {
+ reserved[i] = true;
+ reserved[j] = true;
+ equivalentClass.add(candidates.get(j));
+ }
+ }
+ if (equivalentClass.size() > 1) {
+ equivalenceClasses.add(equivalentClass);
+ Collections.reverse(equivalentClass);
+ }
+ }
+ }
+ for (int i = candidates.size() - 1; i >= 0; i--) {
+ if (!reserved[i]) {
+ candidates.remove(i);
+ }
+ }
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
new file mode 100644
index 0000000..38032dd
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ExtractGbyExpressionsRule.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+/**
+ * Needed only bc. current Hyrax operators require keys to be fields.
+ *
+ */
+public class ExtractGbyExpressionsRule extends AbstractExtractExprRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+
+ if (context.checkIfInDontApplySet(this, op1)) {
+ return false;
+ }
+ context.addToDontApplySet(this, op1);
+ GroupByOperator g = (GroupByOperator) op1;
+ boolean r1 = gbyExprWasRewritten(g, context);
+ boolean r2 = decorExprWasRewritten(g, context);
+ boolean fired = r1 || r2;
+ if (fired) {
+ context.computeAndSetTypeEnvironmentForOperator(g);
+ }
+ return fired;
+ }
+
+ private boolean gbyExprWasRewritten(GroupByOperator g, IOptimizationContext context) throws AlgebricksException {
+ if (!gbyHasComplexExpr(g)) {
+ return false;
+ }
+ Mutable<ILogicalOperator> opRef2 = g.getInputs().get(0);
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getGroupByList()) {
+ ILogicalExpression expr = gbyPair.second.getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
+ gbyPair.second.setValue(new VariableReferenceExpression(v));
+ }
+ }
+ return true;
+ }
+
+ private boolean decorExprWasRewritten(GroupByOperator g, IOptimizationContext context) throws AlgebricksException {
+ if (!decorHasComplexExpr(g)) {
+ return false;
+ }
+ Mutable<ILogicalOperator> opRef2 = g.getInputs().get(0);
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> decorPair : g.getDecorList()) {
+ ILogicalExpression expr = decorPair.second.getValue();
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ LogicalVariable v = extractExprIntoAssignOpRef(expr, opRef2, context);
+ decorPair.second.setValue(new VariableReferenceExpression(v));
+ }
+ }
+ return true;
+ }
+
+ private boolean gbyHasComplexExpr(GroupByOperator g) {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getGroupByList()) {
+ if (gbyPair.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private boolean decorHasComplexExpr(GroupByOperator g) {
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gbyPair : g.getDecorList()) {
+ if (gbyPair.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
new file mode 100644
index 0000000..99ed003
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/FactorRedundantGroupAndDecorVarsRule.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class FactorRedundantGroupAndDecorVarsRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ GroupByOperator gby = (GroupByOperator) op;
+ Map<LogicalVariable, LogicalVariable> varRhsToLhs = new HashMap<LogicalVariable, LogicalVariable>();
+ boolean gvChanged = factorRedundantRhsVars(gby.getGroupByList(), opRef, varRhsToLhs, context);
+ boolean dvChanged = factorRedundantRhsVars(gby.getDecorList(), opRef, varRhsToLhs, context);
+
+ return gvChanged || dvChanged;
+ }
+
+ private boolean factorRedundantRhsVars(List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> veList,
+ Mutable<ILogicalOperator> opRef, Map<LogicalVariable, LogicalVariable> varRhsToLhs,
+ IOptimizationContext context) throws AlgebricksException {
+ varRhsToLhs.clear();
+ ListIterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = veList.listIterator();
+ boolean changed = false;
+ while (iter.hasNext()) {
+ Pair<LogicalVariable, Mutable<ILogicalExpression>> p = iter.next();
+ if (p.second.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ continue;
+ }
+ LogicalVariable v = GroupByOperator.getDecorVariable(p);
+ LogicalVariable lhs = varRhsToLhs.get(v);
+ if (lhs != null) {
+ if (p.first != null) {
+ AssignOperator assign = new AssignOperator(p.first, new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(lhs)));
+ ILogicalOperator op = opRef.getValue();
+ assign.getInputs().add(new MutableObject<ILogicalOperator>(op));
+ opRef.setValue(assign);
+ context.computeAndSetTypeEnvironmentForOperator(assign);
+ }
+ iter.remove();
+ changed = true;
+ } else {
+ varRhsToLhs.put(v, p.first);
+ }
+ }
+ return changed;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
new file mode 100644
index 0000000..e1d5549
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InferTypesRule.java
@@ -0,0 +1,28 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class InferTypesRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator op = opRef.getValue();
+ if (context.getOutputTypeEnvironment(op) != null) {
+ return false;
+ }
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
new file mode 100644
index 0000000..0cc2b53
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InlineVariablesRule.java
@@ -0,0 +1,344 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.EquivalenceClass;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractLogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class InlineVariablesRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ /**
+ *
+ * Does one big DFS sweep over the plan.
+ *
+ */
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+ return false;
+ }
+ VariableSubstitutionVisitor substVisitor = new VariableSubstitutionVisitor(false);
+ VariableSubstitutionVisitor substVisitorForWrites = new VariableSubstitutionVisitor(true);
+ substVisitor.setContext(context);
+ substVisitorForWrites.setContext(context);
+ Pair<Boolean, Boolean> bb = collectEqClassesAndRemoveRedundantOps(opRef, context, true,
+ new LinkedList<EquivalenceClass>(), substVisitor, substVisitorForWrites);
+ return bb.first;
+ }
+
+ private Pair<Boolean, Boolean> collectEqClassesAndRemoveRedundantOps(Mutable<ILogicalOperator> opRef,
+ IOptimizationContext context, boolean first, List<EquivalenceClass> equivClasses,
+ VariableSubstitutionVisitor substVisitor, VariableSubstitutionVisitor substVisitorForWrites)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ // if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+ // return false;
+ // }
+ if (op.getOperatorTag() == LogicalOperatorTag.UNNEST_MAP) {
+ return new Pair<Boolean, Boolean>(false, false);
+ }
+ boolean modified = false;
+ boolean ecChange = false;
+ int cnt = 0;
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ boolean isOuterInputBranch = op.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN && cnt == 1;
+ List<EquivalenceClass> eqc = isOuterInputBranch ? new LinkedList<EquivalenceClass>() : equivClasses;
+
+ Pair<Boolean, Boolean> bb = (collectEqClassesAndRemoveRedundantOps(i, context, false, eqc, substVisitor,
+ substVisitorForWrites));
+
+ if (bb.first) {
+ modified = true;
+ }
+ if (bb.second) {
+ ecChange = true;
+ }
+
+ if (isOuterInputBranch) {
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest("--- Equivalence classes for inner branch of outer op.: "
+ + eqc + "\n");
+ }
+ for (EquivalenceClass ec : eqc) {
+ if (!ec.representativeIsConst()) {
+ equivClasses.add(ec);
+ }
+ }
+ }
+
+ ++cnt;
+ }
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans n = (AbstractOperatorWithNestedPlans) op;
+ List<EquivalenceClass> eqc = equivClasses;
+ if (n.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+ eqc = new LinkedList<EquivalenceClass>();
+ } else {
+ eqc = equivClasses;
+ }
+ for (ILogicalPlan p : n.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ Pair<Boolean, Boolean> bb = collectEqClassesAndRemoveRedundantOps(r, context, false, eqc,
+ substVisitor, substVisitorForWrites);
+ if (bb.first) {
+ modified = true;
+ }
+ if (bb.second) {
+ ecChange = true;
+ }
+ }
+ }
+ }
+ // we assume a variable is assigned a value only once
+ if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator a = (AssignOperator) op;
+ ILogicalExpression rhs = a.getExpressions().get(0).getValue();
+ if (rhs.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ LogicalVariable varLeft = a.getVariables().get(0);
+ VariableReferenceExpression varRef = (VariableReferenceExpression) rhs;
+ LogicalVariable varRight = varRef.getVariableReference();
+
+ EquivalenceClass ecRight = findEquivClass(varRight, equivClasses);
+ if (ecRight != null) {
+ ecRight.addMember(varLeft);
+ } else {
+ List<LogicalVariable> m = new LinkedList<LogicalVariable>();
+ m.add(varRight);
+ m.add(varLeft);
+ EquivalenceClass ec = new EquivalenceClass(m, varRight);
+ equivClasses.add(ec);
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.finest("--- New equivalence class: " + ec + "\n");
+ }
+ }
+ ecChange = true;
+ } else if (((AbstractLogicalExpression) rhs).getExpressionTag() == LogicalExpressionTag.CONSTANT) {
+ LogicalVariable varLeft = a.getVariables().get(0);
+ List<LogicalVariable> m = new LinkedList<LogicalVariable>();
+ m.add(varLeft);
+ EquivalenceClass ec = new EquivalenceClass(m, (ConstantExpression) rhs);
+ // equivClassesForParent.add(ec);
+ equivClasses.add(ec);
+ ecChange = true;
+ }
+ } else if (op.getOperatorTag() == LogicalOperatorTag.GROUP && !(context.checkIfInDontApplySet(this, op))) {
+ GroupByOperator group = (GroupByOperator) op;
+ Pair<Boolean, Boolean> r1 = processVarExprPairs(group.getGroupByList(), equivClasses);
+ Pair<Boolean, Boolean> r2 = processVarExprPairs(group.getDecorList(), equivClasses);
+ modified = modified || r1.first || r2.first;
+ ecChange = r1.second || r2.second;
+ }
+ if (op.getOperatorTag() == LogicalOperatorTag.PROJECT) {
+ assignVarsNeededByProject((ProjectOperator) op, equivClasses, context);
+ } else {
+ if (op.getOperatorTag() == LogicalOperatorTag.WRITE) {
+ substVisitorForWrites.setEquivalenceClasses(equivClasses);
+ if (op.acceptExpressionTransform(substVisitorForWrites)) {
+ modified = true;
+ }
+ } else {
+ substVisitor.setEquivalenceClasses(equivClasses);
+ if (op.acceptExpressionTransform(substVisitor)) {
+ modified = true;
+ if (op.getOperatorTag() == LogicalOperatorTag.GROUP) {
+ GroupByOperator group = (GroupByOperator) op;
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> gp : group.getGroupByList()) {
+ if (gp.first != null
+ && gp.second.getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ LogicalVariable gv = ((VariableReferenceExpression) gp.second.getValue())
+ .getVariableReference();
+ Iterator<Pair<LogicalVariable, Mutable<ILogicalExpression>>> iter = group
+ .getDecorList().iterator();
+ while (iter.hasNext()) {
+ Pair<LogicalVariable, Mutable<ILogicalExpression>> dp = iter.next();
+ if (dp.first == null
+ && dp.second.getValue().getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ LogicalVariable dv = ((VariableReferenceExpression) dp.second.getValue())
+ .getVariableReference();
+ if (dv == gv) {
+ // The decor variable is redundant,
+ // since it is
+ // propagated as a grouping
+ // variable.
+ EquivalenceClass ec1 = findEquivClass(gv, equivClasses);
+ if (ec1 != null) {
+ ec1.addMember(gp.first);
+ ec1.setVariableRepresentative(gp.first);
+ } else {
+ List<LogicalVariable> varList = new ArrayList<LogicalVariable>();
+ varList.add(gp.first);
+ varList.add(gv);
+ ec1 = new EquivalenceClass(varList, gp.first);
+ }
+ iter.remove();
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ return new Pair<Boolean, Boolean>(modified, ecChange);
+ }
+
+ private Pair<Boolean, Boolean> processVarExprPairs(
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> vePairs, List<EquivalenceClass> equivClasses) {
+ boolean ecFromGroup = false;
+ boolean modified = false;
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : vePairs) {
+ ILogicalExpression expr = p.second.getValue();
+ if (p.first != null && expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+ LogicalVariable rhsVar = varRef.getVariableReference();
+ ecFromGroup = true;
+ EquivalenceClass ecRight = findEquivClass(rhsVar, equivClasses);
+ if (ecRight != null) {
+ LogicalVariable replacingVar = ecRight.getVariableRepresentative();
+ if (replacingVar != null && replacingVar != rhsVar) {
+ varRef.setVariable(replacingVar);
+ modified = true;
+ }
+ }
+ }
+ }
+ return new Pair<Boolean, Boolean>(modified, ecFromGroup);
+ }
+
+ // Instead of doing this, we could make Projection to be more expressive and
+ // also take constants (or even expression), at the expense of a more
+ // complex project push down.
+ private void assignVarsNeededByProject(ProjectOperator op, List<EquivalenceClass> equivClasses,
+ IOptimizationContext context) throws AlgebricksException {
+ List<LogicalVariable> prVars = op.getVariables();
+ int sz = prVars.size();
+ for (int i = 0; i < sz; i++) {
+ EquivalenceClass ec = findEquivClass(prVars.get(i), equivClasses);
+ if (ec != null) {
+ if (!ec.representativeIsConst()) {
+ prVars.set(i, ec.getVariableRepresentative());
+ }
+ }
+ }
+ }
+
+ private final static EquivalenceClass findEquivClass(LogicalVariable var, List<EquivalenceClass> equivClasses) {
+ for (EquivalenceClass ec : equivClasses) {
+ if (ec.contains(var)) {
+ return ec;
+ }
+ }
+ return null;
+ }
+
+ private class VariableSubstitutionVisitor implements ILogicalExpressionReferenceTransform {
+ private List<EquivalenceClass> equivClasses;
+ private IOptimizationContext context;
+ private final boolean doNotSubstWithConst;
+
+ public VariableSubstitutionVisitor(boolean doNotSubstWithConst) {
+ this.doNotSubstWithConst = doNotSubstWithConst;
+ }
+
+ public void setContext(IOptimizationContext context) {
+ this.context = context;
+ }
+
+ public void setEquivalenceClasses(List<EquivalenceClass> equivClasses) {
+ this.equivClasses = equivClasses;
+ }
+
+ @Override
+ public boolean transform(Mutable<ILogicalExpression> exprRef) {
+ ILogicalExpression e = exprRef.getValue();
+ switch (((AbstractLogicalExpression) e).getExpressionTag()) {
+ case VARIABLE: {
+ // look for a required substitution
+ LogicalVariable var = ((VariableReferenceExpression) e).getVariableReference();
+ if (context.shouldNotBeInlined(var)) {
+ return false;
+ }
+ EquivalenceClass ec = findEquivClass(var, equivClasses);
+ if (ec == null) {
+ return false;
+ }
+ if (ec.representativeIsConst()) {
+ if (doNotSubstWithConst) {
+ return false;
+ }
+ exprRef.setValue(ec.getConstRepresentative());
+ return true;
+ } else {
+ LogicalVariable r = ec.getVariableRepresentative();
+ if (!r.equals(var)) {
+ exprRef.setValue(new VariableReferenceExpression(r));
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+ case FUNCTION_CALL: {
+ AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) e;
+ boolean m = false;
+ for (Mutable<ILogicalExpression> arg : fce.getArguments()) {
+ if (transform(arg)) {
+ m = true;
+ }
+ }
+ return m;
+ }
+ default: {
+ return false;
+ }
+ }
+ }
+
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
new file mode 100644
index 0000000..25df119
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertOuterJoinRule.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.Iterator;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class InsertOuterJoinRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+ if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+ return false;
+ }
+ SubplanOperator subplan = (SubplanOperator) op0;
+
+ Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
+ ILogicalPlan p = null;
+ while (plansIter.hasNext()) {
+ p = plansIter.next();
+ }
+ if (p == null) {
+ return false;
+ }
+ if (p.getRoots().size() != 1) {
+ return false;
+ }
+ Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
+ Mutable<ILogicalOperator> opUnder = subplan.getInputs().get(0);
+
+ if (OperatorPropertiesUtil.isNullTest((AbstractLogicalOperator) opUnder.getValue())) {
+ return false;
+ }
+
+ switch (op1.getOperatorTag()) {
+ case INNERJOIN: {
+ InnerJoinOperator join = (InnerJoinOperator) op1;
+ Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
+ Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
+ Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
+ if (ntsRef == null) {
+ ntsRef = getNtsAtEndOfPipeline(rightRef);
+ if (ntsRef == null) {
+ return false;
+ } else {
+ Mutable<ILogicalOperator> t = leftRef;
+ leftRef = rightRef;
+ rightRef = t;
+ }
+ }
+ ntsRef.setValue(opUnder.getValue());
+ LeftOuterJoinOperator loj = new LeftOuterJoinOperator(join.getCondition());
+ loj.getInputs().add(leftRef);
+ loj.getInputs().add(rightRef);
+ opRef.setValue(loj);
+ context.computeAndSetTypeEnvironmentForOperator(loj);
+ return true;
+ }
+ case LEFTOUTERJOIN: {
+ LeftOuterJoinOperator join = (LeftOuterJoinOperator) op1;
+ Mutable<ILogicalOperator> leftRef = join.getInputs().get(0);
+ Mutable<ILogicalOperator> ntsRef = getNtsAtEndOfPipeline(leftRef);
+ if (ntsRef == null) {
+ return false;
+ }
+ ntsRef.setValue(opUnder.getValue());
+ opRef.setValue(join);
+ context.computeAndSetTypeEnvironmentForOperator(join);
+ return true;
+ }
+ default: {
+ return false;
+ }
+ }
+ }
+
+ private Mutable<ILogicalOperator> getNtsAtEndOfPipeline(Mutable<ILogicalOperator> opRef) {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ return opRef;
+ }
+ if (op.getInputs().size() != 1) {
+ return null;
+ }
+ return getNtsAtEndOfPipeline(op.getInputs().get(0));
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
new file mode 100644
index 0000000..337117e
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/InsertProjectBeforeUnionRule.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Triple;
+
+public class InsertProjectBeforeUnionRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ return false;
+ }
+
+ /**
+ * When the input schema to WriteOperator is different from the output
+ * schema in terms of variable order, add a project operator to get the
+ * write order
+ */
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.UNIONALL) {
+ return false;
+ }
+ UnionAllOperator opUnion = (UnionAllOperator) op;
+ List<Triple<LogicalVariable, LogicalVariable, LogicalVariable>> varMap = opUnion.getVariableMappings();
+ ArrayList<LogicalVariable> usedVariablesFromOne = new ArrayList<LogicalVariable>();
+ ArrayList<LogicalVariable> usedVariablesFromTwo = new ArrayList<LogicalVariable>();
+
+ for (Triple<LogicalVariable, LogicalVariable, LogicalVariable> triple : varMap) {
+ usedVariablesFromOne.add(triple.first);
+ usedVariablesFromTwo.add(triple.second);
+ }
+
+ ArrayList<LogicalVariable> inputSchemaOne = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(opUnion.getInputs().get(0).getValue(), inputSchemaOne);
+
+ ArrayList<LogicalVariable> inputSchemaTwo = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(opUnion.getInputs().get(1).getValue(), inputSchemaTwo);
+
+ boolean rewritten = false;
+ if (!isIdentical(usedVariablesFromOne, inputSchemaOne)) {
+ insertProjectOperator(opUnion, 0, usedVariablesFromOne, context);
+ rewritten = true;
+ }
+ if (!isIdentical(usedVariablesFromTwo, inputSchemaTwo)) {
+ insertProjectOperator(opUnion, 1, usedVariablesFromTwo, context);
+ rewritten = true;
+ }
+ return rewritten;
+ }
+
+ private void insertProjectOperator(UnionAllOperator opUnion, int branch, ArrayList<LogicalVariable> usedVariables,
+ IOptimizationContext context) throws AlgebricksException {
+ ProjectOperator projectOp = new ProjectOperator(usedVariables);
+ ILogicalOperator parentOp = opUnion.getInputs().get(branch).getValue();
+ projectOp.getInputs().add(new MutableObject<ILogicalOperator>(parentOp));
+ opUnion.getInputs().get(branch).setValue(projectOp);
+ projectOp.setPhysicalOperator(new StreamProjectPOperator());
+ context.computeAndSetTypeEnvironmentForOperator(parentOp);
+ }
+
+ private boolean isIdentical(List<LogicalVariable> finalSchema, List<LogicalVariable> inputSchema)
+ throws AlgebricksException {
+ int finalSchemaSize = finalSchema.size();
+ int inputSchemaSize = inputSchema.size();
+ if (finalSchemaSize != inputSchemaSize) {
+ return false;
+ }
+ for (int i = 0; i < finalSchemaSize; i++) {
+ LogicalVariable var1 = finalSchema.get(i);
+ LogicalVariable var2 = inputSchema.get(i);
+ if (!var1.equals(var2))
+ return false;
+ }
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
new file mode 100644
index 0000000..0e12bcc
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroHashPartitionMergeExchange.java
@@ -0,0 +1,52 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HashPartitionMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SortMergeExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.OrderColumn;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class IntroHashPartitionMergeExchange implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+ if (op1.getPhysicalOperator() == null
+ || op1.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.HASH_PARTITION_EXCHANGE) {
+ return false;
+ }
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+ if (op2.getPhysicalOperator() == null
+ || op2.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.SORT_MERGE_EXCHANGE) {
+ return false;
+ }
+ HashPartitionExchangePOperator hpe = (HashPartitionExchangePOperator) op1.getPhysicalOperator();
+ SortMergeExchangePOperator sme = (SortMergeExchangePOperator) op2.getPhysicalOperator();
+ List<OrderColumn> ocList = new ArrayList<OrderColumn>();
+ for (OrderColumn oc : sme.getSortColumns()) {
+ ocList.add(oc);
+ }
+ HashPartitionMergeExchangePOperator hpme = new HashPartitionMergeExchangePOperator(ocList, hpe.getHashFields(),
+ hpe.getDomain());
+ op1.setPhysicalOperator(hpme);
+ op1.getInputs().get(0).setValue(op2.getInputs().get(0).getValue());
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
new file mode 100644
index 0000000..acff2de
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroJoinInsideSubplanRule.java
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.ListIterator;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+
+public class IntroJoinInsideSubplanRule extends AbstractDecorrelationRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+ if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+ return false;
+ }
+ SubplanOperator subplan = (SubplanOperator) op0;
+
+ Mutable<ILogicalOperator> leftRef = subplan.getInputs().get(0);
+ if (((AbstractLogicalOperator) leftRef.getValue()).getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE) {
+ return false;
+ }
+
+ ListIterator<ILogicalPlan> plansIter = subplan.getNestedPlans().listIterator();
+ ILogicalPlan p = null;
+ while (plansIter.hasNext()) {
+ p = plansIter.next();
+ }
+ if (p == null) {
+ return false;
+ }
+ if (p.getRoots().size() != 1) {
+ return false;
+ }
+ Mutable<ILogicalOperator> opRef1 = p.getRoots().get(0);
+
+ while (true) {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef1.getValue();
+ if (op1.getInputs().size() != 1) {
+ return false;
+ }
+ if (op1.getOperatorTag() == LogicalOperatorTag.SELECT) {
+ Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.SELECT && descOrSelfIsScanOrJoin(op2)) {
+ Set<LogicalVariable> free2 = new HashSet<LogicalVariable>();
+ OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op2, free2);
+ if (free2.isEmpty()) {
+ Set<LogicalVariable> free1 = new HashSet<LogicalVariable>();
+ OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(op1, free1);
+ if (!free1.isEmpty()) {
+ OperatorManipulationUtil.ntsToEts(op2Ref, context);
+ NestedTupleSourceOperator nts = new NestedTupleSourceOperator(
+ new MutableObject<ILogicalOperator>(subplan));
+ Mutable<ILogicalOperator> ntsRef = new MutableObject<ILogicalOperator>(nts);
+ Mutable<ILogicalOperator> innerRef = new MutableObject<ILogicalOperator>(op2);
+ InnerJoinOperator join = new InnerJoinOperator(new MutableObject<ILogicalExpression>(
+ ConstantExpression.TRUE), ntsRef, innerRef);
+ op2Ref.setValue(join);
+ context.computeAndSetTypeEnvironmentForOperator(nts);
+ context.computeAndSetTypeEnvironmentForOperator(join);
+ return true;
+ }
+ }
+ }
+ }
+ opRef1 = op1.getInputs().get(0);
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java
new file mode 100644
index 0000000..c05572b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceCombinerRule.java
@@ -0,0 +1,323 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AggregateFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class IntroduceCombinerRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (context.checkIfInDontApplySet(this, op)) {
+ return false;
+ }
+ context.addToDontApplySet(this, op);
+ if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ GroupByOperator gbyOp = (GroupByOperator) op;
+ if (gbyOp.getExecutionMode() != ExecutionMode.PARTITIONED) {
+ return false;
+ }
+
+ Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap = new HashMap<AggregateFunctionCallExpression, SimilarAggregatesInfo>();
+ BookkeepingInfo bi = new BookkeepingInfo();
+ bi.toReplaceMap = toReplaceMap;
+ bi.modifGbyMap = new HashMap<GroupByOperator, List<LogicalVariable>>();
+
+ GroupByOperator newGbyOp = opToPush(gbyOp, bi, context);
+ if (newGbyOp == null) {
+ return false;
+ }
+
+ for (Map.Entry<AggregateFunctionCallExpression, SimilarAggregatesInfo> entry : toReplaceMap.entrySet()) {
+ SimilarAggregatesInfo sai = entry.getValue();
+ for (AggregateExprInfo aei : sai.simAggs) {
+ AbstractFunctionCallExpression afce = (AbstractFunctionCallExpression) aei.aggExprRef.getValue();
+ afce.setFunctionInfo(aei.newFunInfo);
+ afce.getArguments().clear();
+ afce.getArguments().add(new MutableObject<ILogicalExpression>(sai.stepOneResult));
+ }
+ }
+
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyOp.getDecorList()) {
+ LogicalVariable newDecorVar = context.newVar();
+ newGbyOp.addDecorExpression(newDecorVar, p.second.getValue());
+ p.second.setValue(new VariableReferenceExpression(newDecorVar));
+ }
+ newGbyOp.setExecutionMode(ExecutionMode.LOCAL);
+ Object v = gbyOp.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY);
+ newGbyOp.getAnnotations().put(OperatorAnnotations.USE_HASH_GROUP_BY, v);
+
+ Object v2 = gbyOp.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY);
+ newGbyOp.getAnnotations().put(OperatorAnnotations.USE_EXTERNAL_GROUP_BY, v2);
+
+ List<LogicalVariable> propagatedVars = new LinkedList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(newGbyOp, propagatedVars);
+
+ Set<LogicalVariable> freeVars = new HashSet<LogicalVariable>();
+ OperatorPropertiesUtil.getFreeVariablesInSubplans(gbyOp, freeVars);
+
+ for (LogicalVariable var : freeVars) {
+ if (!propagatedVars.contains(var)) {
+ LogicalVariable newDecorVar = context.newVar();
+ newGbyOp.addDecorExpression(newDecorVar, new VariableReferenceExpression(var));
+ VariableUtilities.substituteVariables(gbyOp.getNestedPlans().get(0).getRoots().get(0).getValue(), var,
+ newDecorVar, context);
+ }
+ }
+
+ Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
+ opRef3.setValue(newGbyOp);
+ typeGby(newGbyOp, context);
+ typeGby(gbyOp, context);
+ return true;
+ }
+
+ private void typeGby(AbstractOperatorWithNestedPlans op, IOptimizationContext context) throws AlgebricksException {
+ for (ILogicalPlan p : op.getNestedPlans()) {
+ OperatorPropertiesUtil.typePlan(p, context);
+ }
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ }
+
+ private GroupByOperator opToPush(GroupByOperator gbyOp, BookkeepingInfo bi, IOptimizationContext context)
+ throws AlgebricksException {
+
+ Mutable<ILogicalOperator> opRef3 = gbyOp.getInputs().get(0);
+ ILogicalOperator op3 = opRef3.getValue();
+ GroupByOperator newGbyOp = new GroupByOperator();
+ newGbyOp.getInputs().add(new MutableObject<ILogicalOperator>(op3));
+ // copy annotations
+ Map<String, Object> annotations = newGbyOp.getAnnotations();
+ for (Entry<String, Object> a : gbyOp.getAnnotations().entrySet())
+ annotations.put(a.getKey(), a.getValue());
+
+ List<LogicalVariable> gbyVars = gbyOp.getGbyVarList();
+
+ for (ILogicalPlan p : gbyOp.getNestedPlans()) {
+ Pair<Boolean, ILogicalPlan> bip = tryToPushSubplan(p, gbyOp, newGbyOp, bi, gbyVars, context);
+ if (!bip.first) {
+ // for now, if we cannot push everything, give up
+ return null;
+ }
+ ILogicalPlan pushedSubplan = bip.second;
+ if (pushedSubplan != null) {
+ newGbyOp.getNestedPlans().add(pushedSubplan);
+ }
+ }
+
+ ArrayList<LogicalVariable> newOpGbyList = new ArrayList<LogicalVariable>();
+ ArrayList<LogicalVariable> replGbyList = new ArrayList<LogicalVariable>();
+ // find maximal sequence of variable
+ for (Map.Entry<GroupByOperator, List<LogicalVariable>> e : bi.modifGbyMap.entrySet()) {
+ List<LogicalVariable> varList = e.getValue();
+ boolean see1 = true;
+ int sz1 = newOpGbyList.size();
+ int i = 0;
+ for (LogicalVariable v : varList) {
+ if (see1) {
+ if (i < sz1) {
+ LogicalVariable v2 = newOpGbyList.get(i);
+ if (v != v2) {
+ // cannot linearize
+ return null;
+ }
+ } else {
+ see1 = false;
+ newOpGbyList.add(v);
+ replGbyList.add(context.newVar());
+ }
+ i++;
+ } else {
+ newOpGbyList.add(v);
+ replGbyList.add(context.newVar());
+ }
+ }
+ }
+ // set the vars in the new op
+ int n = newOpGbyList.size();
+ for (int i = 0; i < n; i++) {
+ newGbyOp.addGbyExpression(replGbyList.get(i), new VariableReferenceExpression(newOpGbyList.get(i)));
+ VariableUtilities.substituteVariables(gbyOp, newOpGbyList.get(i), replGbyList.get(i), false, context);
+ }
+ return newGbyOp;
+ }
+
+ private Pair<Boolean, ILogicalPlan> tryToPushSubplan(ILogicalPlan p, GroupByOperator oldGbyOp,
+ GroupByOperator newGbyOp, BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context) {
+ List<Mutable<ILogicalOperator>> pushedRoots = new ArrayList<Mutable<ILogicalOperator>>();
+ List<Mutable<ILogicalOperator>> toPushR = new ArrayList<Mutable<ILogicalOperator>>();
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ if (!tryToPushRoot(r, oldGbyOp, newGbyOp, bi, gbyVars, context, toPushR)) {
+ // for now, if we cannot push everything, give up
+ return new Pair<Boolean, ILogicalPlan>(false, null);
+ }
+ }
+ for (Mutable<ILogicalOperator> root : toPushR) {
+ pushedRoots.add(root);
+ }
+ if (pushedRoots.isEmpty()) {
+ return new Pair<Boolean, ILogicalPlan>(true, null);
+ } else {
+ return new Pair<Boolean, ILogicalPlan>(true, new ALogicalPlanImpl(pushedRoots));
+ }
+ }
+
+ private boolean tryToPushRoot(Mutable<ILogicalOperator> r, GroupByOperator oldGbyOp, GroupByOperator newGbyOp,
+ BookkeepingInfo bi, List<LogicalVariable> gbyVars, IOptimizationContext context,
+ List<Mutable<ILogicalOperator>> toPushAccumulate) {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) r.getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ AggregateOperator initAgg = (AggregateOperator) op1;
+ Pair<Boolean, Mutable<ILogicalOperator>> pOpRef = tryToPushAgg(initAgg, newGbyOp, bi.toReplaceMap, context);
+ if (!pOpRef.first) {
+ return false;
+ }
+ Mutable<ILogicalOperator> opRef = pOpRef.second;
+ if (opRef != null) {
+ toPushAccumulate.add(opRef);
+ }
+ bi.modifGbyMap.put(oldGbyOp, gbyVars);
+ return true;
+ } else {
+ while (op2.getOperatorTag() != LogicalOperatorTag.GROUP && op2.getInputs().size() == 1) {
+ op2 = (AbstractLogicalOperator) op2.getInputs().get(0).getValue();
+ }
+ if (op2.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ GroupByOperator nestedGby = (GroupByOperator) op2;
+ List<LogicalVariable> gbyVars2 = nestedGby.getGbyVarList();
+ List<LogicalVariable> concatGbyVars = new ArrayList<LogicalVariable>(gbyVars);
+ concatGbyVars.addAll(gbyVars2);
+ for (ILogicalPlan p : nestedGby.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r2 : p.getRoots()) {
+ if (!tryToPushRoot(r2, nestedGby, newGbyOp, bi, concatGbyVars, context, toPushAccumulate)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ }
+
+ private Pair<Boolean, Mutable<ILogicalOperator>> tryToPushAgg(AggregateOperator initAgg, GroupByOperator newGbyOp,
+ Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap, IOptimizationContext context) {
+
+ ArrayList<LogicalVariable> pushedVars = new ArrayList<LogicalVariable>();
+ ArrayList<Mutable<ILogicalExpression>> pushedExprs = new ArrayList<Mutable<ILogicalExpression>>();
+
+ List<LogicalVariable> initVars = initAgg.getVariables();
+ List<Mutable<ILogicalExpression>> initExprs = initAgg.getExpressions();
+ int sz = initVars.size();
+ for (int i = 0; i < sz; i++) {
+ AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) initExprs.get(i).getValue();
+ if (!aggFun.isTwoStep()) {
+ return new Pair<Boolean, Mutable<ILogicalOperator>>(false, null);
+ }
+ }
+
+ boolean haveAggToReplace = false;
+ for (int i = 0; i < sz; i++) {
+ Mutable<ILogicalExpression> expRef = initExprs.get(i);
+ AggregateFunctionCallExpression aggFun = (AggregateFunctionCallExpression) expRef.getValue();
+ IFunctionInfo fi1 = aggFun.getStepOneAggregate();
+ List<Mutable<ILogicalExpression>> newArgs = new ArrayList<Mutable<ILogicalExpression>>(aggFun
+ .getArguments().size());
+ for (Mutable<ILogicalExpression> er : aggFun.getArguments()) {
+ newArgs.add(new MutableObject<ILogicalExpression>(er.getValue().cloneExpression()));
+ }
+ // AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
+ // pushedExprs.add(new Mutable<ILogicalExpression>(aggLocal));
+
+ IFunctionInfo fi2 = aggFun.getStepTwoAggregate();
+
+ SimilarAggregatesInfo inf = toReplaceMap.get(aggFun);
+ if (inf == null) {
+ inf = new SimilarAggregatesInfo();
+ LogicalVariable newAggVar = context.newVar();
+ pushedVars.add(newAggVar);
+ inf.stepOneResult = new VariableReferenceExpression(newAggVar);
+ inf.simAggs = new ArrayList<AggregateExprInfo>();
+ toReplaceMap.put(aggFun, inf);
+ AggregateFunctionCallExpression aggLocal = new AggregateFunctionCallExpression(fi1, false, newArgs);
+ pushedExprs.add(new MutableObject<ILogicalExpression>(aggLocal));
+ }
+ AggregateExprInfo aei = new AggregateExprInfo();
+ aei.aggExprRef = expRef;
+ aei.newFunInfo = fi2;
+ inf.simAggs.add(aei);
+ haveAggToReplace = true;
+ }
+
+ if (!pushedVars.isEmpty()) {
+ AggregateOperator pushedAgg = new AggregateOperator(pushedVars, pushedExprs);
+ pushedAgg.setExecutionMode(ExecutionMode.LOCAL);
+ NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(newGbyOp));
+ nts.setExecutionMode(ExecutionMode.LOCAL);
+ pushedAgg.getInputs().add(new MutableObject<ILogicalOperator>(nts));
+ return new Pair<Boolean, Mutable<ILogicalOperator>>(true, new MutableObject<ILogicalOperator>(pushedAgg));
+ } else {
+ return new Pair<Boolean, Mutable<ILogicalOperator>>(haveAggToReplace, null);
+ }
+ }
+
+ private class SimilarAggregatesInfo {
+ ILogicalExpression stepOneResult;
+ List<AggregateExprInfo> simAggs;
+ }
+
+ private class AggregateExprInfo {
+ Mutable<ILogicalExpression> aggExprRef;
+ IFunctionInfo newFunInfo;
+ }
+
+ private class BookkeepingInfo {
+ Map<AggregateFunctionCallExpression, SimilarAggregatesInfo> toReplaceMap;
+ Map<GroupByOperator, List<LogicalVariable>> modifGbyMap;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java
new file mode 100644
index 0000000..34060c2
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForStandaloneAggregRule.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+/**
+ *
+ * When aggregates appear w/o group-by, a default group by a constant is
+ * introduced.
+ *
+ */
+
+public class IntroduceGroupByForStandaloneAggregRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ return false;
+ }
+ Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+
+ AssignOperator assign = (AssignOperator) op;
+ AggregateOperator agg = (AggregateOperator) op2;
+ if (agg.getVariables().size() != 1) {
+ return false;
+ }
+ LogicalVariable aggVar = agg.getVariables().get(0);
+ List<LogicalVariable> used = new LinkedList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(assign, used);
+ if (used.contains(aggVar)) {
+ Mutable<ILogicalOperator> opRef3 = op2.getInputs().get(0);
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> groupByList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+ LogicalVariable gbyVar = context.newVar();
+ // ILogicalExpression constOne = new ConstantExpression(new
+ // IntegerLiteral(new Integer(1)));
+ groupByList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(gbyVar,
+ new MutableObject<ILogicalExpression>(ConstantExpression.TRUE)));
+ NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>());
+ List<Mutable<ILogicalOperator>> aggInpList = agg.getInputs();
+ aggInpList.clear();
+ aggInpList.add(new MutableObject<ILogicalOperator>(nts));
+ ILogicalPlan np1 = new ALogicalPlanImpl(opRef2);
+ ArrayList<ILogicalPlan> nestedPlans = new ArrayList<ILogicalPlan>();
+ nestedPlans.add(np1);
+ GroupByOperator gbyOp = new GroupByOperator(groupByList,
+ new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>(), nestedPlans);
+ Mutable<ILogicalOperator> opRefGby = new MutableObject<ILogicalOperator>(gbyOp);
+ nts.getDataSourceReference().setValue(gbyOp);
+ gbyOp.getInputs().add(opRef3);
+ List<Mutable<ILogicalOperator>> asgnInpList = assign.getInputs();
+ context.computeAndSetTypeEnvironmentForOperator(nts);
+ context.computeAndSetTypeEnvironmentForOperator(agg);
+ context.computeAndSetTypeEnvironmentForOperator(gbyOp);
+ asgnInpList.clear();
+ asgnInpList.add(opRefGby);
+ return true;
+ }
+ return false;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
new file mode 100644
index 0000000..87297e7
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IntroduceGroupByForSubplanRule.java
@@ -0,0 +1,270 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.FunctionalDependency;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
+
+public class IntroduceGroupByForSubplanRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+ if (op0.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+ return false;
+ }
+ SubplanOperator subplan = (SubplanOperator) op0;
+
+ Iterator<ILogicalPlan> plansIter = subplan.getNestedPlans().iterator();
+ ILogicalPlan p = null;
+ while (plansIter.hasNext()) {
+ p = plansIter.next();
+ }
+ if (p == null) {
+ return false;
+ }
+ if (p.getRoots().size() != 1) {
+ return false;
+ }
+ Mutable<ILogicalOperator> subplanRoot = p.getRoots().get(0);
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) subplanRoot.getValue();
+
+ Mutable<ILogicalOperator> botRef = subplanRoot;
+ AbstractLogicalOperator op2;
+ // Project is optional
+ if (op1.getOperatorTag() != LogicalOperatorTag.PROJECT) {
+ op2 = op1;
+ } else {
+ ProjectOperator project = (ProjectOperator) op1;
+ botRef = project.getInputs().get(0);
+ op2 = (AbstractLogicalOperator) botRef.getValue();
+ }
+ if (op2.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ AggregateOperator aggregate = (AggregateOperator) op2;
+
+ Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+ VariableUtilities.getUsedVariables(aggregate, free);
+
+ Mutable<ILogicalOperator> op3Ref = aggregate.getInputs().get(0);
+ AbstractLogicalOperator op3 = (AbstractLogicalOperator) op3Ref.getValue();
+
+ while (op3.getInputs().size() == 1) {
+ Set<LogicalVariable> prod = new HashSet<LogicalVariable>();
+ VariableUtilities.getProducedVariables(op3, prod);
+ free.removeAll(prod);
+ VariableUtilities.getUsedVariables(op3, free);
+ botRef = op3Ref;
+ op3Ref = op3.getInputs().get(0);
+ op3 = (AbstractLogicalOperator) op3Ref.getValue();
+ }
+
+ if (op3.getOperatorTag() != LogicalOperatorTag.INNERJOIN
+ && op3.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
+ return false;
+ }
+ AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op3;
+ if (join.getCondition().getValue() == ConstantExpression.TRUE) {
+ return false;
+ }
+ VariableUtilities.getUsedVariables(join, free);
+
+ AbstractLogicalOperator b0 = (AbstractLogicalOperator) join.getInputs().get(0).getValue();
+ // see if there's an NTS at the end of the pipeline
+ NestedTupleSourceOperator outerNts = getNts(b0);
+ if (outerNts == null) {
+ AbstractLogicalOperator b1 = (AbstractLogicalOperator) join.getInputs().get(1).getValue();
+ outerNts = getNts(b1);
+ if (outerNts == null) {
+ return false;
+ }
+ }
+
+ Set<LogicalVariable> pkVars = computeGbyVars(outerNts, free, context);
+ if (pkVars == null || pkVars.size() < 1) {
+ // could not group only by primary keys
+ return false;
+ }
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine("Found FD for introducing group-by: " + pkVars);
+
+ Mutable<ILogicalOperator> rightRef = join.getInputs().get(1);
+ LogicalVariable testForNull = null;
+ AbstractLogicalOperator right = (AbstractLogicalOperator) rightRef.getValue();
+ switch (right.getOperatorTag()) {
+ case UNNEST: {
+ UnnestOperator innerUnnest = (UnnestOperator) right;
+ // Select [ $y != null ]
+ testForNull = innerUnnest.getVariable();
+ break;
+ }
+ case DATASOURCESCAN: {
+ DataSourceScanOperator innerScan = (DataSourceScanOperator) right;
+ // Select [ $y != null ]
+ if (innerScan.getVariables().size() == 1) {
+ testForNull = innerScan.getVariables().get(0);
+ }
+ break;
+ }
+ }
+ if (testForNull == null) {
+ testForNull = context.newVar();
+ AssignOperator tmpAsgn = new AssignOperator(testForNull, new MutableObject<ILogicalExpression>(
+ ConstantExpression.TRUE));
+ tmpAsgn.getInputs().add(new MutableObject<ILogicalOperator>(rightRef.getValue()));
+ rightRef.setValue(tmpAsgn);
+ context.computeAndSetTypeEnvironmentForOperator(tmpAsgn);
+ }
+
+ IFunctionInfo finfoEq = AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.IS_NULL);
+ ILogicalExpression isNullTest = new ScalarFunctionCallExpression(finfoEq,
+ new MutableObject<ILogicalExpression>(new VariableReferenceExpression(testForNull)));
+ IFunctionInfo finfoNot = AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.NOT);
+ ScalarFunctionCallExpression nonNullTest = new ScalarFunctionCallExpression(finfoNot,
+ new MutableObject<ILogicalExpression>(isNullTest));
+ SelectOperator selectNonNull = new SelectOperator(new MutableObject<ILogicalExpression>(nonNullTest));
+ GroupByOperator g = new GroupByOperator();
+ Mutable<ILogicalOperator> newSubplanRef = new MutableObject<ILogicalOperator>(subplan);
+ NestedTupleSourceOperator nts = new NestedTupleSourceOperator(new MutableObject<ILogicalOperator>(g));
+ opRef.setValue(g);
+ selectNonNull.getInputs().add(new MutableObject<ILogicalOperator>(nts));
+
+ List<Mutable<ILogicalOperator>> prodInpList = botRef.getValue().getInputs();
+ prodInpList.clear();
+ prodInpList.add(new MutableObject<ILogicalOperator>(selectNonNull));
+
+ ILogicalPlan gPlan = new ALogicalPlanImpl(new MutableObject<ILogicalOperator>(subplanRoot.getValue()));
+ g.getNestedPlans().add(gPlan);
+ subplanRoot.setValue(op3Ref.getValue());
+ g.getInputs().add(newSubplanRef);
+
+ HashSet<LogicalVariable> underVars = new HashSet<LogicalVariable>();
+ VariableUtilities.getLiveVariables(subplan.getInputs().get(0).getValue(), underVars);
+ underVars.removeAll(pkVars);
+ Map<LogicalVariable, LogicalVariable> mappedVars = buildVarExprList(pkVars, context, g, g.getGroupByList());
+ context.updatePrimaryKeys(mappedVars);
+ for (LogicalVariable uv : underVars) {
+ g.getDecorList().add(
+ new Pair<LogicalVariable, Mutable<ILogicalExpression>>(null, new MutableObject<ILogicalExpression>(
+ new VariableReferenceExpression(uv))));
+ }
+ OperatorPropertiesUtil.typeOpRec(subplanRoot, context);
+ OperatorPropertiesUtil.typeOpRec(gPlan.getRoots().get(0), context);
+ context.computeAndSetTypeEnvironmentForOperator(g);
+ return true;
+ }
+
+ private NestedTupleSourceOperator getNts(AbstractLogicalOperator op) {
+ AbstractLogicalOperator alo = op;
+ do {
+ if (alo.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ return (NestedTupleSourceOperator) alo;
+ }
+ if (alo.getInputs().size() != 1) {
+ return null;
+ }
+ alo = (AbstractLogicalOperator) alo.getInputs().get(0).getValue();
+ } while (true);
+ }
+
+ protected Set<LogicalVariable> computeGbyVars(AbstractLogicalOperator op, Set<LogicalVariable> freeVars,
+ IOptimizationContext context) throws AlgebricksException {
+ PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(op, context);
+ List<FunctionalDependency> fdList = context.getFDList(op);
+ if (fdList == null) {
+ return null;
+ }
+ // check if any of the FDs is a key
+ List<LogicalVariable> all = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(op, all);
+ all.retainAll(freeVars);
+ for (FunctionalDependency fd : fdList) {
+ if (fd.getTail().containsAll(all)) {
+ return new HashSet<LogicalVariable>(fd.getHead());
+ }
+ }
+ return null;
+ }
+
+ private Map<LogicalVariable, LogicalVariable> buildVarExprList(Collection<LogicalVariable> vars,
+ IOptimizationContext context, GroupByOperator g,
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> outVeList) throws AlgebricksException {
+ Map<LogicalVariable, LogicalVariable> m = new HashMap<LogicalVariable, LogicalVariable>();
+ for (LogicalVariable ov : vars) {
+ LogicalVariable newVar = context.newVar();
+ ILogicalExpression varExpr = new VariableReferenceExpression(newVar);
+ outVeList.add(new Pair<LogicalVariable, Mutable<ILogicalExpression>>(ov,
+ new MutableObject<ILogicalExpression>(varExpr)));
+ for (ILogicalPlan p : g.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ OperatorManipulationUtil.substituteVarRec((AbstractLogicalOperator) r.getValue(), ov, newVar, true,
+ context);
+ }
+ }
+ AbstractLogicalOperator opUnder = (AbstractLogicalOperator) g.getInputs().get(0).getValue();
+ OperatorManipulationUtil.substituteVarRec(opUnder, ov, newVar, true, context);
+ m.put(ov, newVar);
+ }
+ return m;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
new file mode 100644
index 0000000..194b818
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/IsolateHyracksOperatorsRule.java
@@ -0,0 +1,140 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ExchangeOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.OneToOneExchangePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class IsolateHyracksOperatorsRule implements IAlgebraicRewriteRule {
+
+ private final PhysicalOperatorTag[] operatorsBelowWhichJobGenIsDisabled;
+
+ public IsolateHyracksOperatorsRule(PhysicalOperatorTag[] operatorsBelowWhichJobGenIsDisabled) {
+ this.operatorsBelowWhichJobGenIsDisabled = operatorsBelowWhichJobGenIsDisabled;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ IPhysicalOperator pt = op.getPhysicalOperator();
+
+ if (pt == null || op.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+ return false;
+ }
+ if (!pt.isMicroOperator()) {
+ return testIfExchangeBelow(opRef, context);
+ } else {
+ return testIfExchangeAbove(opRef, context);
+ }
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ private boolean testIfExchangeBelow(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ boolean exchInserted = false;
+
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue();
+ if (c.getOperatorTag() != LogicalOperatorTag.EXCHANGE) {
+ if (c.getPhysicalOperator() == null) {
+ return false;
+ }
+ insertOneToOneExchange(i, context);
+ exchInserted = true;
+ }
+ }
+ IPhysicalOperator pt = op.getPhysicalOperator();
+ if (pt.isJobGenDisabledBelowMe() || arrayContains(operatorsBelowWhichJobGenIsDisabled, pt.getOperatorTag())) {
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ disableJobGenRec(i.getValue());
+ }
+ }
+ return exchInserted;
+ }
+
+ private void disableJobGenRec(ILogicalOperator operator) {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) operator;
+ op.disableJobGen();
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ disableJobGenRec(i.getValue());
+ }
+ }
+
+ private boolean testIfExchangeAbove(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() == LogicalOperatorTag.EXCHANGE) {
+ return false;
+ }
+ boolean exchInserted = false;
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue();
+ IPhysicalOperator cpop = c.getPhysicalOperator();
+ if (c.getOperatorTag() == LogicalOperatorTag.EXCHANGE || cpop == null) {
+ continue;
+ }
+ if (!cpop.isMicroOperator()) {
+ insertOneToOneExchange(i, context);
+ exchInserted = true;
+ }
+ }
+ return exchInserted;
+ }
+
+ private final static <T> boolean arrayContains(T[] array, T tag) {
+ for (int i = 0; i < array.length; i++) {
+ if (array[i] == tag) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private final static void insertOneToOneExchange(Mutable<ILogicalOperator> i, IOptimizationContext context)
+ throws AlgebricksException {
+ ExchangeOperator e = new ExchangeOperator();
+ e.setPhysicalOperator(new OneToOneExchangePOperator());
+ ILogicalOperator inOp = i.getValue();
+
+ e.getInputs().add(new MutableObject<ILogicalOperator>(inOp));
+ i.setValue(e);
+ // e.recomputeSchema();
+ OperatorPropertiesUtil.computeSchemaAndPropertiesRecIfNull(e, context);
+ ExecutionMode em = ((AbstractLogicalOperator) inOp).getExecutionMode();
+ e.setExecutionMode(em);
+ e.computeDeliveredPhysicalProperties(context);
+ context.computeAndSetTypeEnvironmentForOperator(e);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
new file mode 100644
index 0000000..7ee1f49
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PullSelectOutOfEqJoin.java
@@ -0,0 +1,118 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PullSelectOutOfEqJoin implements IAlgebraicRewriteRule {
+
+ private List<Mutable<ILogicalExpression>> eqVarVarComps = new ArrayList<Mutable<ILogicalExpression>>();
+ private List<Mutable<ILogicalExpression>> otherPredicates = new ArrayList<Mutable<ILogicalExpression>>();
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+ if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN
+ && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
+ return false;
+ }
+ AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op;
+
+ ILogicalExpression expr = join.getCondition().getValue();
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) expr;
+ FunctionIdentifier fi = fexp.getFunctionIdentifier();
+ if (fi != AlgebricksBuiltinFunctions.AND) {
+ return false;
+ }
+ eqVarVarComps.clear();
+ otherPredicates.clear();
+ for (Mutable<ILogicalExpression> arg : fexp.getArguments()) {
+ if (isEqVarVar(arg.getValue())) {
+ eqVarVarComps.add(arg);
+ } else {
+ otherPredicates.add(arg);
+ }
+ }
+ if (eqVarVarComps.isEmpty() || otherPredicates.isEmpty()) {
+ return false;
+ }
+ // pull up
+ ILogicalExpression pulledCond = makeCondition(otherPredicates);
+ SelectOperator select = new SelectOperator(new MutableObject<ILogicalExpression>(pulledCond));
+ ILogicalExpression newJoinCond = makeCondition(eqVarVarComps);
+ join.getCondition().setValue(newJoinCond);
+ select.getInputs().add(new MutableObject<ILogicalOperator>(join));
+ opRef.setValue(select);
+ context.computeAndSetTypeEnvironmentForOperator(select);
+ return true;
+ }
+
+ private ILogicalExpression makeCondition(List<Mutable<ILogicalExpression>> predList) {
+ if (predList.size() > 1) {
+ IFunctionInfo finfo = AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND);
+ return new ScalarFunctionCallExpression(finfo, predList);
+ } else {
+ return predList.get(0).getValue();
+ }
+ }
+
+ private boolean isEqVarVar(ILogicalExpression expr) {
+ if (expr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return false;
+ }
+ AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) expr;
+ if (f.getFunctionIdentifier() != AlgebricksBuiltinFunctions.EQ) {
+ return false;
+ }
+ ILogicalExpression e1 = f.getArguments().get(0).getValue();
+ if (e1.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return false;
+ } else {
+ ILogicalExpression e2 = f.getArguments().get(1).getValue();
+ return e2.getExpressionTag() == LogicalExpressionTag.VARIABLE;
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
new file mode 100644
index 0000000..5eb7160
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushAssignDownThroughProductRule.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushAssignDownThroughProductRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
+ return false;
+ }
+ Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+ return false;
+ }
+ AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
+ if (join.getCondition().getValue() != ConstantExpression.TRUE) {
+ return false;
+ }
+
+ List<LogicalVariable> used = new ArrayList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op1, used);
+
+ Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
+ ILogicalOperator b0 = b0Ref.getValue();
+ List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(b0, b0Scm);
+ if (b0Scm.containsAll(used)) {
+ // push assign on left branch
+ op2Ref.setValue(b0);
+ b0Ref.setValue(op1);
+ opRef.setValue(op2);
+ return true;
+ } else {
+ Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
+ ILogicalOperator b1 = b1Ref.getValue();
+ List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(b1, b1Scm);
+ if (b1Scm.containsAll(used)) {
+ // push assign on right branch
+ op2Ref.setValue(b1);
+ b1Ref.setValue(op1);
+ opRef.setValue(op2);
+ return true;
+ } else {
+ return false;
+ }
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java
new file mode 100644
index 0000000..ef7822f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushDieUpRule.java
@@ -0,0 +1,52 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushDieUpRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+ if (op0.getInputs().size() == 0)
+ return false;
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) op0.getInputs().get(0).getValue();
+
+ if (op1.getInputs().size() == 0)
+ return false;
+ LogicalOperatorTag tag = op1.getOperatorTag();
+ if (tag == LogicalOperatorTag.SINK || tag == LogicalOperatorTag.WRITE
+ || tag == LogicalOperatorTag.INSERT_DELETE || tag == LogicalOperatorTag.WRITE_RESULT)
+ return false;
+
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue();
+ if (op2.getOperatorTag() == LogicalOperatorTag.DIE) {
+ op0.getInputs().get(0).setValue(op2);
+ op1.getInputs().clear();
+ for (Mutable<ILogicalOperator> ref : op2.getInputs())
+ op1.getInputs().add(ref);
+ op2.getInputs().clear();
+ op2.getInputs().add(new MutableObject<ILogicalOperator>(op1));
+
+ context.computeAndSetTypeEnvironmentForOperator(op0);
+ context.computeAndSetTypeEnvironmentForOperator(op1);
+ context.computeAndSetTypeEnvironmentForOperator(op2);
+ return true;
+ } else {
+ return false;
+ }
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java
new file mode 100644
index 0000000..1b096ea
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushLimitDownRule.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.LinkedList;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamLimitPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushLimitDownRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ /**
+ * When a global Limit over a merge-exchange is found, a local Limit is
+ * pushed down.
+ *
+ */
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.LIMIT) {
+ return false;
+ }
+ LimitOperator opLim = (LimitOperator) op;
+ if (!opLim.isTopmostLimitOp()) {
+ return false;
+ }
+
+ Mutable<ILogicalOperator> opRef2 = opLim.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+
+ if (context.checkAndAddToAlreadyCompared(op, op2)) {
+ return false;
+ }
+ if (op2.getOperatorTag() != LogicalOperatorTag.EXCHANGE) {
+ return false;
+ }
+ PhysicalOperatorTag op2PTag = op2.getPhysicalOperator().getOperatorTag();
+ // we should test for any kind of merge
+ if (op2PTag != PhysicalOperatorTag.RANDOM_MERGE_EXCHANGE && op2PTag != PhysicalOperatorTag.SORT_MERGE_EXCHANGE) {
+ return false;
+ }
+
+ LinkedList<LogicalVariable> usedVars1 = new LinkedList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(opLim, usedVars1);
+
+ do {
+ if (op2.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
+ || op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE
+ || op2.getOperatorTag() == LogicalOperatorTag.LIMIT) {
+ return false;
+ }
+ if (op2.getInputs().size() > 1 || !op2.isMap()) {
+ break;
+ }
+ LinkedList<LogicalVariable> vars2 = new LinkedList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(op2, vars2);
+ if (!OperatorPropertiesUtil.disjoint(vars2, usedVars1)) {
+ return false;
+ }
+ // we assume pipelineable ops. have only one input
+ opRef2 = op2.getInputs().get(0);
+ op2 = (AbstractLogicalOperator) opRef2.getValue();
+ } while (true);
+
+ LimitOperator clone2 = null;
+ if (opLim.getOffset().getValue() == null) {
+ clone2 = new LimitOperator(opLim.getMaxObjects().getValue(), false);
+ } else {
+ // push limit (max+offset)
+ IFunctionInfo finfoAdd = AlgebricksBuiltinFunctions
+ .getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.NUMERIC_ADD);
+ ScalarFunctionCallExpression maxPlusOffset = new ScalarFunctionCallExpression(finfoAdd,
+ opLim.getMaxObjects(), opLim.getOffset());
+ clone2 = new LimitOperator(maxPlusOffset, false);
+ }
+ clone2.setPhysicalOperator(new StreamLimitPOperator(false));
+ clone2.getInputs().add(new MutableObject<ILogicalOperator>(op2));
+ clone2.setExecutionMode(op2.getExecutionMode());
+ clone2.recomputeSchema();
+ opRef2.setValue(clone2);
+ context.computeAndSetTypeEnvironmentForOperator(clone2);
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
new file mode 100644
index 0000000..b925cfe
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushNestedOrderByUnderPreSortedGroupByRule.java
@@ -0,0 +1,119 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.PhysicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractPhysicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+public class PushNestedOrderByUnderPreSortedGroupByRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ if (op.getPhysicalOperator() == null) {
+ return false;
+ }
+ AbstractPhysicalOperator pOp = (AbstractPhysicalOperator) op.getPhysicalOperator();
+ if (pOp.getOperatorTag() != PhysicalOperatorTag.PRE_CLUSTERED_GROUP_BY) {
+ return false;
+ }
+ GroupByOperator gby = (GroupByOperator) op;
+ ILogicalPlan plan = gby.getNestedPlans().get(0);
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) plan.getRoots().get(0).getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+ Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.ORDER) {
+ return false;
+ }
+ OrderOperator order1 = (OrderOperator) op2;
+ if (!isIndependentFromChildren(order1)) {
+ return false;
+ }
+ AbstractPhysicalOperator pOrder1 = (AbstractPhysicalOperator) op2.getPhysicalOperator();
+ if (pOrder1.getOperatorTag() != PhysicalOperatorTag.STABLE_SORT
+ && pOrder1.getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT) {
+ return false;
+ }
+ // StableSortPOperator sort1 = (StableSortPOperator) pOrder1;
+ AbstractLogicalOperator op3 = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ if (op3.getOperatorTag() != LogicalOperatorTag.ORDER) {
+ return false;
+ }
+ AbstractPhysicalOperator pOp3 = (AbstractPhysicalOperator) op3.getPhysicalOperator();
+ if (pOp3.getOperatorTag() != PhysicalOperatorTag.STABLE_SORT) {
+ return false;
+ }
+ OrderOperator order2 = (OrderOperator) op3;
+ StableSortPOperator sort2 = (StableSortPOperator) pOp3;
+ // int n1 = sort1.getSortColumns().length;
+ // int n2 = sort2.getSortColumns().length;
+ // OrderColumn[] sortColumns = new OrderColumn[n2 + n1];
+ // System.arraycopy(sort2.getSortColumns(), 0, sortColumns, 0, n2);
+ // int k = 0;
+ for (Pair<IOrder, Mutable<ILogicalExpression>> oe : order1.getOrderExpressions()) {
+ order2.getOrderExpressions().add(oe);
+ // sortColumns[n2 + k] = sort1.getSortColumns()[k];
+ // ++k;
+ }
+ // sort2.setSortColumns(sortColumns);
+ sort2.computeDeliveredProperties(order2, null);
+ // remove order1
+ ILogicalOperator underOrder1 = order1.getInputs().get(0).getValue();
+ opRef2.setValue(underOrder1);
+ return true;
+ }
+
+ private boolean isIndependentFromChildren(OrderOperator order1) throws AlgebricksException {
+ Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+ OperatorPropertiesUtil.getFreeVariablesInSelfOrDesc(order1, free);
+ Set<LogicalVariable> usedInOrder = new HashSet<LogicalVariable>();
+ VariableUtilities.getUsedVariables(order1, usedInOrder);
+ return free.containsAll(usedInOrder);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
new file mode 100644
index 0000000..b70d88d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectDownRule.java
@@ -0,0 +1,220 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+
+/**
+ *
+ * Pushes projections through its input operator, provided that operator does
+ * not produce the projected variables.
+ *
+ * @author Nicola
+ *
+ */
+public class PushProjectDownRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.PROJECT) {
+ return false;
+ }
+ ProjectOperator pi = (ProjectOperator) op;
+ Mutable<ILogicalOperator> opRef2 = pi.getInputs().get(0);
+
+ HashSet<LogicalVariable> toPush = new HashSet<LogicalVariable>();
+ toPush.addAll(pi.getVariables());
+
+ Pair<Boolean, Boolean> p = pushThroughOp(toPush, opRef2, op, context);
+ boolean smthWasPushed = p.first;
+ if (p.second) { // the original projection is redundant
+ opRef.setValue(op.getInputs().get(0).getValue());
+ smthWasPushed = true;
+ }
+
+ return smthWasPushed;
+ }
+
+ private static Pair<Boolean, Boolean> pushThroughOp(HashSet<LogicalVariable> toPush,
+ Mutable<ILogicalOperator> opRef2, ILogicalOperator initialOp, IOptimizationContext context)
+ throws AlgebricksException {
+ List<LogicalVariable> initProjectList = new ArrayList<LogicalVariable>(toPush);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+ do {
+ if (op2.getOperatorTag() == LogicalOperatorTag.EMPTYTUPLESOURCE
+ || op2.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE
+ || op2.getOperatorTag() == LogicalOperatorTag.PROJECT
+ || op2.getOperatorTag() == LogicalOperatorTag.REPLICATE) {
+ return new Pair<Boolean, Boolean>(false, false);
+ }
+ if (!op2.isMap()) {
+ break;
+ }
+ LinkedList<LogicalVariable> usedVars = new LinkedList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op2, usedVars);
+ toPush.addAll(usedVars);
+ LinkedList<LogicalVariable> producedVars = new LinkedList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(op2, producedVars);
+ toPush.removeAll(producedVars);
+ // we assume pipelineable ops. have only one input
+ opRef2 = op2.getInputs().get(0);
+ op2 = (AbstractLogicalOperator) opRef2.getValue();
+ } while (true);
+
+ LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(op2, produced2);
+ LinkedList<LogicalVariable> used2 = new LinkedList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op2, used2);
+
+ boolean canCommuteProjection = initProjectList.containsAll(toPush) && initProjectList.containsAll(produced2)
+ && initProjectList.containsAll(used2);
+ // if true, we can get rid of the initial projection
+
+ // get rid of useless decor vars.
+ if (!canCommuteProjection && op2.getOperatorTag() == LogicalOperatorTag.GROUP) {
+ boolean gbyChanged = false;
+ GroupByOperator gby = (GroupByOperator) op2;
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> newDecorList = new ArrayList<Pair<LogicalVariable, Mutable<ILogicalExpression>>>();
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gby.getDecorList()) {
+ LogicalVariable decorVar = GroupByOperator.getDecorVariable(p);
+ if (!toPush.contains(decorVar)) {
+ used2.remove(decorVar);
+ gbyChanged = true;
+ } else {
+ newDecorList.add(p);
+ }
+ }
+ gby.getDecorList().clear();
+ gby.getDecorList().addAll(newDecorList);
+ if (gbyChanged) {
+ context.computeAndSetTypeEnvironmentForOperator(gby);
+ }
+ }
+ used2.clear();
+ VariableUtilities.getUsedVariables(op2, used2);
+
+ toPush.addAll(used2); // remember that toPush is a Set
+ toPush.removeAll(produced2);
+
+ if (toPush.isEmpty()) {
+ return new Pair<Boolean, Boolean>(false, false);
+ }
+
+ boolean smthWasPushed = false;
+ for (Mutable<ILogicalOperator> c : op2.getInputs()) {
+ if (pushNeededProjections(toPush, c, context, initialOp)) {
+ smthWasPushed = true;
+ }
+ }
+ if (op2.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans n = (AbstractOperatorWithNestedPlans) op2;
+ for (ILogicalPlan p : n.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ if (pushNeededProjections(toPush, r, context, initialOp)) {
+ smthWasPushed = true;
+ }
+ }
+ }
+ }
+ return new Pair<Boolean, Boolean>(smthWasPushed, canCommuteProjection);
+ }
+
+ // It does not try to push above another Projection.
+ private static boolean pushNeededProjections(HashSet<LogicalVariable> toPush, Mutable<ILogicalOperator> opRef,
+ IOptimizationContext context, ILogicalOperator initialOp) throws AlgebricksException {
+ HashSet<LogicalVariable> allP = new HashSet<LogicalVariable>();
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ VariableUtilities.getLiveVariables(op, allP);
+
+ HashSet<LogicalVariable> toProject = new HashSet<LogicalVariable>();
+ for (LogicalVariable v : toPush) {
+ if (allP.contains(v)) {
+ toProject.add(v);
+ }
+ }
+ if (toProject.equals(allP)) {
+ // projection would be redundant, since we would project everything
+ // but we can try with the children
+ boolean push = false;
+ if (pushThroughOp(toProject, opRef, initialOp, context).first) {
+ push = true;
+ }
+ return push;
+ } else {
+ return pushAllProjectionsOnTopOf(toProject, opRef, context, initialOp);
+ }
+ }
+
+ // It does not try to push above another Projection.
+ private static boolean pushAllProjectionsOnTopOf(Collection<LogicalVariable> toPush,
+ Mutable<ILogicalOperator> opRef, IOptimizationContext context, ILogicalOperator initialOp)
+ throws AlgebricksException {
+ if (toPush.isEmpty()) {
+ return false;
+ }
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+
+ if (context.checkAndAddToAlreadyCompared(initialOp, op)) {
+ return false;
+ }
+
+ switch (op.getOperatorTag()) {
+ case EXCHANGE: {
+ opRef = opRef.getValue().getInputs().get(0);
+ op = (AbstractLogicalOperator) opRef.getValue();
+ break;
+ }
+ case PROJECT: {
+ return false;
+ }
+ }
+
+ ProjectOperator pi2 = new ProjectOperator(new ArrayList<LogicalVariable>(toPush));
+ pi2.getInputs().add(new MutableObject<ILogicalOperator>(op));
+ opRef.setValue(pi2);
+ pi2.setExecutionMode(op.getExecutionMode());
+ context.computeAndSetTypeEnvironmentForOperator(pi2);
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
new file mode 100644
index 0000000..4e14f9a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushProjectIntoDataSourceScanRule.java
@@ -0,0 +1,46 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushProjectIntoDataSourceScanRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getInputs().size() <= 0)
+ return false;
+ AbstractLogicalOperator project = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ if (project.getOperatorTag() != LogicalOperatorTag.PROJECT)
+ return false;
+ AbstractLogicalOperator exchange = (AbstractLogicalOperator) project.getInputs().get(0).getValue();
+ if (exchange.getOperatorTag() != LogicalOperatorTag.EXCHANGE)
+ return false;
+ AbstractLogicalOperator inputOp = (AbstractLogicalOperator) exchange.getInputs().get(0).getValue();
+ if (inputOp.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN)
+ return false;
+ DataSourceScanOperator scanOp = (DataSourceScanOperator) inputOp;
+ ProjectOperator projectOp = (ProjectOperator) project;
+ scanOp.addProjectVariables(projectOp.getVariables());
+ if (op.getOperatorTag() != LogicalOperatorTag.EXCHANGE) {
+ op.getInputs().set(0, project.getInputs().get(0));
+ } else {
+ op.getInputs().set(0, exchange.getInputs().get(0));
+ }
+ return true;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
new file mode 100644
index 0000000..fedcad9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectDownRule.java
@@ -0,0 +1,98 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushSelectDownRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+ return false;
+ }
+
+ Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+
+ if (context.checkAndAddToAlreadyCompared(op, op2)) {
+ return false;
+ }
+
+ LogicalOperatorTag tag2 = op2.getOperatorTag();
+
+ if (tag2 == LogicalOperatorTag.INNERJOIN || tag2 == LogicalOperatorTag.LEFTOUTERJOIN
+ || tag2 == LogicalOperatorTag.REPLICATE) {
+ return false;
+ } else { // not a join
+ boolean res = propagateSelectionRec(opRef, opRef2);
+ if (res) {
+ OperatorPropertiesUtil.typeOpRec(opRef, context);
+ }
+ return res;
+ }
+ }
+
+ private static boolean propagateSelectionRec(Mutable<ILogicalOperator> sigmaRef, Mutable<ILogicalOperator> opRef2)
+ throws AlgebricksException {
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+ if (op2.getInputs().size() != 1 || op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ return false;
+ }
+
+ SelectOperator sigma = (SelectOperator) sigmaRef.getValue();
+ LinkedList<LogicalVariable> usedInSigma = new LinkedList<LogicalVariable>();
+ sigma.getCondition().getValue().getUsedVariables(usedInSigma);
+
+ LinkedList<LogicalVariable> produced2 = new LinkedList<LogicalVariable>();
+ VariableUtilities.getProducedVariables(op2, produced2);
+ if (OperatorPropertiesUtil.disjoint(produced2, usedInSigma)) {
+ // just swap
+ opRef2.setValue(sigma);
+ sigmaRef.setValue(op2);
+ List<Mutable<ILogicalOperator>> sigmaInpList = sigma.getInputs();
+ sigmaInpList.clear();
+ sigmaInpList.addAll(op2.getInputs());
+ List<Mutable<ILogicalOperator>> op2InpList = op2.getInputs();
+ op2InpList.clear();
+ op2InpList.add(opRef2);
+ propagateSelectionRec(opRef2, sigma.getInputs().get(0));
+ return true;
+
+ }
+ return false;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
new file mode 100644
index 0000000..315abdb
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSelectIntoJoinRule.java
@@ -0,0 +1,239 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushSelectIntoJoinRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ Collection<LogicalVariable> joinLiveVarsLeft = new HashSet<LogicalVariable>();
+ Collection<LogicalVariable> joinLiveVarsRight = new HashSet<LogicalVariable>();
+ Collection<LogicalVariable> liveInOpsToPushLeft = new HashSet<LogicalVariable>();
+ Collection<LogicalVariable> liveInOpsToPushRight = new HashSet<LogicalVariable>();
+
+ List<ILogicalOperator> pushedOnLeft = new ArrayList<ILogicalOperator>();
+ List<ILogicalOperator> pushedOnRight = new ArrayList<ILogicalOperator>();
+ LinkedList<ILogicalOperator> notPushedStack = new LinkedList<ILogicalOperator>();
+ Collection<LogicalVariable> usedVars = new HashSet<LogicalVariable>();
+ Collection<LogicalVariable> producedVars = new HashSet<LogicalVariable>();
+
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.SELECT) {
+ return false;
+ }
+ SelectOperator select = (SelectOperator) op;
+ Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+ AbstractLogicalOperator son = (AbstractLogicalOperator) opRef2.getValue();
+ AbstractLogicalOperator op2 = son;
+ boolean needToPushOps = false;
+ while (son.isMap()) {
+ needToPushOps = true;
+ Mutable<ILogicalOperator> opRefLink = son.getInputs().get(0);
+ son = (AbstractLogicalOperator) opRefLink.getValue();
+ }
+
+ if (son.getOperatorTag() != LogicalOperatorTag.INNERJOIN
+ && son.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) {
+ return false;
+ }
+ boolean isLoj = son.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN;
+ AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) son;
+
+ Mutable<ILogicalOperator> joinBranchLeftRef = join.getInputs().get(0);
+ Mutable<ILogicalOperator> joinBranchRightRef = join.getInputs().get(1);
+
+ if (needToPushOps) {
+ ILogicalOperator joinBranchLeft = joinBranchLeftRef.getValue();
+ ILogicalOperator joinBranchRight = joinBranchRightRef.getValue();
+ VariableUtilities.getLiveVariables(joinBranchLeft, joinLiveVarsLeft);
+ VariableUtilities.getLiveVariables(joinBranchRight, joinLiveVarsRight);
+ Mutable<ILogicalOperator> opIterRef = opRef2;
+ ILogicalOperator opIter = op2;
+ while (opIter != join) {
+ LogicalOperatorTag tag = ((AbstractLogicalOperator) opIter).getOperatorTag();
+ if (tag == LogicalOperatorTag.PROJECT) {
+ notPushedStack.addFirst(opIter);
+ } else {
+ VariableUtilities.getUsedVariables(opIter, usedVars);
+ VariableUtilities.getProducedVariables(opIter, producedVars);
+ if (joinLiveVarsLeft.containsAll(usedVars)) {
+ pushedOnLeft.add(opIter);
+ liveInOpsToPushLeft.addAll(producedVars);
+ } else if (joinLiveVarsRight.containsAll(usedVars)) {
+ pushedOnRight.add(opIter);
+ liveInOpsToPushRight.addAll(producedVars);
+ } else {
+ return false;
+ }
+ }
+ opIterRef = opIter.getInputs().get(0);
+ opIter = opIterRef.getValue();
+ }
+ if (isLoj && pushedOnLeft.isEmpty()) {
+ return false;
+ }
+ }
+
+ boolean intersectsAllBranches = true;
+ boolean[] intersectsBranch = new boolean[join.getInputs().size()];
+ LinkedList<LogicalVariable> selectVars = new LinkedList<LogicalVariable>();
+ select.getCondition().getValue().getUsedVariables(selectVars);
+ int i = 0;
+ for (Mutable<ILogicalOperator> branch : join.getInputs()) {
+ LinkedList<LogicalVariable> branchVars = new LinkedList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(branch.getValue(), branchVars);
+ if (i == 0) {
+ branchVars.addAll(liveInOpsToPushLeft);
+ } else {
+ branchVars.addAll(liveInOpsToPushRight);
+ }
+ if (OperatorPropertiesUtil.disjoint(selectVars, branchVars)) {
+ intersectsAllBranches = false;
+ } else {
+ intersectsBranch[i] = true;
+ }
+ i++;
+ }
+ if (!intersectsBranch[0] && !intersectsBranch[1]) {
+ return false;
+ }
+ if (intersectsAllBranches) {
+ if (needToPushOps) {
+ pushOps(pushedOnLeft, joinBranchLeftRef, context);
+ pushOps(pushedOnRight, joinBranchRightRef, context);
+ }
+ addCondToJoin(select, join);
+ } else { // push down
+ Iterator<Mutable<ILogicalOperator>> branchIter = join.getInputs().iterator();
+
+ for (int j = 0; j < intersectsBranch.length; j++) {
+ Mutable<ILogicalOperator> branch = branchIter.next();
+ boolean inter = intersectsBranch[j];
+ if (inter) {
+ if (needToPushOps) {
+ if (j == 0) {
+ pushOps(pushedOnLeft, joinBranchLeftRef, context);
+ } else {
+ pushOps(pushedOnRight, joinBranchRightRef, context);
+ }
+ }
+ copySelectToBranch(select, branch, context);
+ }
+
+ // if a left outer join, we can only push conditions into the
+ // outer branch.
+ if (j == 0 && isLoj) {
+ // stop at this branch
+ break;
+ }
+ }
+ }
+ ILogicalOperator top = join;
+ for (ILogicalOperator npOp : notPushedStack) {
+ List<Mutable<ILogicalOperator>> npInpList = npOp.getInputs();
+ npInpList.clear();
+ npInpList.add(new MutableObject<ILogicalOperator>(top));
+ context.computeAndSetTypeEnvironmentForOperator(npOp);
+ top = npOp;
+ }
+ opRef.setValue(top);
+ return true;
+
+ }
+
+ private void pushOps(List<ILogicalOperator> opList, Mutable<ILogicalOperator> joinBranch,
+ IOptimizationContext context) throws AlgebricksException {
+ ILogicalOperator topOp = joinBranch.getValue();
+ ListIterator<ILogicalOperator> iter = opList.listIterator(opList.size());
+ while (iter.hasPrevious()) {
+ ILogicalOperator op = iter.previous();
+ List<Mutable<ILogicalOperator>> opInpList = op.getInputs();
+ opInpList.clear();
+ opInpList.add(new MutableObject<ILogicalOperator>(topOp));
+ topOp = op;
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ }
+ joinBranch.setValue(topOp);
+ }
+
+ private static void addCondToJoin(SelectOperator select, AbstractBinaryJoinOperator join) {
+ ILogicalExpression cond = join.getCondition().getValue();
+ if (OperatorPropertiesUtil.isAlwaysTrueCond(cond)) { // the join was a product
+ join.getCondition().setValue(select.getCondition().getValue());
+ } else {
+ boolean bAddedToConj = false;
+ if (cond.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) {
+ AbstractFunctionCallExpression fcond = (AbstractFunctionCallExpression) cond;
+ if (fcond.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.AND)) {
+ AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(
+ AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND));
+ newCond.getArguments().add(select.getCondition());
+ newCond.getArguments().addAll(fcond.getArguments());
+ join.getCondition().setValue(newCond);
+ bAddedToConj = true;
+ }
+ }
+ if (!bAddedToConj) {
+ AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression(
+ AlgebricksBuiltinFunctions.getBuiltinFunctionInfo(AlgebricksBuiltinFunctions.AND),
+ select.getCondition(), new MutableObject<ILogicalExpression>(join.getCondition().getValue()));
+ join.getCondition().setValue(newCond);
+ }
+ }
+ }
+
+ private static void copySelectToBranch(SelectOperator select, Mutable<ILogicalOperator> branch,
+ IOptimizationContext context) throws AlgebricksException {
+ ILogicalOperator newSelect = new SelectOperator(select.getCondition());
+ Mutable<ILogicalOperator> newRef = new MutableObject<ILogicalOperator>(branch.getValue());
+ newSelect.getInputs().add(newRef);
+ branch.setValue(newSelect);
+ context.computeAndSetTypeEnvironmentForOperator(newSelect);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
new file mode 100644
index 0000000..25855e5
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/PushSubplanWithAggregateDownThroughProductRule.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class PushSubplanWithAggregateDownThroughProductRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+ if (op1.getOperatorTag() != LogicalOperatorTag.SUBPLAN) {
+ return false;
+ }
+ SubplanOperator subplan = (SubplanOperator) op1;
+ if (subplan.getNestedPlans().size() != 1) {
+ return false;
+ }
+ ILogicalPlan p = subplan.getNestedPlans().get(0);
+ if (p.getRoots().size() != 1) {
+ return false;
+ }
+ Mutable<ILogicalOperator> r = p.getRoots().get(0);
+ if (((AbstractLogicalOperator) r.getValue()).getOperatorTag() != LogicalOperatorTag.AGGREGATE) {
+ return false;
+ }
+
+ Set<LogicalVariable> free = new HashSet<LogicalVariable>();
+ OperatorPropertiesUtil.getFreeVariablesInSubplans(subplan, free);
+
+ Mutable<ILogicalOperator> op2Ref = op1.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) op2Ref.getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.INNERJOIN) {
+ return false;
+ }
+ AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) op2;
+ if (!OperatorPropertiesUtil.isAlwaysTrueCond(join.getCondition().getValue())) {
+ return false;
+ }
+
+ Mutable<ILogicalOperator> b0Ref = op2.getInputs().get(0);
+ ILogicalOperator b0 = b0Ref.getValue();
+ List<LogicalVariable> b0Scm = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(b0, b0Scm);
+ if (b0Scm.containsAll(free)) {
+ // push subplan on left branch
+ op2Ref.setValue(b0);
+ b0Ref.setValue(op1);
+ opRef.setValue(op2);
+ return true;
+ } else {
+ Mutable<ILogicalOperator> b1Ref = op2.getInputs().get(1);
+ ILogicalOperator b1 = b1Ref.getValue();
+ List<LogicalVariable> b1Scm = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(b1, b1Scm);
+ if (b1Scm.containsAll(free)) {
+ // push subplan on right branch
+ op2Ref.setValue(b1);
+ b1Ref.setValue(op1);
+ opRef.setValue(op2);
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
new file mode 100644
index 0000000..f06a2ac
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/ReinferAllTypesRule.java
@@ -0,0 +1,50 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class ReinferAllTypesRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+ return false;
+ }
+ typeOpRec(opRef, context);
+ return true;
+ }
+
+ private void typePlan(ILogicalPlan p, IOptimizationContext context) throws AlgebricksException {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ typeOpRec(r, context);
+ }
+ }
+
+ private void typeOpRec(Mutable<ILogicalOperator> r, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) r.getValue();
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ typeOpRec(i, context);
+ }
+ if (op.hasNestedPlans()) {
+ for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
+ typePlan(p, context);
+ }
+ }
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ context.addToDontApplySet(this, op);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
new file mode 100644
index 0000000..cdc6086
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveRedundantProjectionRule.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.ProjectOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/*
+ * project [var-list1]
+ * project [var-list2]
+ * P
+ *
+ * if var-list1.equals(var-list2) becomes
+ *
+ * project [var-list1]
+ * P
+ *
+ */
+
+public class RemoveRedundantProjectionRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) opRef.getValue();
+ if (op1.getOperatorTag() == LogicalOperatorTag.PROJECT) {
+ Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.PROJECT) {
+ return false;
+ }
+ ProjectOperator pi2 = (ProjectOperator) op2;
+ opRef2.setValue(pi2.getInputs().get(0).getValue());
+ } else {
+ if (op1.getInputs().size() <= 0)
+ return false;
+ Mutable<ILogicalOperator> opRef2 = op1.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+ if (op2.getOperatorTag() != LogicalOperatorTag.PROJECT) {
+ return false;
+ }
+ if (op2.getInputs().size() <= 0)
+ return false;
+ Mutable<ILogicalOperator> opRef3 = op2.getInputs().get(0);
+ AbstractLogicalOperator op3 = (AbstractLogicalOperator) opRef3.getValue();
+
+ List<LogicalVariable> liveVars2 = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> liveVars3 = new ArrayList<LogicalVariable>();
+
+ VariableUtilities.getLiveVariables(op2, liveVars2);
+ VariableUtilities.getLiveVariables(op3, liveVars3);
+
+ if (!VariableUtilities.varListEqualUnordered(liveVars2, liveVars3))
+ return false;
+ opRef2.setValue(op3);
+ }
+
+ return true;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
new file mode 100644
index 0000000..ff1a102
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/RemoveUnusedAssignAndAggregateRule.java
@@ -0,0 +1,150 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AssignOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class RemoveUnusedAssignAndAggregateRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ if (context.checkIfInDontApplySet(this, opRef.getValue())) {
+ return false;
+ }
+ Set<LogicalVariable> toRemove = new HashSet<LogicalVariable>();
+ collectUnusedAssignedVars((AbstractLogicalOperator) opRef.getValue(), toRemove, true, context);
+ boolean smthToRemove = !toRemove.isEmpty();
+ if (smthToRemove) {
+ removeUnusedAssigns(opRef, toRemove, context);
+ }
+ return smthToRemove;
+ }
+
+ private void removeUnusedAssigns(Mutable<ILogicalOperator> opRef, Set<LogicalVariable> toRemove,
+ IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ while (removeFromAssigns(op, toRemove, context) == 0) {
+ if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+ break;
+ }
+ op = (AbstractLogicalOperator) op.getInputs().get(0).getValue();
+ opRef.setValue(op);
+ }
+ Iterator<Mutable<ILogicalOperator>> childIter = op.getInputs().iterator();
+ while (childIter.hasNext()) {
+ Mutable<ILogicalOperator> cRef = childIter.next();
+ removeUnusedAssigns(cRef, toRemove, context);
+ }
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans opWithNest = (AbstractOperatorWithNestedPlans) op;
+ Iterator<ILogicalPlan> planIter = opWithNest.getNestedPlans().iterator();
+ while (planIter.hasNext()) {
+ ILogicalPlan p = planIter.next();
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ removeUnusedAssigns(r, toRemove, context);
+ }
+ }
+ }
+ }
+
+ private int removeFromAssigns(AbstractLogicalOperator op, Set<LogicalVariable> toRemove,
+ IOptimizationContext context) throws AlgebricksException {
+ if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator assign = (AssignOperator) op;
+ if (removeUnusedVarsAndExprs(toRemove, assign.getVariables(), assign.getExpressions())) {
+ context.computeAndSetTypeEnvironmentForOperator(assign);
+ }
+ return assign.getVariables().size();
+ } else if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+ AggregateOperator agg = (AggregateOperator) op;
+ if (removeUnusedVarsAndExprs(toRemove, agg.getVariables(), agg.getExpressions())) {
+ context.computeAndSetTypeEnvironmentForOperator(agg);
+ }
+ return agg.getVariables().size();
+ }
+ return -1;
+ }
+
+ private boolean removeUnusedVarsAndExprs(Set<LogicalVariable> toRemove, List<LogicalVariable> varList,
+ List<Mutable<ILogicalExpression>> exprList) {
+ boolean changed = false;
+ Iterator<LogicalVariable> varIter = varList.iterator();
+ Iterator<Mutable<ILogicalExpression>> exprIter = exprList.iterator();
+ while (varIter.hasNext()) {
+ LogicalVariable v = varIter.next();
+ exprIter.next();
+ if (toRemove.contains(v)) {
+ varIter.remove();
+ exprIter.remove();
+ changed = true;
+ }
+ }
+ return changed;
+ }
+
+ private void collectUnusedAssignedVars(AbstractLogicalOperator op, Set<LogicalVariable> toRemove, boolean first,
+ IOptimizationContext context) throws AlgebricksException {
+ if (!first) {
+ context.addToDontApplySet(this, op);
+ }
+ for (Mutable<ILogicalOperator> c : op.getInputs()) {
+ collectUnusedAssignedVars((AbstractLogicalOperator) c.getValue(), toRemove, false, context);
+ }
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans opWithNested = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan plan : opWithNested.getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : plan.getRoots()) {
+ collectUnusedAssignedVars((AbstractLogicalOperator) r.getValue(), toRemove, false, context);
+ }
+ }
+ }
+ if (op.getOperatorTag() == LogicalOperatorTag.ASSIGN) {
+ AssignOperator assign = (AssignOperator) op;
+ toRemove.addAll(assign.getVariables());
+ } else if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) {
+ AggregateOperator agg = (AggregateOperator) op;
+ toRemove.addAll(agg.getVariables());
+ }
+ List<LogicalVariable> used = new LinkedList<LogicalVariable>();
+ VariableUtilities.getUsedVariables(op, used);
+ toRemove.removeAll(used);
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
new file mode 100644
index 0000000..f1f3533
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
@@ -0,0 +1,350 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.OperatorAnnotations;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IMergeAggregationExpressionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IDataSource;
+import edu.uci.ics.hyracks.algebricks.core.algebra.metadata.IMetadataProvider;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator.ExecutionMode;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AggregateOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DataSourceScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.DistinctOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.IndexInsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InsertDeleteOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LeftOuterJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.LimitOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.OrderOperator.IOrder;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.WriteResultOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AggregatePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AssignPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.DataSourceScanPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.EmptyTupleSourcePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ExternalGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryStableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.IndexInsertDeletePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InsertDeletePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.MicroPreclusteredGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.NestedTupleSourcePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreSortedDistinctByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.PreclusteredGroupByPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.ReplicatePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.RunningAggregatePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SinkPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SinkWritePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StableSortPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamDiePOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamLimitPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamProjectPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StreamSelectPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.StringStreamingScriptPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.SubplanPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.UnionAllPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.UnnestPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.WriteResultPOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig;
+import edu.uci.ics.hyracks.algebricks.core.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.rewriter.util.JoinUtils;
+
+public class SetAlgebricksPhysicalOperatorsRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ // if (context.checkIfInDontApplySet(this, op)) {
+ // return false;
+ // }
+ if (op.getPhysicalOperator() != null) {
+ return false;
+ }
+
+ computeDefaultPhysicalOp(op, true, context);
+ // context.addToDontApplySet(this, op);
+ return true;
+ }
+
+ private static void setPhysicalOperators(ILogicalPlan plan, boolean topLevelOp, IOptimizationContext context)
+ throws AlgebricksException {
+ for (Mutable<ILogicalOperator> root : plan.getRoots()) {
+ computeDefaultPhysicalOp((AbstractLogicalOperator) root.getValue(), topLevelOp, context);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void computeDefaultPhysicalOp(AbstractLogicalOperator op, boolean topLevelOp,
+ IOptimizationContext context) throws AlgebricksException {
+ PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig();
+ if (op.getPhysicalOperator() == null) {
+ switch (op.getOperatorTag()) {
+ case AGGREGATE: {
+ op.setPhysicalOperator(new AggregatePOperator());
+ break;
+ }
+ case ASSIGN: {
+ op.setPhysicalOperator(new AssignPOperator());
+ break;
+ }
+ case DISTINCT: {
+ DistinctOperator distinct = (DistinctOperator) op;
+ distinct.setPhysicalOperator(new PreSortedDistinctByPOperator(distinct.getDistinctByVarList()));
+ break;
+ }
+ case EMPTYTUPLESOURCE: {
+ op.setPhysicalOperator(new EmptyTupleSourcePOperator());
+ break;
+ }
+ case EXCHANGE: {
+ if (op.getPhysicalOperator() == null) {
+ throw new AlgebricksException("Implementation for EXCHANGE operator was not set.");
+ }
+ // implem. choice for exchange should be set by a parent op.
+ break;
+ }
+ case GROUP: {
+ GroupByOperator gby = (GroupByOperator) op;
+
+ if (gby.getNestedPlans().size() == 1) {
+ ILogicalPlan p0 = gby.getNestedPlans().get(0);
+ if (p0.getRoots().size() == 1) {
+ if (gby.getAnnotations().get(OperatorAnnotations.USE_HASH_GROUP_BY) == Boolean.TRUE
+ || gby.getAnnotations().get(OperatorAnnotations.USE_EXTERNAL_GROUP_BY) == Boolean.TRUE) {
+ if (!topLevelOp) {
+ throw new NotImplementedException(
+ "External hash group-by for nested grouping is not implemented.");
+ }
+ ExternalGroupByPOperator externalGby = new ExternalGroupByPOperator(
+ gby.getGroupByList(), physicalOptimizationConfig.getMaxFramesExternalGroupBy(),
+ physicalOptimizationConfig.getExternalGroupByTableSize());
+ op.setPhysicalOperator(externalGby);
+ generateMergeAggregationExpressions(gby, context);
+ break;
+ }
+ }
+ }
+
+ List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> gbyList = gby.getGroupByList();
+ List<LogicalVariable> columnList = new ArrayList<LogicalVariable>(gbyList.size());
+ for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : gbyList) {
+ ILogicalExpression expr = p.second.getValue();
+ if (expr.getExpressionTag() == LogicalExpressionTag.VARIABLE) {
+ VariableReferenceExpression varRef = (VariableReferenceExpression) expr;
+ columnList.add(varRef.getVariableReference());
+ }
+ }
+ if (topLevelOp) {
+ op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList));
+ } else {
+ op.setPhysicalOperator(new MicroPreclusteredGroupByPOperator(columnList));
+ }
+ break;
+ }
+ case INNERJOIN: {
+ JoinUtils.setJoinAlgorithmAndExchangeAlgo((InnerJoinOperator) op, context);
+ break;
+ }
+ case LEFTOUTERJOIN: {
+ JoinUtils.setJoinAlgorithmAndExchangeAlgo((LeftOuterJoinOperator) op, context);
+ break;
+ }
+ case LIMIT: {
+ LimitOperator opLim = (LimitOperator) op;
+ op.setPhysicalOperator(new StreamLimitPOperator(opLim.isTopmostLimitOp()
+ && opLim.getExecutionMode() == ExecutionMode.PARTITIONED));
+ break;
+ }
+ case NESTEDTUPLESOURCE: {
+ op.setPhysicalOperator(new NestedTupleSourcePOperator());
+ break;
+ }
+ case ORDER: {
+ OrderOperator oo = (OrderOperator) op;
+ for (Pair<IOrder, Mutable<ILogicalExpression>> p : oo.getOrderExpressions()) {
+ ILogicalExpression e = p.second.getValue();
+ if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new AlgebricksException("Order expression " + e + " has not been normalized.");
+ }
+ }
+ if (topLevelOp) {
+ op.setPhysicalOperator(new StableSortPOperator(physicalOptimizationConfig
+ .getMaxFramesExternalSort()));
+ } else {
+ op.setPhysicalOperator(new InMemoryStableSortPOperator());
+ }
+ break;
+ }
+ case PROJECT: {
+ op.setPhysicalOperator(new StreamProjectPOperator());
+ break;
+ }
+ case RUNNINGAGGREGATE: {
+ op.setPhysicalOperator(new RunningAggregatePOperator());
+ break;
+ }
+ case REPLICATE: {
+ op.setPhysicalOperator(new ReplicatePOperator());
+ break;
+ }
+ case SCRIPT: {
+ op.setPhysicalOperator(new StringStreamingScriptPOperator());
+ break;
+ }
+ case SELECT: {
+ op.setPhysicalOperator(new StreamSelectPOperator());
+ break;
+ }
+ case SUBPLAN: {
+ op.setPhysicalOperator(new SubplanPOperator());
+ break;
+ }
+ case UNIONALL: {
+ op.setPhysicalOperator(new UnionAllPOperator());
+ break;
+ }
+
+ case UNNEST: {
+ op.setPhysicalOperator(new UnnestPOperator());
+ break;
+ }
+ case DATASOURCESCAN: {
+ DataSourceScanOperator scan = (DataSourceScanOperator) op;
+ IDataSource dataSource = scan.getDataSource();
+ DataSourceScanPOperator dss = new DataSourceScanPOperator(dataSource);
+ IMetadataProvider mp = context.getMetadataProvider();
+ if (mp.scannerOperatorIsLeaf(dataSource)) {
+ dss.disableJobGenBelowMe();
+ }
+ op.setPhysicalOperator(dss);
+ break;
+ }
+ case WRITE: {
+ op.setPhysicalOperator(new SinkWritePOperator());
+ break;
+ }
+ case WRITE_RESULT: {
+ WriteResultOperator opLoad = (WriteResultOperator) op;
+ LogicalVariable payload;
+ List<LogicalVariable> keys = new ArrayList<LogicalVariable>();
+ payload = getKeysAndLoad(opLoad.getPayloadExpression(), opLoad.getKeyExpressions(), keys);
+ op.setPhysicalOperator(new WriteResultPOperator(opLoad.getDataSource(), payload, keys));
+ break;
+ }
+ case INSERT_DELETE: {
+ InsertDeleteOperator opLoad = (InsertDeleteOperator) op;
+ LogicalVariable payload;
+ List<LogicalVariable> keys = new ArrayList<LogicalVariable>();
+ payload = getKeysAndLoad(opLoad.getPayloadExpression(), opLoad.getPrimaryKeyExpressions(), keys);
+ op.setPhysicalOperator(new InsertDeletePOperator(payload, keys, opLoad.getDataSource()));
+ break;
+ }
+ case INDEX_INSERT_DELETE: {
+ IndexInsertDeleteOperator opLoad = (IndexInsertDeleteOperator) op;
+ List<LogicalVariable> primaryKeys = new ArrayList<LogicalVariable>();
+ List<LogicalVariable> secondaryKeys = new ArrayList<LogicalVariable>();
+ getKeys(opLoad.getPrimaryKeyExpressions(), primaryKeys);
+ getKeys(opLoad.getSecondaryKeyExpressions(), secondaryKeys);
+ op.setPhysicalOperator(new IndexInsertDeletePOperator(primaryKeys, secondaryKeys, opLoad
+ .getDataSourceIndex()));
+ break;
+ }
+ case SINK: {
+ op.setPhysicalOperator(new SinkPOperator());
+ break;
+ }
+ case DIE: {
+ op.setPhysicalOperator(new StreamDiePOperator());
+ break;
+ }
+ }
+ }
+ if (op.hasNestedPlans()) {
+ AbstractOperatorWithNestedPlans nested = (AbstractOperatorWithNestedPlans) op;
+ for (ILogicalPlan p : nested.getNestedPlans()) {
+ setPhysicalOperators(p, false, context);
+ }
+ }
+ for (Mutable<ILogicalOperator> opRef : op.getInputs()) {
+ computeDefaultPhysicalOp((AbstractLogicalOperator) opRef.getValue(), topLevelOp, context);
+ }
+ }
+
+ private static void getKeys(List<Mutable<ILogicalExpression>> keyExpressions, List<LogicalVariable> keys) {
+ for (Mutable<ILogicalExpression> kExpr : keyExpressions) {
+ ILogicalExpression e = kExpr.getValue();
+ if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new NotImplementedException();
+ }
+ keys.add(((VariableReferenceExpression) e).getVariableReference());
+ }
+ }
+
+ private static LogicalVariable getKeysAndLoad(Mutable<ILogicalExpression> payloadExpr,
+ List<Mutable<ILogicalExpression>> keyExpressions, List<LogicalVariable> keys) {
+ LogicalVariable payload;
+ if (payloadExpr.getValue().getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new NotImplementedException();
+ }
+ payload = ((VariableReferenceExpression) payloadExpr.getValue()).getVariableReference();
+
+ for (Mutable<ILogicalExpression> kExpr : keyExpressions) {
+ ILogicalExpression e = kExpr.getValue();
+ if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ throw new NotImplementedException();
+ }
+ keys.add(((VariableReferenceExpression) e).getVariableReference());
+ }
+ return payload;
+ }
+
+ private static void generateMergeAggregationExpressions(GroupByOperator gby, IOptimizationContext context)
+ throws AlgebricksException {
+ if (gby.getNestedPlans().size() != 1) {
+ throw new AlgebricksException(
+ "External group-by currently works only for one nested plan with one root containing"
+ + "an aggregate and a nested-tuple-source.");
+ }
+ ILogicalPlan p0 = gby.getNestedPlans().get(0);
+ if (p0.getRoots().size() != 1) {
+ throw new AlgebricksException(
+ "External group-by currently works only for one nested plan with one root containing"
+ + "an aggregate and a nested-tuple-source.");
+ }
+ IMergeAggregationExpressionFactory mergeAggregationExpressionFactory = context
+ .getMergeAggregationExpressionFactory();
+ Mutable<ILogicalOperator> r0 = p0.getRoots().get(0);
+ AggregateOperator aggOp = (AggregateOperator) r0.getValue();
+ List<Mutable<ILogicalExpression>> aggFuncRefs = aggOp.getExpressions();
+ int n = aggOp.getExpressions().size();
+ List<Mutable<ILogicalExpression>> mergeExpressionRefs = new ArrayList<Mutable<ILogicalExpression>>();
+ for (int i = 0; i < n; i++) {
+ ILogicalExpression mergeExpr = mergeAggregationExpressionFactory.createMergeAggregation(aggFuncRefs.get(i)
+ .getValue(), context);
+ mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
+ }
+ aggOp.setMergeExpressions(mergeExpressionRefs);
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
new file mode 100644
index 0000000..bdd56b6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SetExecutionModeRule.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.NotImplementedException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ *
+ * This rule sets the executionMode property of an operator, w/o introducing
+ * EXCHANGE operators in the plan. Previously, i.e. before having physical
+ * optimizations in place, we were using the IntroduceExchangeRule, which was
+ * doing both, to both set excutionMode and introduce data exchange ops.
+ *
+ *
+ * @author Nicola
+ *
+ */
+public class SetExecutionModeRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ boolean changed = OperatorManipulationUtil.setOperatorMode(op);
+ if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED
+ || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) {
+ return changed;
+ }
+ switch (op.getOperatorTag()) {
+ // case DISTINCT:
+ // case AGGREGATE:
+ // case GROUP:
+ // case ORDER:
+ // case INNERJOIN:
+ // case LEFTOUTERJOIN: {
+ // op.setExecutionMode(ExecutionMode.GLOBAL);
+ // return true;
+ // }
+
+ case PARTITIONINGSPLIT: {
+ throw new NotImplementedException();
+ }
+ default: {
+ return changed;
+ }
+ }
+
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
new file mode 100644
index 0000000..fb7a49f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SimpleUnnestToProductRule.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.ConstantExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractScanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.EmptyTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.InnerJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+public class SimpleUnnestToProductRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue();
+ if (op.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
+ return false;
+ }
+
+ Mutable<ILogicalOperator> opRef2 = op.getInputs().get(0);
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) opRef2.getValue();
+
+ if (!(op2 instanceof AbstractScanOperator) && !descOrSelfIsSourceScan(op2)) {
+ return false;
+ }
+ InnerJoinOperator product = new InnerJoinOperator(
+ new MutableObject<ILogicalExpression>(ConstantExpression.TRUE));
+
+ EmptyTupleSourceOperator ets = new EmptyTupleSourceOperator();
+ context.computeAndSetTypeEnvironmentForOperator(ets);
+ Mutable<ILogicalOperator> emptySrc = new MutableObject<ILogicalOperator>(ets);
+ List<Mutable<ILogicalOperator>> opInpList = op.getInputs();
+ opInpList.clear();
+ opInpList.add(emptySrc);
+ product.getInputs().add(opRef2); // outer branch
+ product.getInputs().add(new MutableObject<ILogicalOperator>(op));
+ opRef.setValue(product); // plug the product in the plan
+ context.computeAndSetTypeEnvironmentForOperator(product);
+ return true;
+ }
+
+ private boolean descOrSelfIsSourceScan(AbstractLogicalOperator op2) {
+ if (op2.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
+ return true;
+ }
+ for (Mutable<ILogicalOperator> cRef : op2.getInputs()) {
+ AbstractLogicalOperator alo = (AbstractLogicalOperator) cRef.getValue();
+ if (descOrSelfIsSourceScan(alo)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
new file mode 100644
index 0000000..7c31fb9
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/rules/SubplanOutOfGroupRule.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.rules;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
+import edu.uci.ics.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+
+/**
+ *
+ * Looks for a nested group-by plan ending in
+ *
+ * subplan {
+ *
+ * ...
+ *
+ * }
+ *
+ * select (function-call: algebricks:not, Args:[function-call:
+ * algebricks:is-null, Args:[...]])
+ *
+ * nested tuple source -- |UNPARTITIONED|
+ *
+ *
+ *
+ */
+
+public class SubplanOutOfGroupRule implements IAlgebraicRewriteRule {
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException {
+ return false;
+ }
+
+ @Override
+ public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ AbstractLogicalOperator op0 = (AbstractLogicalOperator) opRef.getValue();
+ if (op0.getOperatorTag() != LogicalOperatorTag.GROUP) {
+ return false;
+ }
+ GroupByOperator gby = (GroupByOperator) op0;
+
+ Iterator<ILogicalPlan> plansIter = gby.getNestedPlans().iterator();
+ ILogicalPlan p = null;
+ while (plansIter.hasNext()) {
+ p = plansIter.next();
+ }
+ if (p == null) {
+ return false;
+ }
+ if (p.getRoots().size() != 1) {
+ return false;
+ }
+ Mutable<ILogicalOperator> op1Ref = p.getRoots().get(0);
+ AbstractLogicalOperator op1 = (AbstractLogicalOperator) op1Ref.getValue();
+ boolean found = false;
+ while (op1.getInputs().size() == 1) {
+ if (op1.getOperatorTag() == LogicalOperatorTag.SUBPLAN) {
+ SubplanOperator subplan = (SubplanOperator) op1;
+ AbstractLogicalOperator op2 = (AbstractLogicalOperator) subplan.getInputs().get(0).getValue();
+ if (OperatorPropertiesUtil.isNullTest(op2)) {
+ if (subplan.getNestedPlans().size() == 1) {
+ ILogicalPlan p1 = subplan.getNestedPlans().get(0);
+ if (p1.getRoots().size() == 1) {
+ AbstractLogicalOperator r1 = (AbstractLogicalOperator) p1.getRoots().get(0).getValue();
+ if (r1.getOperatorTag() == LogicalOperatorTag.INNERJOIN
+ || r1.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) {
+ // now, check that it propagates all variables,
+ // so it can be pushed
+ List<LogicalVariable> op2Vars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(op2, op2Vars);
+ List<LogicalVariable> op1Vars = new ArrayList<LogicalVariable>();
+ VariableUtilities.getLiveVariables(subplan, op1Vars);
+ if (op1Vars.containsAll(op2Vars)) {
+ found = true;
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+ op1Ref = op1.getInputs().get(0);
+ op1 = (AbstractLogicalOperator) op1Ref.getValue();
+ }
+ if (!found) {
+ return false;
+ }
+
+ ILogicalOperator subplan = op1;
+ ILogicalOperator op2 = op1.getInputs().get(0).getValue();
+ op1Ref.setValue(op2);
+ Mutable<ILogicalOperator> opUnderRef = gby.getInputs().get(0);
+ ILogicalOperator opUnder = opUnderRef.getValue();
+ subplan.getInputs().clear();
+ subplan.getInputs().add(new MutableObject<ILogicalOperator>(opUnder));
+ opUnderRef.setValue(subplan);
+
+ return true;
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
new file mode 100644
index 0000000..1714a24
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/JoinUtils.java
@@ -0,0 +1,216 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.rewriter.util;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalVariable;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation.BroadcastSide;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.AlgebricksBuiltinFunctions.ComparisonKind;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractBinaryJoinOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalPropertiesVisitor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.AbstractJoinPOperator.JoinPartitioningType;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.HybridHashJoinPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.InMemoryHashJoinPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.physical.NLJoinPOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.properties.ILogicalPropertiesVector;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public class JoinUtils {
+
+ private final static int MB = 1048576;
+
+ private final static double DEFAULT_FUDGE_FACTOR = 1.3;
+ private final static int MAX_RECORDS_PER_FRAME = 512;
+ private final static int DEFAULT_FRAME_SIZE = 32768;
+ private final static int MAX_LEFT_INPUT_SIZE_HYBRID_HASH = (int) (140L * 1024 * MB / DEFAULT_FRAME_SIZE);
+ private final static int DEFAULT_MEMORY_SIZE_HYBRID_HASH = (int) (256L * MB / DEFAULT_FRAME_SIZE);
+
+ public static void setJoinAlgorithmAndExchangeAlgo(AbstractBinaryJoinOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ List<LogicalVariable> sideLeft = new LinkedList<LogicalVariable>();
+ List<LogicalVariable> sideRight = new LinkedList<LogicalVariable>();
+ List<LogicalVariable> varsLeft = op.getInputs().get(0).getValue().getSchema();
+ List<LogicalVariable> varsRight = op.getInputs().get(1).getValue().getSchema();
+ if (isHashJoinCondition(op.getCondition().getValue(), varsLeft, varsRight, sideLeft, sideRight)) {
+ BroadcastSide side = getBroadcastJoinSide(op.getCondition().getValue(), varsLeft, varsRight);
+ if (side == null) {
+ setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
+ } else {
+ switch (side) {
+ case RIGHT:
+ setHashJoinOp(op, JoinPartitioningType.BROADCAST, sideLeft, sideRight, context);
+ break;
+ case LEFT:
+ Mutable<ILogicalOperator> opRef0 = op.getInputs().get(0);
+ Mutable<ILogicalOperator> opRef1 = op.getInputs().get(1);
+ ILogicalOperator tmp = opRef0.getValue();
+ opRef0.setValue(opRef1.getValue());
+ opRef1.setValue(tmp);
+ setHashJoinOp(op, JoinPartitioningType.BROADCAST, sideRight, sideLeft, context);
+ break;
+ default:
+ setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context);
+ }
+ }
+ } else {
+ setNLJoinOp(op);
+ }
+ }
+
+ private static void setNLJoinOp(AbstractBinaryJoinOperator op) {
+ op.setPhysicalOperator(new NLJoinPOperator(op.getJoinKind(), JoinPartitioningType.BROADCAST,
+ DEFAULT_MEMORY_SIZE_HYBRID_HASH));
+ }
+
+ private static void setHashJoinOp(AbstractBinaryJoinOperator op, JoinPartitioningType partitioningType,
+ List<LogicalVariable> sideLeft, List<LogicalVariable> sideRight, IOptimizationContext context)
+ throws AlgebricksException {
+ op.setPhysicalOperator(new HybridHashJoinPOperator(op.getJoinKind(), partitioningType, sideLeft, sideRight,
+ DEFAULT_MEMORY_SIZE_HYBRID_HASH, MAX_LEFT_INPUT_SIZE_HYBRID_HASH, MAX_RECORDS_PER_FRAME,
+ DEFAULT_FUDGE_FACTOR));
+ if (partitioningType == JoinPartitioningType.BROADCAST) {
+ hybridToInMemHashJoin(op, context);
+ }
+ // op.setPhysicalOperator(new
+ // InMemoryHashJoinPOperator(op.getJoinKind(), partitioningType,
+ // sideLeft, sideRight,
+ // 1024 * 512));
+ }
+
+ private static void hybridToInMemHashJoin(AbstractBinaryJoinOperator op, IOptimizationContext context)
+ throws AlgebricksException {
+ ILogicalOperator opBuild = op.getInputs().get(1).getValue();
+ LogicalPropertiesVisitor.computeLogicalPropertiesDFS(opBuild, context);
+ ILogicalPropertiesVector v = context.getLogicalPropertiesVector(opBuild);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine("// HybridHashJoin inner branch -- Logical properties for " + opBuild
+ + ": " + v + "\n");
+ if (v != null) {
+ int size2 = v.getMaxOutputFrames();
+ HybridHashJoinPOperator hhj = (HybridHashJoinPOperator) op.getPhysicalOperator();
+ if (size2 > 0 && size2 * hhj.getFudgeFactor() <= hhj.getMemSizeInFrames()) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine("// HybridHashJoin inner branch " + opBuild
+ + " fits in memory\n");
+ // maintains the local properties on the probe side
+ op.setPhysicalOperator(new InMemoryHashJoinPOperator(hhj.getKind(), hhj.getPartitioningType(), hhj
+ .getKeysLeftBranch(), hhj.getKeysRightBranch(), v.getNumberOfTuples() * 2));
+ }
+ }
+
+ }
+
+ private static boolean isHashJoinCondition(ILogicalExpression e, Collection<LogicalVariable> inLeftAll,
+ Collection<LogicalVariable> inRightAll, Collection<LogicalVariable> outLeftFields,
+ Collection<LogicalVariable> outRightFields) {
+ switch (e.getExpressionTag()) {
+ case FUNCTION_CALL: {
+ AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
+ FunctionIdentifier fi = fexp.getFunctionIdentifier();
+ if (fi == AlgebricksBuiltinFunctions.AND) {
+ for (Mutable<ILogicalExpression> a : fexp.getArguments()) {
+ if (!isHashJoinCondition(a.getValue(), inLeftAll, inRightAll, outLeftFields,
+ outRightFields)) {
+ return false;
+ }
+ }
+ return true;
+ } else {
+ ComparisonKind ck = AlgebricksBuiltinFunctions.getComparisonType(fi);
+ if (ck != ComparisonKind.EQ) {
+ return false;
+ }
+ ILogicalExpression opLeft = fexp.getArguments().get(0).getValue();
+ ILogicalExpression opRight = fexp.getArguments().get(1).getValue();
+ if (opLeft.getExpressionTag() != LogicalExpressionTag.VARIABLE
+ || opRight.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
+ return false;
+ }
+ LogicalVariable var1 = ((VariableReferenceExpression) opLeft).getVariableReference();
+ if (inLeftAll.contains(var1) && !outLeftFields.contains(var1)) {
+ outLeftFields.add(var1);
+ } else if (inRightAll.contains(var1) && !outRightFields.contains(var1)) {
+ outRightFields.add(var1);
+ } else {
+ return false;
+ }
+ LogicalVariable var2 = ((VariableReferenceExpression) opRight).getVariableReference();
+ if (inLeftAll.contains(var2) && !outLeftFields.contains(var2)) {
+ outLeftFields.add(var2);
+ } else if (inRightAll.contains(var2) && !outRightFields.contains(var2)) {
+ outRightFields.add(var2);
+ } else {
+ return false;
+ }
+ return true;
+ }
+ }
+ default: {
+ return false;
+ }
+ }
+ }
+
+ private static BroadcastSide getBroadcastJoinSide(ILogicalExpression e, List<LogicalVariable> varsLeft,
+ List<LogicalVariable> varsRight) {
+ if (e.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return null;
+ }
+ AbstractFunctionCallExpression fexp = (AbstractFunctionCallExpression) e;
+ IExpressionAnnotation ann = fexp.getAnnotations().get(BroadcastExpressionAnnotation.BROADCAST_ANNOTATION_KEY);
+ if (ann == null) {
+ return null;
+ }
+ BroadcastSide side = (BroadcastSide) ann.getObject();
+ if (side == null) {
+ return null;
+ }
+ int i;
+ switch (side) {
+ case LEFT:
+ i = 0;
+ break;
+ case RIGHT:
+ i = 1;
+ break;
+ default:
+ return null;
+ }
+ ArrayList<LogicalVariable> vars = new ArrayList<LogicalVariable>();
+ fexp.getArguments().get(i).getValue().getUsedVariables(vars);
+ if (varsLeft.containsAll(vars)) {
+ return BroadcastSide.LEFT;
+ } else if (varsRight.containsAll(vars)) {
+ return BroadcastSide.RIGHT;
+ } else {
+ return null;
+ }
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
new file mode 100644
index 0000000..795c767
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-rewriter/src/main/java/edu/uci/ics/hyracks/algebricks/rewriter/util/PhysicalOptimizationsUtil.java
@@ -0,0 +1,57 @@
+package edu.uci.ics.hyracks.algebricks.rewriter.util;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.lang3.mutable.Mutable;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.ILogicalPlan;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import edu.uci.ics.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.AbstractOperatorWithNestedPlans;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.operators.logical.visitors.FDsAndEquivClassesVisitor;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+
+public class PhysicalOptimizationsUtil {
+
+ public static void computeFDsAndEquivalenceClasses(AbstractLogicalOperator op, IOptimizationContext ctx)
+ throws AlgebricksException {
+ FDsAndEquivClassesVisitor visitor = new FDsAndEquivClassesVisitor();
+ Set<ILogicalOperator> visitSet = new HashSet<ILogicalOperator>();
+ computeFDsAndEqClassesWithVisitorRec(op, ctx, visitor, visitSet);
+ }
+
+ private static void computeFDsAndEqClassesWithVisitorRec(AbstractLogicalOperator op, IOptimizationContext ctx,
+ FDsAndEquivClassesVisitor visitor, Set<ILogicalOperator> visitSet) throws AlgebricksException {
+ visitSet.add(op);
+ for (Mutable<ILogicalOperator> i : op.getInputs()) {
+ computeFDsAndEqClassesWithVisitorRec((AbstractLogicalOperator) i.getValue(), ctx, visitor, visitSet);
+ }
+ if (op.hasNestedPlans()) {
+ for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) {
+ for (Mutable<ILogicalOperator> r : p.getRoots()) {
+ AbstractLogicalOperator rootOp = (AbstractLogicalOperator) r.getValue();
+ computeFDsAndEqClassesWithVisitorRec(rootOp, ctx, visitor, visitSet);
+ }
+ }
+ }
+ if (op.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) {
+ NestedTupleSourceOperator nts = (NestedTupleSourceOperator) op;
+ ILogicalOperator source = nts.getDataSourceReference().getValue().getInputs().get(0).getValue();
+ if (!visitSet.contains(source)) {
+ computeFDsAndEqClassesWithVisitorRec((AbstractLogicalOperator) source, ctx, visitor, visitSet);
+ }
+ }
+ op.accept(visitor, ctx);
+ if (AlgebricksConfig.DEBUG) {
+ AlgebricksConfig.ALGEBRICKS_LOGGER.fine("--> op. type = " + op.getOperatorTag() + "\n"
+ + " equiv. classes = " + ctx.getEquivalenceClassMap(op) + "\n" + " FDs = "
+ + ctx.getFDList(op) + "\n");
+ }
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/build-script.xml b/hyracks-algebricks/hyracks-algebricks-tests/build-script.xml
new file mode 100644
index 0000000..cb41069
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/build-script.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<project name="asterix" default="build" basedir=".">
+
+ <target name="init">
+ <condition property="isWindows">
+ <os family="windows" />
+ </condition>
+ <condition property="isUnix">
+ <os family="unix" />
+ </condition>
+ </target>
+
+ <target name="win" depends="init" if="isWindows">
+ <property name="os.family" value="windows" />
+ <property name="source.ext" value=".cmd" />
+ <property name="target.ext" value=".cmd" />
+ </target>
+
+ <target name="unix" depends="init" if="isUnix">
+ <property name="os.family" value="unix" />
+ <property name="source.ext" value=".sh" />
+ <property name="target.ext" value="" />
+ </target>
+
+ <target name="build" depends="win,unix">
+ <mkdir dir="${target.dir}"/>
+ <pathconvert targetos="${os.family}" property="classpath">
+ <path>
+ <pathelement path="${script.classpath}"/>
+ </path>
+ </pathconvert>
+ <copy file="${source}${source.ext}" tofile="${target.dir}/${target}${target.ext}">
+ <filterset>
+ <filter token="main.class" value="${main.class}"/>
+ <filter token="classpath" value="${classpath}"/>
+ <filter token="jvm.params" value="${jvm.params}"/>
+ <filter token="program.params" value="${program.params}"/>
+ </filterset>
+ </copy>
+ <chmod perm="ugo+x">
+ <fileset dir="${target.dir}"/>
+ </chmod>
+ </target>
+
+</project>
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/simple/int-part1.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/simple/int-part1.tbl
new file mode 100644
index 0000000..91a5152
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/simple/int-part1.tbl
@@ -0,0 +1,7 @@
+0|
+2|
+4|
+6|
+8|
+10|
+12|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/simple/int-part2.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/simple/int-part2.tbl
new file mode 100644
index 0000000..a481d5d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/simple/int-part2.tbl
@@ -0,0 +1,6 @@
+1|
+3|
+5|
+7|
+9|
+11|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/customer-part1.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/customer-part1.tbl
new file mode 100644
index 0000000..8395f01
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/customer-part1.tbl
@@ -0,0 +1,75 @@
+1|Customer#000000001|IVhzIApeRb ot,c,E|15|25-989-741-2988|711.56|BUILDING|to the even, regular platelets. regular, ironic epitaphs nag e|
+2|Customer#000000002|XSTf4,NCwDVaWNe6tEgvwfmRchLXak|13|23-768-687-3665|121.65|AUTOMOBILE|l accounts. blithely ironic theodolites integrate boldly: caref|
+3|Customer#000000003|MG9kdTD2WBHm|1|11-719-748-3364|7498.12|AUTOMOBILE| deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov|
+4|Customer#000000004|XxVSJsLAGtn|4|14-128-190-5944|2866.83|MACHINERY| requests. final, regular ideas sleep final accou|
+5|Customer#000000005|KvpyuHCplrB84WgAiGV6sYpZq7Tj|3|13-750-942-6364|794.47|HOUSEHOLD|n accounts will have to unwind. foxes cajole accor|
+6|Customer#000000006|sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn|20|30-114-968-4951|7638.57|AUTOMOBILE|tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious|
+7|Customer#000000007|TcGe5gaZNgVePxU5kRrvXBfkasDTea|18|28-190-982-9759|9561.95|AUTOMOBILE|ainst the ironic, express theodolites. express, even pinto beans among the exp|
+8|Customer#000000008|I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5|17|27-147-574-9335|6819.74|BUILDING|among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide|
+9|Customer#000000009|xKiAFTjUsCuxfeleNqefumTrjS|8|18-338-906-3675|8324.07|FURNITURE|r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl|
+10|Customer#000000010|6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2|5|15-741-346-9870|2753.54|HOUSEHOLD|es regular deposits haggle. fur|
+11|Customer#000000011|PkWS 3HlXqwTuzrKg633BEi|23|33-464-151-3439|-272.60|BUILDING|ckages. requests sleep slyly. quickly even pinto beans promise above the slyly regular pinto beans. |
+12|Customer#000000012|9PWKuhzT4Zr1Q|13|23-791-276-1263|3396.49|HOUSEHOLD| to the carefully final braids. blithely regular requests nag. ironic theodolites boost quickly along|
+13|Customer#000000013|nsXQu0oVjD7PM659uC3SRSp|3|13-761-547-5974|3857.34|BUILDING|ounts sleep carefully after the close frays. carefully bold notornis use ironic requests. blithely|
+14|Customer#000000014|KXkletMlL2JQEA |1|11-845-129-3851|5266.30|FURNITURE|, ironic packages across the unus|
+15|Customer#000000015|YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn|23|33-687-542-7601|2788.52|HOUSEHOLD| platelets. regular deposits detect asymptotes. blithely unusual packages nag slyly at the fluf|
+16|Customer#000000016|cYiaeMLZSMAOQ2 d0W,|10|20-781-609-3107|4681.03|FURNITURE|kly silent courts. thinly regular theodolites sleep fluffily after |
+17|Customer#000000017|izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7|2|12-970-682-3487|6.34|AUTOMOBILE|packages wake! blithely even pint|
+18|Customer#000000018|3txGO AiuFux3zT0Z9NYaFRnZt|6|16-155-215-1315|5494.43|BUILDING|s sleep. carefully even instructions nag furiously alongside of t|
+19|Customer#000000019|uc,3bHIx84H,wdrmLOjVsiqXCq2tr|18|28-396-526-5053|8914.71|HOUSEHOLD| nag. furiously careful packages are slyly at the accounts. furiously regular in|
+20|Customer#000000020|JrPk8Pqplj4Ne|22|32-957-234-8742|7603.40|FURNITURE|g alongside of the special excuses-- fluffily enticing packages wake |
+21|Customer#000000021|XYmVpr9yAHDEn|8|18-902-614-8344|1428.25|MACHINERY| quickly final accounts integrate blithely furiously u|
+22|Customer#000000022|QI6p41,FNs5k7RZoCCVPUTkUdYpB|3|13-806-545-9701|591.98|MACHINERY|s nod furiously above the furiously ironic ideas. |
+23|Customer#000000023|OdY W13N7Be3OC5MpgfmcYss0Wn6TKT|3|13-312-472-8245|3332.02|HOUSEHOLD|deposits. special deposits cajole slyly. fluffily special deposits about the furiously |
+24|Customer#000000024|HXAFgIAyjxtdqwimt13Y3OZO 4xeLe7U8PqG|13|23-127-851-8031|9255.67|MACHINERY|into beans. fluffily final ideas haggle fluffily|
+25|Customer#000000025|Hp8GyFQgGHFYSilH5tBfe|12|22-603-468-3533|7133.70|FURNITURE|y. accounts sleep ruthlessly according to the regular theodolites. unusual instructions sleep. ironic, final|
+26|Customer#000000026|8ljrc5ZeMl7UciP|22|32-363-455-4837|5182.05|AUTOMOBILE|c requests use furiously ironic requests. slyly ironic dependencies us|
+27|Customer#000000027|IS8GIyxpBrLpMT0u7|3|13-137-193-2709|5679.84|BUILDING| about the carefully ironic pinto beans. accoun|
+28|Customer#000000028|iVyg0daQ,Tha8x2WPWA9m2529m|8|18-774-241-1462|1007.18|FURNITURE| along the regular deposits. furiously final pac|
+29|Customer#000000029|sJ5adtfyAkCK63df2,vF25zyQMVYE34uh|0|10-773-203-7342|7618.27|FURNITURE|its after the carefully final platelets x-ray against |
+30|Customer#000000030|nJDsELGAavU63Jl0c5NKsKfL8rIJQQkQnYL2QJY|1|11-764-165-5076|9321.01|BUILDING|lithely final requests. furiously unusual account|
+31|Customer#000000031|LUACbO0viaAv6eXOAebryDB xjVst|23|33-197-837-7094|5236.89|HOUSEHOLD|s use among the blithely pending depo|
+32|Customer#000000032|jD2xZzi UmId,DCtNBLXKj9q0Tlp2iQ6ZcO3J|15|25-430-914-2194|3471.53|BUILDING|cial ideas. final, furious requests across the e|
+33|Customer#000000033|qFSlMuLucBmx9xnn5ib2csWUweg D|17|27-375-391-1280|-78.56|AUTOMOBILE|s. slyly regular accounts are furiously. carefully pending requests|
+34|Customer#000000034|Q6G9wZ6dnczmtOx509xgE,M2KV|15|25-344-968-5422|8589.70|HOUSEHOLD|nder against the even, pending accounts. even|
+35|Customer#000000035|TEjWGE4nBzJL2|17|27-566-888-7431|1228.24|HOUSEHOLD|requests. special, express requests nag slyly furiousl|
+36|Customer#000000036|3TvCzjuPzpJ0,DdJ8kW5U|21|31-704-669-5769|4987.27|BUILDING|haggle. enticing, quiet platelets grow quickly bold sheaves. carefully regular acc|
+37|Customer#000000037|7EV4Pwh,3SboctTWt|8|18-385-235-7162|-917.75|FURNITURE|ilent packages are carefully among the deposits. furiousl|
+38|Customer#000000038|a5Ee5e9568R8RLP 2ap7|12|22-306-880-7212|6345.11|HOUSEHOLD|lar excuses. closely even asymptotes cajole blithely excuses. carefully silent pinto beans sleep carefully fin|
+39|Customer#000000039|nnbRg,Pvy33dfkorYE FdeZ60|2|12-387-467-6509|6264.31|AUTOMOBILE|tions. slyly silent excuses slee|
+40|Customer#000000040|gOnGWAyhSV1ofv|3|13-652-915-8939|1335.30|BUILDING|rges impress after the slyly ironic courts. foxes are. blithely |
+41|Customer#000000041|IM9mzmyoxeBmvNw8lA7G3Ydska2nkZF|10|20-917-711-4011|270.95|HOUSEHOLD|ly regular accounts hang bold, silent packages. unusual foxes haggle slyly above the special, final depo|
+42|Customer#000000042|ziSrvyyBke|5|15-416-330-4175|8727.01|BUILDING|ssly according to the pinto beans: carefully special requests across the even, pending accounts wake special|
+43|Customer#000000043|ouSbjHk8lh5fKX3zGso3ZSIj9Aa3PoaFd|19|29-316-665-2897|9904.28|MACHINERY|ial requests: carefully pending foxes detect quickly. carefully final courts cajole quickly. carefully|
+44|Customer#000000044|Oi,dOSPwDu4jo4x,,P85E0dmhZGvNtBwi|16|26-190-260-5375|7315.94|AUTOMOBILE|r requests around the unusual, bold a|
+45|Customer#000000045|4v3OcpFgoOmMG,CbnF,4mdC|9|19-715-298-9917|9983.38|AUTOMOBILE|nto beans haggle slyly alongside of t|
+46|Customer#000000046|eaTXWWm10L9|6|16-357-681-2007|5744.59|AUTOMOBILE|ctions. accounts sleep furiously even requests. regular, regular accounts cajole blithely around the final pa|
+47|Customer#000000047|b0UgocSqEW5 gdVbhNT|2|12-427-271-9466|274.58|BUILDING|ions. express, ironic instructions sleep furiously ironic ideas. furi|
+48|Customer#000000048|0UU iPhBupFvemNB|0|10-508-348-5882|3792.50|BUILDING|re fluffily pending foxes. pending, bold platelets sleep slyly. even platelets cajo|
+49|Customer#000000049|cNgAeX7Fqrdf7HQN9EwjUa4nxT,68L FKAxzl|10|20-908-631-4424|4573.94|FURNITURE|nusual foxes! fluffily pending packages maintain to the regular |
+50|Customer#000000050|9SzDYlkzxByyJ1QeTI o|6|16-658-112-3221|4266.13|MACHINERY|ts. furiously ironic accounts cajole furiously slyly ironic dinos.|
+51|Customer#000000051|uR,wEaiTvo4|12|22-344-885-4251|855.87|FURNITURE|eposits. furiously regular requests integrate carefully packages. furious|
+52|Customer#000000052|7 QOqGqqSy9jfV51BC71jcHJSD0|11|21-186-284-5998|5630.28|HOUSEHOLD|ic platelets use evenly even accounts. stealthy theodolites cajole furiou|
+53|Customer#000000053|HnaxHzTfFTZs8MuCpJyTbZ47Cm4wFOOgib|15|25-168-852-5363|4113.64|HOUSEHOLD|ar accounts are. even foxes are blithely. fluffily pending deposits boost|
+54|Customer#000000054|,k4vf 5vECGWFy,hosTE,|4|14-776-370-4745|868.90|AUTOMOBILE|sual, silent accounts. furiously express accounts cajole special deposits. final, final accounts use furi|
+55|Customer#000000055|zIRBR4KNEl HzaiV3a i9n6elrxzDEh8r8pDom|10|20-180-440-8525|4572.11|MACHINERY|ully unusual packages wake bravely bold packages. unusual requests boost deposits! blithely ironic packages ab|
+56|Customer#000000056|BJYZYJQk4yD5B|10|20-895-685-6920|6530.86|FURNITURE|. notornis wake carefully. carefully fluffy requests are furiously even accounts. slyly expre|
+57|Customer#000000057|97XYbsuOPRXPWU|21|31-835-306-1650|4151.93|AUTOMOBILE|ove the carefully special packages. even, unusual deposits sleep slyly pend|
+58|Customer#000000058|g9ap7Dk1Sv9fcXEWjpMYpBZIRUohi T|13|23-244-493-2508|6478.46|HOUSEHOLD|ideas. ironic ideas affix furiously express, final instructions. regular excuses use quickly e|
+59|Customer#000000059|zLOCP0wh92OtBihgspOGl4|1|11-355-584-3112|3458.60|MACHINERY|ously final packages haggle blithely after the express deposits. furiou|
+60|Customer#000000060|FyodhjwMChsZmUz7Jz0H|12|22-480-575-5866|2741.87|MACHINERY|latelets. blithely unusual courts boost furiously about the packages. blithely final instruct|
+61|Customer#000000061|9kndve4EAJxhg3veF BfXr7AqOsT39o gtqjaYE|17|27-626-559-8599|1536.24|FURNITURE|egular packages shall have to impress along the |
+62|Customer#000000062|upJK2Dnw13,|7|17-361-978-7059|595.61|MACHINERY|kly special dolphins. pinto beans are slyly. quickly regular accounts are furiously a|
+63|Customer#000000063|IXRSpVWWZraKII|21|31-952-552-9584|9331.13|AUTOMOBILE|ithely even accounts detect slyly above the fluffily ir|
+64|Customer#000000064|MbCeGY20kaKK3oalJD,OT|3|13-558-731-7204|-646.64|BUILDING|structions after the quietly ironic theodolites cajole be|
+65|Customer#000000065|RGT yzQ0y4l0H90P783LG4U95bXQFDRXbWa1sl,X|23|33-733-623-5267|8795.16|AUTOMOBILE|y final foxes serve carefully. theodolites are carefully. pending i|
+66|Customer#000000066|XbsEqXH1ETbJYYtA1A|22|32-213-373-5094|242.77|HOUSEHOLD|le slyly accounts. carefully silent packages benea|
+67|Customer#000000067|rfG0cOgtr5W8 xILkwp9fpCS8|9|19-403-114-4356|8166.59|MACHINERY|indle furiously final, even theodo|
+68|Customer#000000068|o8AibcCRkXvQFh8hF,7o|12|22-918-832-2411|6853.37|HOUSEHOLD| pending pinto beans impress realms. final dependencies |
+69|Customer#000000069|Ltx17nO9Wwhtdbe9QZVxNgP98V7xW97uvSH1prEw|9|19-225-978-5670|1709.28|HOUSEHOLD|thely final ideas around the quickly final dependencies affix carefully quickly final theodolites. final accounts c|
+70|Customer#000000070|mFowIuhnHjp2GjCiYYavkW kUwOjIaTCQ|22|32-828-107-2832|4867.52|FURNITURE|fter the special asymptotes. ideas after the unusual frets cajole quickly regular pinto be|
+71|Customer#000000071|TlGalgdXWBmMV,6agLyWYDyIz9MKzcY8gl,w6t1B|7|17-710-812-5403|-611.19|HOUSEHOLD|g courts across the regular, final pinto beans are blithely pending ac|
+72|Customer#000000072|putjlmskxE,zs,HqeIA9Wqu7dhgH5BVCwDwHHcf|2|12-759-144-9689|-362.86|FURNITURE|ithely final foxes sleep always quickly bold accounts. final wat|
+73|Customer#000000073|8IhIxreu4Ug6tt5mog4|0|10-473-439-3214|4288.50|BUILDING|usual, unusual packages sleep busily along the furiou|
+74|Customer#000000074|IkJHCA3ZThF7qL7VKcrU nRLl,kylf |4|14-199-862-7209|2764.43|MACHINERY|onic accounts. blithely slow packages would haggle carefully. qui|
+75|Customer#000000075|Dh 6jZ,cwxWLKQfRKkiGrzv6pm|18|28-247-803-9025|6684.10|AUTOMOBILE| instructions cajole even, even deposits. finally bold deposits use above the even pains. slyl|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/customer-part2.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/customer-part2.tbl
new file mode 100644
index 0000000..3cb282c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/customer-part2.tbl
@@ -0,0 +1,75 @@
+76|Customer#000000076|m3sbCvjMOHyaOofH,e UkGPtqc4|0|10-349-718-3044|5745.33|FURNITURE|pecial deposits. ironic ideas boost blithely according to the closely ironic theodolites! furiously final deposits n|
+77|Customer#000000077|4tAE5KdMFGD4byHtXF92vx|17|27-269-357-4674|1738.87|BUILDING|uffily silent requests. carefully ironic asymptotes among the ironic hockey players are carefully bli|
+78|Customer#000000078|HBOta,ZNqpg3U2cSL0kbrftkPwzX|9|19-960-700-9191|7136.97|FURNITURE|ests. blithely bold pinto beans h|
+79|Customer#000000079|n5hH2ftkVRwW8idtD,BmM2|15|25-147-850-4166|5121.28|MACHINERY|es. packages haggle furiously. regular, special requests poach after the quickly express ideas. blithely pending re|
+80|Customer#000000080|K,vtXp8qYB |0|10-267-172-7101|7383.53|FURNITURE|tect among the dependencies. bold accounts engage closely even pinto beans. ca|
+81|Customer#000000081|SH6lPA7JiiNC6dNTrR|20|30-165-277-3269|2023.71|BUILDING|r packages. fluffily ironic requests cajole fluffily. ironically regular theodolit|
+82|Customer#000000082|zhG3EZbap4c992Gj3bK,3Ne,Xn|18|28-159-442-5305|9468.34|AUTOMOBILE|s wake. bravely regular accounts are furiously. regula|
+83|Customer#000000083|HnhTNB5xpnSF20JBH4Ycs6psVnkC3RDf|22|32-817-154-4122|6463.51|BUILDING|ccording to the quickly bold warhorses. final, regular foxes integrate carefully. bold packages nag blithely ev|
+84|Customer#000000084|lpXz6Fwr9945rnbtMc8PlueilS1WmASr CB|11|21-546-818-3802|5174.71|FURNITURE|ly blithe foxes. special asymptotes haggle blithely against the furiously regular depo|
+85|Customer#000000085|siRerlDwiolhYR 8FgksoezycLj|5|15-745-585-8219|3386.64|FURNITURE|ronic ideas use above the slowly pendin|
+86|Customer#000000086|US6EGGHXbTTXPL9SBsxQJsuvy|0|10-677-951-2353|3306.32|HOUSEHOLD|quests. pending dugouts are carefully aroun|
+87|Customer#000000087|hgGhHVSWQl 6jZ6Ev|23|33-869-884-7053|6327.54|FURNITURE|hely ironic requests integrate according to the ironic accounts. slyly regular pla|
+88|Customer#000000088|wtkjBN9eyrFuENSMmMFlJ3e7jE5KXcg|16|26-516-273-2566|8031.44|AUTOMOBILE|s are quickly above the quickly ironic instructions; even requests about the carefully final deposi|
+89|Customer#000000089|dtR, y9JQWUO6FoJExyp8whOU|14|24-394-451-5404|1530.76|FURNITURE|counts are slyly beyond the slyly final accounts. quickly final ideas wake. r|
+90|Customer#000000090|QxCzH7VxxYUWwfL7|16|26-603-491-1238|7354.23|BUILDING|sly across the furiously even |
+91|Customer#000000091|S8OMYFrpHwoNHaGBeuS6E 6zhHGZiprw1b7 q|8|18-239-400-3677|4643.14|AUTOMOBILE|onic accounts. fluffily silent pinto beans boost blithely according to the fluffily exp|
+92|Customer#000000092|obP PULk2LH LqNF,K9hcbNqnLAkJVsl5xqSrY,|2|12-446-416-8471|1182.91|MACHINERY|. pinto beans hang slyly final deposits. ac|
+93|Customer#000000093|EHXBr2QGdh|7|17-359-388-5266|2182.52|MACHINERY|press deposits. carefully regular platelets r|
+94|Customer#000000094|IfVNIN9KtkScJ9dUjK3Pg5gY1aFeaXewwf|9|19-953-499-8833|5500.11|HOUSEHOLD|latelets across the bold, final requests sleep according to the fluffily bold accounts. unusual deposits amon|
+95|Customer#000000095|EU0xvmWvOmUUn5J,2z85DQyG7QCJ9Xq7|15|25-923-255-2929|5327.38|MACHINERY|ithely. ruthlessly final requests wake slyly alongside of the furiously silent pinto beans. even the|
+96|Customer#000000096|vWLOrmXhRR|8|18-422-845-1202|6323.92|AUTOMOBILE|press requests believe furiously. carefully final instructions snooze carefully. |
+97|Customer#000000097|OApyejbhJG,0Iw3j rd1M|17|27-588-919-5638|2164.48|AUTOMOBILE|haggle slyly. bold, special ideas are blithely above the thinly bold theo|
+98|Customer#000000098|7yiheXNSpuEAwbswDW|12|22-885-845-6889|-551.37|BUILDING|ages. furiously pending accounts are quickly carefully final foxes: busily pe|
+99|Customer#000000099|szsrOiPtCHVS97Lt|15|25-515-237-9232|4088.65|HOUSEHOLD|cajole slyly about the regular theodolites! furiously bold requests nag along the pending, regular packages. somas|
+100|Customer#000000100|fptUABXcmkC5Wx|20|30-749-445-4907|9889.89|FURNITURE|was furiously fluffily quiet deposits. silent, pending requests boost against |
+101|Customer#000000101|sMmL2rNeHDltovSm Y|2|12-514-298-3699|7470.96|MACHINERY| sleep. pending packages detect slyly ironic pack|
+102|Customer#000000102|UAtflJ06 fn9zBfKjInkQZlWtqaA|19|29-324-978-8538|8462.17|BUILDING|ously regular dependencies nag among the furiously express dinos. blithely final|
+103|Customer#000000103|8KIsQX4LJ7QMsj6DrtFtXu0nUEdV,8a|9|19-216-107-2107|2757.45|BUILDING|furiously pending notornis boost slyly around the blithely ironic ideas? final, even instructions cajole fl|
+104|Customer#000000104|9mcCK L7rt0SwiYtrbO88DiZS7U d7M|10|20-966-284-8065|-588.38|FURNITURE|rate carefully slyly special pla|
+105|Customer#000000105|4iSJe4L SPjg7kJj98Yz3z0B|10|20-793-553-6417|9091.82|MACHINERY|l pains cajole even accounts. quietly final instructi|
+106|Customer#000000106|xGCOEAUjUNG|1|11-751-989-4627|3288.42|MACHINERY|lose slyly. ironic accounts along the evenly regular theodolites wake about the special, final gifts. |
+107|Customer#000000107|Zwg64UZ,q7GRqo3zm7P1tZIRshBDz|15|25-336-529-9919|2514.15|AUTOMOBILE|counts cajole slyly. regular requests wake. furiously regular deposits about the blithely final fo|
+108|Customer#000000108|GPoeEvpKo1|5|15-908-619-7526|2259.38|BUILDING|refully ironic deposits sleep. regular, unusual requests wake slyly|
+109|Customer#000000109|OOOkYBgCMzgMQXUmkocoLb56rfrdWp2NE2c|16|26-992-422-8153|-716.10|BUILDING|es. fluffily final dependencies sleep along the blithely even pinto beans. final deposits haggle furiously furiou|
+110|Customer#000000110|mymPfgphaYXNYtk|10|20-893-536-2069|7462.99|AUTOMOBILE|nto beans cajole around the even, final deposits. quickly bold packages according to the furiously regular dept|
+111|Customer#000000111|CBSbPyOWRorloj2TBvrK9qp9tHBs|22|32-582-283-7528|6505.26|MACHINERY|ly unusual instructions detect fluffily special deposits-- theodolites nag carefully during the ironic dependencies|
+112|Customer#000000112|RcfgG3bO7QeCnfjqJT1|19|29-233-262-8382|2953.35|FURNITURE|rmanently unusual multipliers. blithely ruthless deposits are furiously along the|
+113|Customer#000000113|eaOl5UBXIvdY57rglaIzqvfPD,MYfK|12|22-302-930-4756|2912.00|BUILDING|usly regular theodolites boost furiously doggedly pending instructio|
+114|Customer#000000114|xAt 5f5AlFIU|14|24-805-212-7646|1027.46|FURNITURE|der the carefully express theodolites are after the packages. packages are. bli|
+115|Customer#000000115|0WFt1IXENmUT2BgbsB0ShVKJZt0HCBCbFl0aHc|8|18-971-699-1843|7508.92|HOUSEHOLD|sits haggle above the carefully ironic theodolite|
+116|Customer#000000116|yCuVxIgsZ3,qyK2rloThy3u|16|26-632-309-5792|8403.99|BUILDING|as. quickly final sauternes haggle slyly carefully even packages. brave, ironic pinto beans are above the furious|
+117|Customer#000000117|uNhM,PzsRA3S,5Y Ge5Npuhi|24|34-403-631-3505|3950.83|FURNITURE|affix. instructions are furiously sl|
+118|Customer#000000118|OVnFuHygK9wx3xpg8|18|28-639-943-7051|3582.37|AUTOMOBILE|uick packages alongside of the furiously final deposits haggle above the fluffily even foxes. blithely dogged dep|
+119|Customer#000000119|M1ETOIecuvH8DtM0Y0nryXfW|7|17-697-919-8406|3930.35|FURNITURE|express ideas. blithely ironic foxes thrash. special acco|
+120|Customer#000000120|zBNna00AEInqyO1|12|22-291-534-1571|363.75|MACHINERY| quickly. slyly ironic requests cajole blithely furiously final dependen|
+121|Customer#000000121|tv nCR2YKupGN73mQudO|17|27-411-990-2959|6428.32|BUILDING|uriously stealthy ideas. carefully final courts use carefully|
+122|Customer#000000122|yp5slqoNd26lAENZW3a67wSfXA6hTF|3|13-702-694-4520|7865.46|HOUSEHOLD| the special packages hinder blithely around the permanent requests. bold depos|
+123|Customer#000000123|YsOnaaER8MkvK5cpf4VSlq|5|15-817-151-1168|5897.83|BUILDING|ependencies. regular, ironic requests are fluffily regu|
+124|Customer#000000124|aTbyVAW5tCd,v09O|18|28-183-750-7809|1842.49|AUTOMOBILE|le fluffily even dependencies. quietly s|
+125|Customer#000000125|,wSZXdVR xxIIfm9s8ITyLl3kgjT6UC07GY0Y|19|29-261-996-3120|-234.12|FURNITURE|x-ray finally after the packages? regular requests c|
+126|Customer#000000126|ha4EHmbx3kg DYCsP6DFeUOmavtQlHhcfaqr|22|32-755-914-7592|1001.39|HOUSEHOLD|s about the even instructions boost carefully furiously ironic pearls. ruthless, |
+127|Customer#000000127|Xyge4DX2rXKxXyye1Z47LeLVEYMLf4Bfcj|21|31-101-672-2951|9280.71|MACHINERY|ic, unusual theodolites nod silently after the final, ironic instructions: pending r|
+128|Customer#000000128|AmKUMlJf2NRHcKGmKjLS|4|14-280-874-8044|-986.96|HOUSEHOLD|ing packages integrate across the slyly unusual dugouts. blithely silent ideas sublate carefully. blithely expr|
+129|Customer#000000129|q7m7rbMM0BpaCdmxloCgBDRCleXsXkdD8kf|7|17-415-148-7416|9127.27|HOUSEHOLD| unusual deposits boost carefully furiously silent ideas. pending accounts cajole slyly across|
+130|Customer#000000130|RKPx2OfZy0Vn 8wGWZ7F2EAvmMORl1k8iH|9|19-190-993-9281|5073.58|HOUSEHOLD|ix slowly. express packages along the furiously ironic requests integrate daringly deposits. fur|
+131|Customer#000000131|jyN6lAjb1FtH10rMC,XzlWyCBrg75|11|21-840-210-3572|8595.53|HOUSEHOLD|jole special packages. furiously final dependencies about the furiously speci|
+132|Customer#000000132|QM5YabAsTLp9|4|14-692-150-9717|162.57|HOUSEHOLD|uickly carefully special theodolites. carefully regular requests against the blithely unusual instructions |
+133|Customer#000000133|IMCuXdpIvdkYO92kgDGuyHgojcUs88p|17|27-408-997-8430|2314.67|AUTOMOBILE|t packages. express pinto beans are blithely along the unusual, even theodolites. silent packages use fu|
+134|Customer#000000134|sUiZ78QCkTQPICKpA9OBzkUp2FM|11|21-200-159-5932|4608.90|BUILDING|yly fluffy foxes boost final ideas. b|
+135|Customer#000000135|oZK,oC0 fdEpqUML|19|29-399-293-6241|8732.91|FURNITURE| the slyly final accounts. deposits cajole carefully. carefully sly packag|
+136|Customer#000000136|QoLsJ0v5C1IQbh,DS1|7|17-501-210-4726|-842.39|FURNITURE|ackages sleep ironic, final courts. even requests above the blithely bold requests g|
+137|Customer#000000137|cdW91p92rlAEHgJafqYyxf1Q|16|26-777-409-5654|7838.30|HOUSEHOLD|carefully regular theodolites use. silent dolphins cajo|
+138|Customer#000000138|5uyLAeY7HIGZqtu66Yn08f|5|15-394-860-4589|430.59|MACHINERY|ts doze on the busy ideas. regular|
+139|Customer#000000139|3ElvBwudHKL02732YexGVFVt |9|19-140-352-1403|7897.78|MACHINERY|nstructions. quickly ironic ideas are carefully. bold, |
+140|Customer#000000140|XRqEPiKgcETII,iOLDZp5jA|4|14-273-885-6505|9963.15|MACHINERY|ies detect slyly ironic accounts. slyly ironic theodolites hag|
+141|Customer#000000141|5IW,WROVnikc3l7DwiUDGQNGsLBGOL6Dc0|1|11-936-295-6204|6706.14|FURNITURE|packages nag furiously. carefully unusual accounts snooze according to the fluffily regular pinto beans. slyly spec|
+142|Customer#000000142|AnJ5lxtLjioClr2khl9pb8NLxG2,|9|19-407-425-2584|2209.81|AUTOMOBILE|. even, express theodolites upo|
+143|Customer#000000143|681r22uL452zqk 8By7I9o9enQfx0|16|26-314-406-7725|2186.50|MACHINERY|across the blithely unusual requests haggle theodo|
+144|Customer#000000144|VxYZ3ebhgbltnetaGjNC8qCccjYU05 fePLOno8y|1|11-717-379-4478|6417.31|MACHINERY|ges. slyly regular accounts are slyly. bold, idle reque|
+145|Customer#000000145|kQjHmt2kcec cy3hfMh969u|13|23-562-444-8454|9748.93|HOUSEHOLD|ests? express, express instructions use. blithely fina|
+146|Customer#000000146|GdxkdXG9u7iyI1,,y5tq4ZyrcEy|3|13-835-723-3223|3328.68|FURNITURE|ffily regular dinos are slyly unusual requests. slyly specia|
+147|Customer#000000147|6VvIwbVdmcsMzuu,C84GtBWPaipGfi7DV|18|28-803-187-4335|8071.40|AUTOMOBILE|ress packages above the blithely regular packages sleep fluffily blithely ironic accounts. |
+148|Customer#000000148|BhSPlEWGvIJyT9swk vCWE|11|21-562-498-6636|2135.60|HOUSEHOLD|ing to the carefully ironic requests. carefully regular dependencies about the theodolites wake furious|
+149|Customer#000000149|3byTHCp2mNLPigUrrq|19|29-797-439-6760|8959.65|AUTOMOBILE|al instructions haggle against the slyly bold w|
+150|Customer#000000150|zeoGShTjCwGPplOWFkLURrh41O0AZ8dwNEEN4 |18|28-328-564-7630|3849.48|MACHINERY|ole blithely among the furiously pending packages. furiously bold ideas wake fluffily ironic idea|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/customer.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/customer.tbl
new file mode 100644
index 0000000..5d39c80
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/customer.tbl
@@ -0,0 +1,150 @@
+1|Customer#000000001|IVhzIApeRb ot,c,E|15|25-989-741-2988|711.56|BUILDING|to the even, regular platelets. regular, ironic epitaphs nag e|
+2|Customer#000000002|XSTf4,NCwDVaWNe6tEgvwfmRchLXak|13|23-768-687-3665|121.65|AUTOMOBILE|l accounts. blithely ironic theodolites integrate boldly: caref|
+3|Customer#000000003|MG9kdTD2WBHm|1|11-719-748-3364|7498.12|AUTOMOBILE| deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov|
+4|Customer#000000004|XxVSJsLAGtn|4|14-128-190-5944|2866.83|MACHINERY| requests. final, regular ideas sleep final accou|
+5|Customer#000000005|KvpyuHCplrB84WgAiGV6sYpZq7Tj|3|13-750-942-6364|794.47|HOUSEHOLD|n accounts will have to unwind. foxes cajole accor|
+6|Customer#000000006|sKZz0CsnMD7mp4Xd0YrBvx,LREYKUWAh yVn|20|30-114-968-4951|7638.57|AUTOMOBILE|tions. even deposits boost according to the slyly bold packages. final accounts cajole requests. furious|
+7|Customer#000000007|TcGe5gaZNgVePxU5kRrvXBfkasDTea|18|28-190-982-9759|9561.95|AUTOMOBILE|ainst the ironic, express theodolites. express, even pinto beans among the exp|
+8|Customer#000000008|I0B10bB0AymmC, 0PrRYBCP1yGJ8xcBPmWhl5|17|27-147-574-9335|6819.74|BUILDING|among the slyly regular theodolites kindle blithely courts. carefully even theodolites haggle slyly along the ide|
+9|Customer#000000009|xKiAFTjUsCuxfeleNqefumTrjS|8|18-338-906-3675|8324.07|FURNITURE|r theodolites according to the requests wake thinly excuses: pending requests haggle furiousl|
+10|Customer#000000010|6LrEaV6KR6PLVcgl2ArL Q3rqzLzcT1 v2|5|15-741-346-9870|2753.54|HOUSEHOLD|es regular deposits haggle. fur|
+11|Customer#000000011|PkWS 3HlXqwTuzrKg633BEi|23|33-464-151-3439|-272.60|BUILDING|ckages. requests sleep slyly. quickly even pinto beans promise above the slyly regular pinto beans. |
+12|Customer#000000012|9PWKuhzT4Zr1Q|13|23-791-276-1263|3396.49|HOUSEHOLD| to the carefully final braids. blithely regular requests nag. ironic theodolites boost quickly along|
+13|Customer#000000013|nsXQu0oVjD7PM659uC3SRSp|3|13-761-547-5974|3857.34|BUILDING|ounts sleep carefully after the close frays. carefully bold notornis use ironic requests. blithely|
+14|Customer#000000014|KXkletMlL2JQEA |1|11-845-129-3851|5266.30|FURNITURE|, ironic packages across the unus|
+15|Customer#000000015|YtWggXoOLdwdo7b0y,BZaGUQMLJMX1Y,EC,6Dn|23|33-687-542-7601|2788.52|HOUSEHOLD| platelets. regular deposits detect asymptotes. blithely unusual packages nag slyly at the fluf|
+16|Customer#000000016|cYiaeMLZSMAOQ2 d0W,|10|20-781-609-3107|4681.03|FURNITURE|kly silent courts. thinly regular theodolites sleep fluffily after |
+17|Customer#000000017|izrh 6jdqtp2eqdtbkswDD8SG4SzXruMfIXyR7|2|12-970-682-3487|6.34|AUTOMOBILE|packages wake! blithely even pint|
+18|Customer#000000018|3txGO AiuFux3zT0Z9NYaFRnZt|6|16-155-215-1315|5494.43|BUILDING|s sleep. carefully even instructions nag furiously alongside of t|
+19|Customer#000000019|uc,3bHIx84H,wdrmLOjVsiqXCq2tr|18|28-396-526-5053|8914.71|HOUSEHOLD| nag. furiously careful packages are slyly at the accounts. furiously regular in|
+20|Customer#000000020|JrPk8Pqplj4Ne|22|32-957-234-8742|7603.40|FURNITURE|g alongside of the special excuses-- fluffily enticing packages wake |
+21|Customer#000000021|XYmVpr9yAHDEn|8|18-902-614-8344|1428.25|MACHINERY| quickly final accounts integrate blithely furiously u|
+22|Customer#000000022|QI6p41,FNs5k7RZoCCVPUTkUdYpB|3|13-806-545-9701|591.98|MACHINERY|s nod furiously above the furiously ironic ideas. |
+23|Customer#000000023|OdY W13N7Be3OC5MpgfmcYss0Wn6TKT|3|13-312-472-8245|3332.02|HOUSEHOLD|deposits. special deposits cajole slyly. fluffily special deposits about the furiously |
+24|Customer#000000024|HXAFgIAyjxtdqwimt13Y3OZO 4xeLe7U8PqG|13|23-127-851-8031|9255.67|MACHINERY|into beans. fluffily final ideas haggle fluffily|
+25|Customer#000000025|Hp8GyFQgGHFYSilH5tBfe|12|22-603-468-3533|7133.70|FURNITURE|y. accounts sleep ruthlessly according to the regular theodolites. unusual instructions sleep. ironic, final|
+26|Customer#000000026|8ljrc5ZeMl7UciP|22|32-363-455-4837|5182.05|AUTOMOBILE|c requests use furiously ironic requests. slyly ironic dependencies us|
+27|Customer#000000027|IS8GIyxpBrLpMT0u7|3|13-137-193-2709|5679.84|BUILDING| about the carefully ironic pinto beans. accoun|
+28|Customer#000000028|iVyg0daQ,Tha8x2WPWA9m2529m|8|18-774-241-1462|1007.18|FURNITURE| along the regular deposits. furiously final pac|
+29|Customer#000000029|sJ5adtfyAkCK63df2,vF25zyQMVYE34uh|0|10-773-203-7342|7618.27|FURNITURE|its after the carefully final platelets x-ray against |
+30|Customer#000000030|nJDsELGAavU63Jl0c5NKsKfL8rIJQQkQnYL2QJY|1|11-764-165-5076|9321.01|BUILDING|lithely final requests. furiously unusual account|
+31|Customer#000000031|LUACbO0viaAv6eXOAebryDB xjVst|23|33-197-837-7094|5236.89|HOUSEHOLD|s use among the blithely pending depo|
+32|Customer#000000032|jD2xZzi UmId,DCtNBLXKj9q0Tlp2iQ6ZcO3J|15|25-430-914-2194|3471.53|BUILDING|cial ideas. final, furious requests across the e|
+33|Customer#000000033|qFSlMuLucBmx9xnn5ib2csWUweg D|17|27-375-391-1280|-78.56|AUTOMOBILE|s. slyly regular accounts are furiously. carefully pending requests|
+34|Customer#000000034|Q6G9wZ6dnczmtOx509xgE,M2KV|15|25-344-968-5422|8589.70|HOUSEHOLD|nder against the even, pending accounts. even|
+35|Customer#000000035|TEjWGE4nBzJL2|17|27-566-888-7431|1228.24|HOUSEHOLD|requests. special, express requests nag slyly furiousl|
+36|Customer#000000036|3TvCzjuPzpJ0,DdJ8kW5U|21|31-704-669-5769|4987.27|BUILDING|haggle. enticing, quiet platelets grow quickly bold sheaves. carefully regular acc|
+37|Customer#000000037|7EV4Pwh,3SboctTWt|8|18-385-235-7162|-917.75|FURNITURE|ilent packages are carefully among the deposits. furiousl|
+38|Customer#000000038|a5Ee5e9568R8RLP 2ap7|12|22-306-880-7212|6345.11|HOUSEHOLD|lar excuses. closely even asymptotes cajole blithely excuses. carefully silent pinto beans sleep carefully fin|
+39|Customer#000000039|nnbRg,Pvy33dfkorYE FdeZ60|2|12-387-467-6509|6264.31|AUTOMOBILE|tions. slyly silent excuses slee|
+40|Customer#000000040|gOnGWAyhSV1ofv|3|13-652-915-8939|1335.30|BUILDING|rges impress after the slyly ironic courts. foxes are. blithely |
+41|Customer#000000041|IM9mzmyoxeBmvNw8lA7G3Ydska2nkZF|10|20-917-711-4011|270.95|HOUSEHOLD|ly regular accounts hang bold, silent packages. unusual foxes haggle slyly above the special, final depo|
+42|Customer#000000042|ziSrvyyBke|5|15-416-330-4175|8727.01|BUILDING|ssly according to the pinto beans: carefully special requests across the even, pending accounts wake special|
+43|Customer#000000043|ouSbjHk8lh5fKX3zGso3ZSIj9Aa3PoaFd|19|29-316-665-2897|9904.28|MACHINERY|ial requests: carefully pending foxes detect quickly. carefully final courts cajole quickly. carefully|
+44|Customer#000000044|Oi,dOSPwDu4jo4x,,P85E0dmhZGvNtBwi|16|26-190-260-5375|7315.94|AUTOMOBILE|r requests around the unusual, bold a|
+45|Customer#000000045|4v3OcpFgoOmMG,CbnF,4mdC|9|19-715-298-9917|9983.38|AUTOMOBILE|nto beans haggle slyly alongside of t|
+46|Customer#000000046|eaTXWWm10L9|6|16-357-681-2007|5744.59|AUTOMOBILE|ctions. accounts sleep furiously even requests. regular, regular accounts cajole blithely around the final pa|
+47|Customer#000000047|b0UgocSqEW5 gdVbhNT|2|12-427-271-9466|274.58|BUILDING|ions. express, ironic instructions sleep furiously ironic ideas. furi|
+48|Customer#000000048|0UU iPhBupFvemNB|0|10-508-348-5882|3792.50|BUILDING|re fluffily pending foxes. pending, bold platelets sleep slyly. even platelets cajo|
+49|Customer#000000049|cNgAeX7Fqrdf7HQN9EwjUa4nxT,68L FKAxzl|10|20-908-631-4424|4573.94|FURNITURE|nusual foxes! fluffily pending packages maintain to the regular |
+50|Customer#000000050|9SzDYlkzxByyJ1QeTI o|6|16-658-112-3221|4266.13|MACHINERY|ts. furiously ironic accounts cajole furiously slyly ironic dinos.|
+51|Customer#000000051|uR,wEaiTvo4|12|22-344-885-4251|855.87|FURNITURE|eposits. furiously regular requests integrate carefully packages. furious|
+52|Customer#000000052|7 QOqGqqSy9jfV51BC71jcHJSD0|11|21-186-284-5998|5630.28|HOUSEHOLD|ic platelets use evenly even accounts. stealthy theodolites cajole furiou|
+53|Customer#000000053|HnaxHzTfFTZs8MuCpJyTbZ47Cm4wFOOgib|15|25-168-852-5363|4113.64|HOUSEHOLD|ar accounts are. even foxes are blithely. fluffily pending deposits boost|
+54|Customer#000000054|,k4vf 5vECGWFy,hosTE,|4|14-776-370-4745|868.90|AUTOMOBILE|sual, silent accounts. furiously express accounts cajole special deposits. final, final accounts use furi|
+55|Customer#000000055|zIRBR4KNEl HzaiV3a i9n6elrxzDEh8r8pDom|10|20-180-440-8525|4572.11|MACHINERY|ully unusual packages wake bravely bold packages. unusual requests boost deposits! blithely ironic packages ab|
+56|Customer#000000056|BJYZYJQk4yD5B|10|20-895-685-6920|6530.86|FURNITURE|. notornis wake carefully. carefully fluffy requests are furiously even accounts. slyly expre|
+57|Customer#000000057|97XYbsuOPRXPWU|21|31-835-306-1650|4151.93|AUTOMOBILE|ove the carefully special packages. even, unusual deposits sleep slyly pend|
+58|Customer#000000058|g9ap7Dk1Sv9fcXEWjpMYpBZIRUohi T|13|23-244-493-2508|6478.46|HOUSEHOLD|ideas. ironic ideas affix furiously express, final instructions. regular excuses use quickly e|
+59|Customer#000000059|zLOCP0wh92OtBihgspOGl4|1|11-355-584-3112|3458.60|MACHINERY|ously final packages haggle blithely after the express deposits. furiou|
+60|Customer#000000060|FyodhjwMChsZmUz7Jz0H|12|22-480-575-5866|2741.87|MACHINERY|latelets. blithely unusual courts boost furiously about the packages. blithely final instruct|
+61|Customer#000000061|9kndve4EAJxhg3veF BfXr7AqOsT39o gtqjaYE|17|27-626-559-8599|1536.24|FURNITURE|egular packages shall have to impress along the |
+62|Customer#000000062|upJK2Dnw13,|7|17-361-978-7059|595.61|MACHINERY|kly special dolphins. pinto beans are slyly. quickly regular accounts are furiously a|
+63|Customer#000000063|IXRSpVWWZraKII|21|31-952-552-9584|9331.13|AUTOMOBILE|ithely even accounts detect slyly above the fluffily ir|
+64|Customer#000000064|MbCeGY20kaKK3oalJD,OT|3|13-558-731-7204|-646.64|BUILDING|structions after the quietly ironic theodolites cajole be|
+65|Customer#000000065|RGT yzQ0y4l0H90P783LG4U95bXQFDRXbWa1sl,X|23|33-733-623-5267|8795.16|AUTOMOBILE|y final foxes serve carefully. theodolites are carefully. pending i|
+66|Customer#000000066|XbsEqXH1ETbJYYtA1A|22|32-213-373-5094|242.77|HOUSEHOLD|le slyly accounts. carefully silent packages benea|
+67|Customer#000000067|rfG0cOgtr5W8 xILkwp9fpCS8|9|19-403-114-4356|8166.59|MACHINERY|indle furiously final, even theodo|
+68|Customer#000000068|o8AibcCRkXvQFh8hF,7o|12|22-918-832-2411|6853.37|HOUSEHOLD| pending pinto beans impress realms. final dependencies |
+69|Customer#000000069|Ltx17nO9Wwhtdbe9QZVxNgP98V7xW97uvSH1prEw|9|19-225-978-5670|1709.28|HOUSEHOLD|thely final ideas around the quickly final dependencies affix carefully quickly final theodolites. final accounts c|
+70|Customer#000000070|mFowIuhnHjp2GjCiYYavkW kUwOjIaTCQ|22|32-828-107-2832|4867.52|FURNITURE|fter the special asymptotes. ideas after the unusual frets cajole quickly regular pinto be|
+71|Customer#000000071|TlGalgdXWBmMV,6agLyWYDyIz9MKzcY8gl,w6t1B|7|17-710-812-5403|-611.19|HOUSEHOLD|g courts across the regular, final pinto beans are blithely pending ac|
+72|Customer#000000072|putjlmskxE,zs,HqeIA9Wqu7dhgH5BVCwDwHHcf|2|12-759-144-9689|-362.86|FURNITURE|ithely final foxes sleep always quickly bold accounts. final wat|
+73|Customer#000000073|8IhIxreu4Ug6tt5mog4|0|10-473-439-3214|4288.50|BUILDING|usual, unusual packages sleep busily along the furiou|
+74|Customer#000000074|IkJHCA3ZThF7qL7VKcrU nRLl,kylf |4|14-199-862-7209|2764.43|MACHINERY|onic accounts. blithely slow packages would haggle carefully. qui|
+75|Customer#000000075|Dh 6jZ,cwxWLKQfRKkiGrzv6pm|18|28-247-803-9025|6684.10|AUTOMOBILE| instructions cajole even, even deposits. finally bold deposits use above the even pains. slyl|
+76|Customer#000000076|m3sbCvjMOHyaOofH,e UkGPtqc4|0|10-349-718-3044|5745.33|FURNITURE|pecial deposits. ironic ideas boost blithely according to the closely ironic theodolites! furiously final deposits n|
+77|Customer#000000077|4tAE5KdMFGD4byHtXF92vx|17|27-269-357-4674|1738.87|BUILDING|uffily silent requests. carefully ironic asymptotes among the ironic hockey players are carefully bli|
+78|Customer#000000078|HBOta,ZNqpg3U2cSL0kbrftkPwzX|9|19-960-700-9191|7136.97|FURNITURE|ests. blithely bold pinto beans h|
+79|Customer#000000079|n5hH2ftkVRwW8idtD,BmM2|15|25-147-850-4166|5121.28|MACHINERY|es. packages haggle furiously. regular, special requests poach after the quickly express ideas. blithely pending re|
+80|Customer#000000080|K,vtXp8qYB |0|10-267-172-7101|7383.53|FURNITURE|tect among the dependencies. bold accounts engage closely even pinto beans. ca|
+81|Customer#000000081|SH6lPA7JiiNC6dNTrR|20|30-165-277-3269|2023.71|BUILDING|r packages. fluffily ironic requests cajole fluffily. ironically regular theodolit|
+82|Customer#000000082|zhG3EZbap4c992Gj3bK,3Ne,Xn|18|28-159-442-5305|9468.34|AUTOMOBILE|s wake. bravely regular accounts are furiously. regula|
+83|Customer#000000083|HnhTNB5xpnSF20JBH4Ycs6psVnkC3RDf|22|32-817-154-4122|6463.51|BUILDING|ccording to the quickly bold warhorses. final, regular foxes integrate carefully. bold packages nag blithely ev|
+84|Customer#000000084|lpXz6Fwr9945rnbtMc8PlueilS1WmASr CB|11|21-546-818-3802|5174.71|FURNITURE|ly blithe foxes. special asymptotes haggle blithely against the furiously regular depo|
+85|Customer#000000085|siRerlDwiolhYR 8FgksoezycLj|5|15-745-585-8219|3386.64|FURNITURE|ronic ideas use above the slowly pendin|
+86|Customer#000000086|US6EGGHXbTTXPL9SBsxQJsuvy|0|10-677-951-2353|3306.32|HOUSEHOLD|quests. pending dugouts are carefully aroun|
+87|Customer#000000087|hgGhHVSWQl 6jZ6Ev|23|33-869-884-7053|6327.54|FURNITURE|hely ironic requests integrate according to the ironic accounts. slyly regular pla|
+88|Customer#000000088|wtkjBN9eyrFuENSMmMFlJ3e7jE5KXcg|16|26-516-273-2566|8031.44|AUTOMOBILE|s are quickly above the quickly ironic instructions; even requests about the carefully final deposi|
+89|Customer#000000089|dtR, y9JQWUO6FoJExyp8whOU|14|24-394-451-5404|1530.76|FURNITURE|counts are slyly beyond the slyly final accounts. quickly final ideas wake. r|
+90|Customer#000000090|QxCzH7VxxYUWwfL7|16|26-603-491-1238|7354.23|BUILDING|sly across the furiously even |
+91|Customer#000000091|S8OMYFrpHwoNHaGBeuS6E 6zhHGZiprw1b7 q|8|18-239-400-3677|4643.14|AUTOMOBILE|onic accounts. fluffily silent pinto beans boost blithely according to the fluffily exp|
+92|Customer#000000092|obP PULk2LH LqNF,K9hcbNqnLAkJVsl5xqSrY,|2|12-446-416-8471|1182.91|MACHINERY|. pinto beans hang slyly final deposits. ac|
+93|Customer#000000093|EHXBr2QGdh|7|17-359-388-5266|2182.52|MACHINERY|press deposits. carefully regular platelets r|
+94|Customer#000000094|IfVNIN9KtkScJ9dUjK3Pg5gY1aFeaXewwf|9|19-953-499-8833|5500.11|HOUSEHOLD|latelets across the bold, final requests sleep according to the fluffily bold accounts. unusual deposits amon|
+95|Customer#000000095|EU0xvmWvOmUUn5J,2z85DQyG7QCJ9Xq7|15|25-923-255-2929|5327.38|MACHINERY|ithely. ruthlessly final requests wake slyly alongside of the furiously silent pinto beans. even the|
+96|Customer#000000096|vWLOrmXhRR|8|18-422-845-1202|6323.92|AUTOMOBILE|press requests believe furiously. carefully final instructions snooze carefully. |
+97|Customer#000000097|OApyejbhJG,0Iw3j rd1M|17|27-588-919-5638|2164.48|AUTOMOBILE|haggle slyly. bold, special ideas are blithely above the thinly bold theo|
+98|Customer#000000098|7yiheXNSpuEAwbswDW|12|22-885-845-6889|-551.37|BUILDING|ages. furiously pending accounts are quickly carefully final foxes: busily pe|
+99|Customer#000000099|szsrOiPtCHVS97Lt|15|25-515-237-9232|4088.65|HOUSEHOLD|cajole slyly about the regular theodolites! furiously bold requests nag along the pending, regular packages. somas|
+100|Customer#000000100|fptUABXcmkC5Wx|20|30-749-445-4907|9889.89|FURNITURE|was furiously fluffily quiet deposits. silent, pending requests boost against |
+101|Customer#000000101|sMmL2rNeHDltovSm Y|2|12-514-298-3699|7470.96|MACHINERY| sleep. pending packages detect slyly ironic pack|
+102|Customer#000000102|UAtflJ06 fn9zBfKjInkQZlWtqaA|19|29-324-978-8538|8462.17|BUILDING|ously regular dependencies nag among the furiously express dinos. blithely final|
+103|Customer#000000103|8KIsQX4LJ7QMsj6DrtFtXu0nUEdV,8a|9|19-216-107-2107|2757.45|BUILDING|furiously pending notornis boost slyly around the blithely ironic ideas? final, even instructions cajole fl|
+104|Customer#000000104|9mcCK L7rt0SwiYtrbO88DiZS7U d7M|10|20-966-284-8065|-588.38|FURNITURE|rate carefully slyly special pla|
+105|Customer#000000105|4iSJe4L SPjg7kJj98Yz3z0B|10|20-793-553-6417|9091.82|MACHINERY|l pains cajole even accounts. quietly final instructi|
+106|Customer#000000106|xGCOEAUjUNG|1|11-751-989-4627|3288.42|MACHINERY|lose slyly. ironic accounts along the evenly regular theodolites wake about the special, final gifts. |
+107|Customer#000000107|Zwg64UZ,q7GRqo3zm7P1tZIRshBDz|15|25-336-529-9919|2514.15|AUTOMOBILE|counts cajole slyly. regular requests wake. furiously regular deposits about the blithely final fo|
+108|Customer#000000108|GPoeEvpKo1|5|15-908-619-7526|2259.38|BUILDING|refully ironic deposits sleep. regular, unusual requests wake slyly|
+109|Customer#000000109|OOOkYBgCMzgMQXUmkocoLb56rfrdWp2NE2c|16|26-992-422-8153|-716.10|BUILDING|es. fluffily final dependencies sleep along the blithely even pinto beans. final deposits haggle furiously furiou|
+110|Customer#000000110|mymPfgphaYXNYtk|10|20-893-536-2069|7462.99|AUTOMOBILE|nto beans cajole around the even, final deposits. quickly bold packages according to the furiously regular dept|
+111|Customer#000000111|CBSbPyOWRorloj2TBvrK9qp9tHBs|22|32-582-283-7528|6505.26|MACHINERY|ly unusual instructions detect fluffily special deposits-- theodolites nag carefully during the ironic dependencies|
+112|Customer#000000112|RcfgG3bO7QeCnfjqJT1|19|29-233-262-8382|2953.35|FURNITURE|rmanently unusual multipliers. blithely ruthless deposits are furiously along the|
+113|Customer#000000113|eaOl5UBXIvdY57rglaIzqvfPD,MYfK|12|22-302-930-4756|2912.00|BUILDING|usly regular theodolites boost furiously doggedly pending instructio|
+114|Customer#000000114|xAt 5f5AlFIU|14|24-805-212-7646|1027.46|FURNITURE|der the carefully express theodolites are after the packages. packages are. bli|
+115|Customer#000000115|0WFt1IXENmUT2BgbsB0ShVKJZt0HCBCbFl0aHc|8|18-971-699-1843|7508.92|HOUSEHOLD|sits haggle above the carefully ironic theodolite|
+116|Customer#000000116|yCuVxIgsZ3,qyK2rloThy3u|16|26-632-309-5792|8403.99|BUILDING|as. quickly final sauternes haggle slyly carefully even packages. brave, ironic pinto beans are above the furious|
+117|Customer#000000117|uNhM,PzsRA3S,5Y Ge5Npuhi|24|34-403-631-3505|3950.83|FURNITURE|affix. instructions are furiously sl|
+118|Customer#000000118|OVnFuHygK9wx3xpg8|18|28-639-943-7051|3582.37|AUTOMOBILE|uick packages alongside of the furiously final deposits haggle above the fluffily even foxes. blithely dogged dep|
+119|Customer#000000119|M1ETOIecuvH8DtM0Y0nryXfW|7|17-697-919-8406|3930.35|FURNITURE|express ideas. blithely ironic foxes thrash. special acco|
+120|Customer#000000120|zBNna00AEInqyO1|12|22-291-534-1571|363.75|MACHINERY| quickly. slyly ironic requests cajole blithely furiously final dependen|
+121|Customer#000000121|tv nCR2YKupGN73mQudO|17|27-411-990-2959|6428.32|BUILDING|uriously stealthy ideas. carefully final courts use carefully|
+122|Customer#000000122|yp5slqoNd26lAENZW3a67wSfXA6hTF|3|13-702-694-4520|7865.46|HOUSEHOLD| the special packages hinder blithely around the permanent requests. bold depos|
+123|Customer#000000123|YsOnaaER8MkvK5cpf4VSlq|5|15-817-151-1168|5897.83|BUILDING|ependencies. regular, ironic requests are fluffily regu|
+124|Customer#000000124|aTbyVAW5tCd,v09O|18|28-183-750-7809|1842.49|AUTOMOBILE|le fluffily even dependencies. quietly s|
+125|Customer#000000125|,wSZXdVR xxIIfm9s8ITyLl3kgjT6UC07GY0Y|19|29-261-996-3120|-234.12|FURNITURE|x-ray finally after the packages? regular requests c|
+126|Customer#000000126|ha4EHmbx3kg DYCsP6DFeUOmavtQlHhcfaqr|22|32-755-914-7592|1001.39|HOUSEHOLD|s about the even instructions boost carefully furiously ironic pearls. ruthless, |
+127|Customer#000000127|Xyge4DX2rXKxXyye1Z47LeLVEYMLf4Bfcj|21|31-101-672-2951|9280.71|MACHINERY|ic, unusual theodolites nod silently after the final, ironic instructions: pending r|
+128|Customer#000000128|AmKUMlJf2NRHcKGmKjLS|4|14-280-874-8044|-986.96|HOUSEHOLD|ing packages integrate across the slyly unusual dugouts. blithely silent ideas sublate carefully. blithely expr|
+129|Customer#000000129|q7m7rbMM0BpaCdmxloCgBDRCleXsXkdD8kf|7|17-415-148-7416|9127.27|HOUSEHOLD| unusual deposits boost carefully furiously silent ideas. pending accounts cajole slyly across|
+130|Customer#000000130|RKPx2OfZy0Vn 8wGWZ7F2EAvmMORl1k8iH|9|19-190-993-9281|5073.58|HOUSEHOLD|ix slowly. express packages along the furiously ironic requests integrate daringly deposits. fur|
+131|Customer#000000131|jyN6lAjb1FtH10rMC,XzlWyCBrg75|11|21-840-210-3572|8595.53|HOUSEHOLD|jole special packages. furiously final dependencies about the furiously speci|
+132|Customer#000000132|QM5YabAsTLp9|4|14-692-150-9717|162.57|HOUSEHOLD|uickly carefully special theodolites. carefully regular requests against the blithely unusual instructions |
+133|Customer#000000133|IMCuXdpIvdkYO92kgDGuyHgojcUs88p|17|27-408-997-8430|2314.67|AUTOMOBILE|t packages. express pinto beans are blithely along the unusual, even theodolites. silent packages use fu|
+134|Customer#000000134|sUiZ78QCkTQPICKpA9OBzkUp2FM|11|21-200-159-5932|4608.90|BUILDING|yly fluffy foxes boost final ideas. b|
+135|Customer#000000135|oZK,oC0 fdEpqUML|19|29-399-293-6241|8732.91|FURNITURE| the slyly final accounts. deposits cajole carefully. carefully sly packag|
+136|Customer#000000136|QoLsJ0v5C1IQbh,DS1|7|17-501-210-4726|-842.39|FURNITURE|ackages sleep ironic, final courts. even requests above the blithely bold requests g|
+137|Customer#000000137|cdW91p92rlAEHgJafqYyxf1Q|16|26-777-409-5654|7838.30|HOUSEHOLD|carefully regular theodolites use. silent dolphins cajo|
+138|Customer#000000138|5uyLAeY7HIGZqtu66Yn08f|5|15-394-860-4589|430.59|MACHINERY|ts doze on the busy ideas. regular|
+139|Customer#000000139|3ElvBwudHKL02732YexGVFVt |9|19-140-352-1403|7897.78|MACHINERY|nstructions. quickly ironic ideas are carefully. bold, |
+140|Customer#000000140|XRqEPiKgcETII,iOLDZp5jA|4|14-273-885-6505|9963.15|MACHINERY|ies detect slyly ironic accounts. slyly ironic theodolites hag|
+141|Customer#000000141|5IW,WROVnikc3l7DwiUDGQNGsLBGOL6Dc0|1|11-936-295-6204|6706.14|FURNITURE|packages nag furiously. carefully unusual accounts snooze according to the fluffily regular pinto beans. slyly spec|
+142|Customer#000000142|AnJ5lxtLjioClr2khl9pb8NLxG2,|9|19-407-425-2584|2209.81|AUTOMOBILE|. even, express theodolites upo|
+143|Customer#000000143|681r22uL452zqk 8By7I9o9enQfx0|16|26-314-406-7725|2186.50|MACHINERY|across the blithely unusual requests haggle theodo|
+144|Customer#000000144|VxYZ3ebhgbltnetaGjNC8qCccjYU05 fePLOno8y|1|11-717-379-4478|6417.31|MACHINERY|ges. slyly regular accounts are slyly. bold, idle reque|
+145|Customer#000000145|kQjHmt2kcec cy3hfMh969u|13|23-562-444-8454|9748.93|HOUSEHOLD|ests? express, express instructions use. blithely fina|
+146|Customer#000000146|GdxkdXG9u7iyI1,,y5tq4ZyrcEy|3|13-835-723-3223|3328.68|FURNITURE|ffily regular dinos are slyly unusual requests. slyly specia|
+147|Customer#000000147|6VvIwbVdmcsMzuu,C84GtBWPaipGfi7DV|18|28-803-187-4335|8071.40|AUTOMOBILE|ress packages above the blithely regular packages sleep fluffily blithely ironic accounts. |
+148|Customer#000000148|BhSPlEWGvIJyT9swk vCWE|11|21-562-498-6636|2135.60|HOUSEHOLD|ing to the carefully ironic requests. carefully regular dependencies about the theodolites wake furious|
+149|Customer#000000149|3byTHCp2mNLPigUrrq|19|29-797-439-6760|8959.65|AUTOMOBILE|al instructions haggle against the slyly bold w|
+150|Customer#000000150|zeoGShTjCwGPplOWFkLURrh41O0AZ8dwNEEN4 |18|28-328-564-7630|3849.48|MACHINERY|ole blithely among the furiously pending packages. furiously bold ideas wake fluffily ironic idea|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/lineitem.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/lineitem.tbl
new file mode 100644
index 0000000..58d47c6
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/lineitem.tbl
@@ -0,0 +1,6005 @@
+1|156|4|1|17|17954.55|0.04|0.02|N|O|1996-03-13|1996-02-12|1996-03-22|DELIVER IN PERSON|TRUCK|egular courts above the|
+1|68|9|2|36|34850.16|0.09|0.06|N|O|1996-04-12|1996-02-28|1996-04-20|TAKE BACK RETURN|MAIL|ly final dependencies: slyly bold |
+1|64|5|3|8|7712.48|0.10|0.02|N|O|1996-01-29|1996-03-05|1996-01-31|TAKE BACK RETURN|REG AIR|riously. regular, express dep|
+1|3|6|4|28|25284.00|0.09|0.06|N|O|1996-04-21|1996-03-30|1996-05-16|NONE|AIR|lites. fluffily even de|
+1|25|8|5|24|22200.48|0.10|0.04|N|O|1996-03-30|1996-03-14|1996-04-01|NONE|FOB| pending foxes. slyly re|
+1|16|3|6|32|29312.32|0.07|0.02|N|O|1996-01-30|1996-02-07|1996-02-03|DELIVER IN PERSON|MAIL|arefully slyly ex|
+2|107|2|1|38|38269.80|0.00|0.05|N|O|1997-01-28|1997-01-14|1997-02-02|TAKE BACK RETURN|RAIL|ven requests. deposits breach a|
+3|5|2|1|45|40725.00|0.06|0.00|R|F|1994-02-02|1994-01-04|1994-02-23|NONE|AIR|ongside of the furiously brave acco|
+3|20|10|2|49|45080.98|0.10|0.00|R|F|1993-11-09|1993-12-20|1993-11-24|TAKE BACK RETURN|RAIL| unusual accounts. eve|
+3|129|8|3|27|27786.24|0.06|0.07|A|F|1994-01-16|1993-11-22|1994-01-23|DELIVER IN PERSON|SHIP|nal foxes wake. |
+3|30|5|4|2|1860.06|0.01|0.06|A|F|1993-12-04|1994-01-07|1994-01-01|NONE|TRUCK|y. fluffily pending d|
+3|184|5|5|28|30357.04|0.04|0.00|R|F|1993-12-14|1994-01-10|1994-01-01|TAKE BACK RETURN|FOB|ages nag slyly pending|
+3|63|8|6|26|25039.56|0.10|0.02|A|F|1993-10-29|1993-12-18|1993-11-04|TAKE BACK RETURN|RAIL|ges sleep after the caref|
+4|89|10|1|30|29672.40|0.03|0.08|N|O|1996-01-10|1995-12-14|1996-01-18|DELIVER IN PERSON|REG AIR|- quickly regular packages sleep. idly|
+5|109|10|1|15|15136.50|0.02|0.04|R|F|1994-10-31|1994-08-31|1994-11-20|NONE|AIR|ts wake furiously |
+5|124|5|2|26|26627.12|0.07|0.08|R|F|1994-10-16|1994-09-25|1994-10-19|NONE|FOB|sts use slyly quickly special instruc|
+5|38|4|3|50|46901.50|0.08|0.03|A|F|1994-08-08|1994-10-13|1994-08-26|DELIVER IN PERSON|AIR|eodolites. fluffily unusual|
+6|140|6|1|37|38485.18|0.08|0.03|A|F|1992-04-27|1992-05-15|1992-05-02|TAKE BACK RETURN|TRUCK|p furiously special foxes|
+7|183|4|1|12|12998.16|0.07|0.03|N|O|1996-05-07|1996-03-13|1996-06-03|TAKE BACK RETURN|FOB|ss pinto beans wake against th|
+7|146|3|2|9|9415.26|0.08|0.08|N|O|1996-02-01|1996-03-02|1996-02-19|TAKE BACK RETURN|SHIP|es. instructions|
+7|95|8|3|46|45774.14|0.10|0.07|N|O|1996-01-15|1996-03-27|1996-02-03|COLLECT COD|MAIL| unusual reques|
+7|164|5|4|28|29796.48|0.03|0.04|N|O|1996-03-21|1996-04-08|1996-04-20|NONE|FOB|. slyly special requests haggl|
+7|152|4|5|38|39981.70|0.08|0.01|N|O|1996-02-11|1996-02-24|1996-02-18|DELIVER IN PERSON|TRUCK|ns haggle carefully ironic deposits. bl|
+7|80|10|6|35|34302.80|0.06|0.03|N|O|1996-01-16|1996-02-23|1996-01-22|TAKE BACK RETURN|FOB|jole. excuses wake carefully alongside of |
+7|158|3|7|5|5290.75|0.04|0.02|N|O|1996-02-10|1996-03-26|1996-02-13|NONE|FOB|ithely regula|
+32|83|4|1|28|27526.24|0.05|0.08|N|O|1995-10-23|1995-08-27|1995-10-26|TAKE BACK RETURN|TRUCK|sleep quickly. req|
+32|198|10|2|32|35142.08|0.02|0.00|N|O|1995-08-14|1995-10-07|1995-08-27|COLLECT COD|AIR|lithely regular deposits. fluffily |
+32|45|2|3|2|1890.08|0.09|0.02|N|O|1995-08-07|1995-10-07|1995-08-23|DELIVER IN PERSON|AIR| express accounts wake according to the|
+32|3|8|4|4|3612.00|0.09|0.03|N|O|1995-08-04|1995-10-01|1995-09-03|NONE|REG AIR|e slyly final pac|
+32|86|7|5|44|43387.52|0.05|0.06|N|O|1995-08-28|1995-08-20|1995-09-14|DELIVER IN PERSON|AIR|symptotes nag according to the ironic depo|
+32|12|6|6|6|5472.06|0.04|0.03|N|O|1995-07-21|1995-09-23|1995-07-25|COLLECT COD|RAIL| gifts cajole carefully.|
+33|62|7|1|31|29823.86|0.09|0.04|A|F|1993-10-29|1993-12-19|1993-11-08|COLLECT COD|TRUCK|ng to the furiously ironic package|
+33|61|8|2|32|30753.92|0.02|0.05|A|F|1993-12-09|1994-01-04|1993-12-28|COLLECT COD|MAIL|gular theodolites|
+33|138|4|3|5|5190.65|0.05|0.03|A|F|1993-12-09|1993-12-25|1993-12-23|TAKE BACK RETURN|AIR|. stealthily bold exc|
+33|34|5|4|41|38295.23|0.09|0.00|R|F|1993-11-09|1994-01-24|1993-11-11|TAKE BACK RETURN|MAIL|unusual packages doubt caref|
+34|89|10|1|13|12858.04|0.00|0.07|N|O|1998-10-23|1998-09-14|1998-11-06|NONE|REG AIR|nic accounts. deposits are alon|
+34|90|1|2|22|21781.98|0.08|0.06|N|O|1998-10-09|1998-10-16|1998-10-12|NONE|FOB|thely slyly p|
+34|170|7|3|6|6421.02|0.02|0.06|N|O|1998-10-30|1998-09-20|1998-11-05|NONE|FOB|ar foxes sleep |
+35|1|4|1|24|21624.00|0.02|0.00|N|O|1996-02-21|1996-01-03|1996-03-18|TAKE BACK RETURN|FOB|, regular tithe|
+35|162|1|2|34|36113.44|0.06|0.08|N|O|1996-01-22|1996-01-06|1996-01-27|DELIVER IN PERSON|RAIL|s are carefully against the f|
+35|121|4|3|7|7147.84|0.06|0.04|N|O|1996-01-19|1995-12-22|1996-01-29|NONE|MAIL| the carefully regular |
+35|86|7|4|25|24652.00|0.06|0.05|N|O|1995-11-26|1995-12-25|1995-12-21|DELIVER IN PERSON|SHIP| quickly unti|
+35|120|7|5|34|34684.08|0.08|0.06|N|O|1995-11-08|1996-01-15|1995-11-26|COLLECT COD|MAIL|. silent, unusual deposits boost|
+35|31|7|6|28|26068.84|0.03|0.02|N|O|1996-02-01|1995-12-24|1996-02-28|COLLECT COD|RAIL|ly alongside of |
+36|120|1|1|42|42845.04|0.09|0.00|N|O|1996-02-03|1996-01-21|1996-02-23|COLLECT COD|SHIP| careful courts. special |
+37|23|8|1|40|36920.80|0.09|0.03|A|F|1992-07-21|1992-08-01|1992-08-15|NONE|REG AIR|luffily regular requests. slyly final acco|
+37|127|6|2|39|40057.68|0.05|0.02|A|F|1992-07-02|1992-08-18|1992-07-28|TAKE BACK RETURN|RAIL|the final requests. ca|
+37|13|7|3|43|39259.43|0.05|0.08|A|F|1992-07-10|1992-07-06|1992-08-02|DELIVER IN PERSON|TRUCK|iously ste|
+38|176|5|1|44|47351.48|0.04|0.02|N|O|1996-09-29|1996-11-17|1996-09-30|COLLECT COD|MAIL|s. blithely unusual theodolites am|
+39|3|10|1|44|39732.00|0.09|0.06|N|O|1996-11-14|1996-12-15|1996-12-12|COLLECT COD|RAIL|eodolites. careful|
+39|187|8|2|26|28266.68|0.08|0.04|N|O|1996-11-04|1996-10-20|1996-11-20|NONE|FOB|ckages across the slyly silent|
+39|68|3|3|46|44530.76|0.06|0.08|N|O|1996-09-26|1996-12-19|1996-10-26|DELIVER IN PERSON|AIR|he carefully e|
+39|21|6|4|32|29472.64|0.07|0.05|N|O|1996-10-02|1996-12-19|1996-10-14|COLLECT COD|MAIL|heodolites sleep silently pending foxes. ac|
+39|55|10|5|43|41067.15|0.01|0.01|N|O|1996-10-17|1996-11-14|1996-10-26|COLLECT COD|MAIL|yly regular i|
+39|95|7|6|40|39803.60|0.06|0.05|N|O|1996-12-08|1996-10-22|1997-01-01|COLLECT COD|AIR|quickly ironic fox|
+64|86|7|1|21|20707.68|0.05|0.02|R|F|1994-09-30|1994-09-18|1994-10-26|DELIVER IN PERSON|REG AIR|ch slyly final, thin platelets.|
+65|60|5|1|26|24961.56|0.03|0.03|A|F|1995-04-20|1995-04-25|1995-05-13|NONE|TRUCK|pending deposits nag even packages. ca|
+65|74|3|2|22|21429.54|0.00|0.05|N|O|1995-07-17|1995-06-04|1995-07-19|COLLECT COD|FOB| ideas. special, r|
+65|2|5|3|21|18942.00|0.09|0.07|N|O|1995-07-06|1995-05-14|1995-07-31|DELIVER IN PERSON|RAIL|bove the even packages. accounts nag carefu|
+66|116|10|1|31|31499.41|0.00|0.08|R|F|1994-02-19|1994-03-11|1994-02-20|TAKE BACK RETURN|RAIL|ut the unusual accounts sleep at the bo|
+66|174|5|2|41|44040.97|0.04|0.07|A|F|1994-02-21|1994-03-01|1994-03-18|COLLECT COD|AIR| regular de|
+67|22|5|1|4|3688.08|0.09|0.04|N|O|1997-04-17|1997-01-31|1997-04-20|NONE|SHIP| cajole thinly expres|
+67|21|10|2|12|11052.24|0.09|0.05|N|O|1997-01-27|1997-02-21|1997-02-22|NONE|REG AIR| even packages cajole|
+67|174|4|3|5|5370.85|0.03|0.07|N|O|1997-02-20|1997-02-12|1997-02-21|DELIVER IN PERSON|TRUCK|y unusual packages thrash pinto |
+67|88|9|4|44|43475.52|0.08|0.06|N|O|1997-03-18|1997-01-29|1997-04-13|DELIVER IN PERSON|RAIL|se quickly above the even, express reques|
+67|41|10|5|23|21643.92|0.05|0.07|N|O|1997-04-19|1997-02-14|1997-05-06|DELIVER IN PERSON|REG AIR|ly regular deposit|
+67|179|9|6|29|31295.93|0.02|0.05|N|O|1997-01-25|1997-01-27|1997-01-27|DELIVER IN PERSON|FOB|ultipliers |
+68|8|1|1|3|2724.00|0.05|0.02|N|O|1998-07-04|1998-06-05|1998-07-21|NONE|RAIL|fully special instructions cajole. furious|
+68|176|4|2|46|49503.82|0.02|0.05|N|O|1998-06-26|1998-06-07|1998-07-05|NONE|MAIL| requests are unusual, regular pinto |
+68|35|1|3|46|43011.38|0.04|0.05|N|O|1998-08-13|1998-07-08|1998-08-29|NONE|RAIL|egular dependencies affix ironically along |
+68|95|9|4|20|19901.80|0.07|0.01|N|O|1998-06-27|1998-05-23|1998-07-02|NONE|REG AIR| excuses integrate fluffily |
+68|83|4|5|27|26543.16|0.03|0.06|N|O|1998-06-19|1998-06-25|1998-06-29|DELIVER IN PERSON|SHIP|ccounts. deposits use. furiously|
+68|103|6|6|30|30093.00|0.05|0.06|N|O|1998-08-11|1998-07-11|1998-08-14|NONE|RAIL|oxes are slyly blithely fin|
+68|140|6|7|41|42645.74|0.09|0.08|N|O|1998-06-24|1998-06-27|1998-07-06|NONE|SHIP|eposits nag special ideas. furiousl|
+69|116|10|1|48|48773.28|0.01|0.07|A|F|1994-08-17|1994-08-11|1994-09-08|NONE|TRUCK|regular epitaphs. carefully even ideas hag|
+69|105|10|2|32|32163.20|0.08|0.06|A|F|1994-08-24|1994-08-17|1994-08-31|NONE|REG AIR|s sleep carefully bold, |
+69|138|4|3|17|17648.21|0.09|0.00|A|F|1994-07-02|1994-07-07|1994-07-03|TAKE BACK RETURN|AIR|final, pending instr|
+69|38|9|4|3|2814.09|0.09|0.04|R|F|1994-06-06|1994-07-27|1994-06-15|NONE|MAIL| blithely final d|
+69|93|6|5|42|41709.78|0.07|0.04|R|F|1994-07-31|1994-07-26|1994-08-28|DELIVER IN PERSON|REG AIR|tect regular, speci|
+69|19|3|6|23|21137.23|0.05|0.00|A|F|1994-10-03|1994-08-06|1994-10-24|NONE|SHIP|nding accounts ca|
+70|65|2|1|8|7720.48|0.03|0.08|R|F|1994-01-12|1994-02-27|1994-01-14|TAKE BACK RETURN|FOB|ggle. carefully pending dependenc|
+70|197|10|2|13|14263.47|0.06|0.06|A|F|1994-03-03|1994-02-13|1994-03-26|COLLECT COD|AIR|lyly special packag|
+70|180|8|3|1|1080.18|0.03|0.05|R|F|1994-01-26|1994-03-05|1994-01-28|TAKE BACK RETURN|RAIL|quickly. fluffily unusual theodolites c|
+70|46|9|4|11|10406.44|0.01|0.05|A|F|1994-03-17|1994-03-17|1994-03-27|NONE|MAIL|alongside of the deposits. fur|
+70|38|9|5|37|34707.11|0.09|0.04|R|F|1994-02-13|1994-03-16|1994-02-21|COLLECT COD|MAIL|n accounts are. q|
+70|56|8|6|19|18164.95|0.06|0.03|A|F|1994-01-26|1994-02-17|1994-02-06|TAKE BACK RETURN|SHIP| packages wake pending accounts.|
+71|62|3|1|25|24051.50|0.09|0.07|N|O|1998-04-10|1998-04-22|1998-04-11|COLLECT COD|FOB|ckly. slyly|
+71|66|1|2|3|2898.18|0.09|0.07|N|O|1998-05-23|1998-04-03|1998-06-02|COLLECT COD|SHIP|y. pinto beans haggle after the|
+71|35|1|3|45|42076.35|0.00|0.07|N|O|1998-02-23|1998-03-20|1998-03-24|DELIVER IN PERSON|SHIP| ironic packages believe blithely a|
+71|97|9|4|33|32903.97|0.00|0.01|N|O|1998-04-12|1998-03-20|1998-04-15|NONE|FOB| serve quickly fluffily bold deposi|
+71|104|7|5|39|39159.90|0.08|0.06|N|O|1998-01-29|1998-04-07|1998-02-18|DELIVER IN PERSON|RAIL|l accounts sleep across the pack|
+71|196|9|6|34|37270.46|0.04|0.01|N|O|1998-03-05|1998-04-22|1998-03-30|DELIVER IN PERSON|TRUCK|s cajole. |
+96|124|7|1|23|23554.76|0.10|0.06|A|F|1994-07-19|1994-06-29|1994-07-25|DELIVER IN PERSON|TRUCK|ep-- carefully reg|
+96|136|7|2|30|31083.90|0.01|0.06|R|F|1994-06-03|1994-05-29|1994-06-22|DELIVER IN PERSON|TRUCK|e quickly even ideas. furiou|
+97|120|4|1|13|13261.56|0.00|0.02|R|F|1993-04-01|1993-04-04|1993-04-08|NONE|TRUCK|ayers cajole against the furiously|
+97|50|7|2|37|35151.85|0.02|0.06|A|F|1993-04-13|1993-03-30|1993-04-14|DELIVER IN PERSON|SHIP|ic requests boost carefully quic|
+97|78|6|3|19|18583.33|0.06|0.08|R|F|1993-05-14|1993-03-05|1993-05-25|TAKE BACK RETURN|RAIL|gifts. furiously ironic packages cajole. |
+98|41|2|1|28|26349.12|0.06|0.07|A|F|1994-12-24|1994-10-25|1995-01-16|COLLECT COD|REG AIR| pending, regular accounts s|
+98|110|7|2|1|1010.11|0.00|0.00|A|F|1994-12-01|1994-12-12|1994-12-15|DELIVER IN PERSON|TRUCK|. unusual instructions against|
+98|45|6|3|14|13230.56|0.05|0.02|A|F|1994-12-30|1994-11-22|1995-01-27|COLLECT COD|AIR| cajole furiously. blithely ironic ideas |
+98|168|9|4|10|10681.60|0.03|0.03|A|F|1994-10-23|1994-11-08|1994-11-09|COLLECT COD|RAIL| carefully. quickly ironic ideas|
+99|88|9|1|10|9880.80|0.02|0.01|A|F|1994-05-18|1994-06-03|1994-05-23|COLLECT COD|RAIL|kages. requ|
+99|124|5|2|5|5120.60|0.02|0.07|R|F|1994-05-06|1994-05-28|1994-05-20|TAKE BACK RETURN|RAIL|ests cajole fluffily waters. blithe|
+99|135|1|3|42|43475.46|0.02|0.02|A|F|1994-04-19|1994-05-18|1994-04-20|NONE|RAIL|kages are fluffily furiously ir|
+99|109|2|4|36|36327.60|0.09|0.02|A|F|1994-07-04|1994-04-17|1994-07-30|DELIVER IN PERSON|AIR|slyly. slyly e|
+100|63|4|1|28|26965.68|0.04|0.05|N|O|1998-05-08|1998-05-13|1998-06-07|COLLECT COD|TRUCK|sts haggle. slowl|
+100|116|10|2|22|22354.42|0.00|0.07|N|O|1998-06-24|1998-04-12|1998-06-29|DELIVER IN PERSON|SHIP|nto beans alongside of the fi|
+100|47|4|3|46|43563.84|0.03|0.04|N|O|1998-05-02|1998-04-10|1998-05-22|TAKE BACK RETURN|SHIP|ular accounts. even|
+100|39|10|4|14|13146.42|0.06|0.03|N|O|1998-05-22|1998-05-01|1998-06-03|COLLECT COD|MAIL|y. furiously ironic ideas gr|
+100|54|6|5|37|35299.85|0.05|0.00|N|O|1998-03-06|1998-04-16|1998-03-31|TAKE BACK RETURN|TRUCK|nd the quickly s|
+101|119|9|1|49|49936.39|0.10|0.00|N|O|1996-06-21|1996-05-27|1996-06-29|DELIVER IN PERSON|REG AIR|ts-- final packages sleep furiousl|
+101|164|9|2|36|38309.76|0.00|0.01|N|O|1996-05-19|1996-05-01|1996-06-04|DELIVER IN PERSON|AIR|tes. blithely pending dolphins x-ray f|
+101|139|5|3|12|12469.56|0.06|0.02|N|O|1996-03-29|1996-04-20|1996-04-12|COLLECT COD|MAIL|. quickly regular|
+102|89|10|1|37|36595.96|0.06|0.00|N|O|1997-07-24|1997-08-02|1997-08-07|TAKE BACK RETURN|SHIP|ully across the ideas. final deposit|
+102|170|5|2|34|36385.78|0.03|0.08|N|O|1997-08-09|1997-07-28|1997-08-26|TAKE BACK RETURN|SHIP|eposits cajole across|
+102|183|4|3|25|27079.50|0.01|0.01|N|O|1997-07-31|1997-07-24|1997-08-17|NONE|RAIL|bits. ironic accoun|
+102|62|7|4|15|14430.90|0.07|0.07|N|O|1997-06-02|1997-07-13|1997-06-04|DELIVER IN PERSON|SHIP|final packages. carefully even excu|
+103|195|9|1|6|6571.14|0.03|0.05|N|O|1996-10-11|1996-07-25|1996-10-28|NONE|FOB|cajole. carefully ex|
+103|11|5|2|37|33707.37|0.02|0.07|N|O|1996-09-17|1996-07-27|1996-09-20|TAKE BACK RETURN|MAIL|ies. quickly ironic requests use blithely|
+103|29|10|3|23|21367.46|0.01|0.04|N|O|1996-09-11|1996-09-18|1996-09-26|NONE|FOB|ironic accou|
+103|30|9|4|32|29760.96|0.01|0.07|N|O|1996-07-30|1996-08-06|1996-08-04|NONE|RAIL|kages doze. special, regular deposit|
+128|107|10|1|38|38269.80|0.06|0.01|A|F|1992-09-01|1992-08-27|1992-10-01|TAKE BACK RETURN|FOB| cajole careful|
+129|3|6|1|46|41538.00|0.08|0.02|R|F|1993-02-15|1993-01-24|1993-03-05|COLLECT COD|TRUCK|uietly bold theodolites. fluffil|
+129|186|7|2|36|39102.48|0.01|0.02|A|F|1992-11-25|1992-12-25|1992-12-09|TAKE BACK RETURN|REG AIR|packages are care|
+129|40|6|3|33|31021.32|0.04|0.06|A|F|1993-01-08|1993-02-14|1993-01-29|COLLECT COD|SHIP|sts nag bravely. fluffily|
+129|136|7|4|34|35228.42|0.00|0.01|R|F|1993-01-29|1993-02-14|1993-02-10|COLLECT COD|MAIL|quests. express ideas|
+129|32|8|5|24|22368.72|0.06|0.00|A|F|1992-12-07|1993-01-02|1992-12-11|TAKE BACK RETURN|FOB|uests. foxes cajole slyly after the ca|
+129|78|6|6|22|21517.54|0.06|0.01|R|F|1993-02-15|1993-01-31|1993-02-24|COLLECT COD|SHIP|e. fluffily regular |
+129|169|6|7|1|1069.16|0.05|0.04|R|F|1993-01-26|1993-01-08|1993-02-24|DELIVER IN PERSON|FOB|e carefully blithely bold dolp|
+130|129|10|1|14|14407.68|0.08|0.05|A|F|1992-08-15|1992-07-25|1992-09-13|COLLECT COD|RAIL| requests. final instruction|
+130|2|5|2|48|43296.00|0.03|0.02|R|F|1992-07-01|1992-07-12|1992-07-24|NONE|AIR|lithely alongside of the regu|
+130|12|3|3|18|16416.18|0.04|0.08|A|F|1992-07-04|1992-06-14|1992-07-29|DELIVER IN PERSON|MAIL| slyly ironic decoys abou|
+130|116|6|4|13|13209.43|0.09|0.02|R|F|1992-06-26|1992-07-29|1992-07-05|NONE|FOB| pending dolphins sleep furious|
+130|70|7|5|31|30072.17|0.06|0.05|R|F|1992-09-01|1992-07-18|1992-09-02|TAKE BACK RETURN|RAIL|thily about the ruth|
+131|168|7|1|45|48067.20|0.10|0.02|R|F|1994-09-14|1994-09-02|1994-10-04|NONE|FOB|ironic, bold accounts. careful|
+131|45|8|2|50|47252.00|0.02|0.04|A|F|1994-09-17|1994-08-10|1994-09-21|NONE|SHIP|ending requests. final, ironic pearls slee|
+131|190|1|3|4|4360.76|0.04|0.03|A|F|1994-09-20|1994-08-30|1994-09-23|COLLECT COD|REG AIR| are carefully slyly i|
+132|141|8|1|18|18740.52|0.00|0.08|R|F|1993-07-10|1993-08-05|1993-07-13|NONE|TRUCK|ges. platelets wake furio|
+132|120|1|2|43|43865.16|0.01|0.08|R|F|1993-09-01|1993-08-16|1993-09-22|NONE|TRUCK|y pending theodolites|
+132|115|6|3|32|32483.52|0.04|0.04|A|F|1993-07-12|1993-08-05|1993-08-05|COLLECT COD|TRUCK|d instructions hagg|
+132|29|2|4|23|21367.46|0.10|0.00|A|F|1993-06-16|1993-08-27|1993-06-23|DELIVER IN PERSON|AIR|refully blithely bold acco|
+133|104|7|1|27|27110.70|0.00|0.02|N|O|1997-12-21|1998-02-23|1997-12-27|TAKE BACK RETURN|MAIL|yly even gifts after the sl|
+133|177|5|2|12|12926.04|0.02|0.06|N|O|1997-12-02|1998-01-15|1997-12-29|DELIVER IN PERSON|REG AIR|ts cajole fluffily quickly i|
+133|118|8|3|29|29525.19|0.09|0.08|N|O|1998-02-28|1998-01-30|1998-03-09|DELIVER IN PERSON|RAIL| the carefully regular theodoli|
+133|90|1|4|11|10890.99|0.06|0.01|N|O|1998-03-21|1998-01-15|1998-04-04|DELIVER IN PERSON|REG AIR|e quickly across the dolphins|
+134|1|2|1|21|18921.00|0.00|0.03|A|F|1992-07-17|1992-07-08|1992-07-26|COLLECT COD|SHIP|s. quickly regular|
+134|165|2|2|35|37280.60|0.06|0.07|A|F|1992-08-23|1992-06-01|1992-08-24|NONE|MAIL|ajole furiously. instructio|
+134|189|10|3|26|28318.68|0.09|0.06|A|F|1992-06-20|1992-07-12|1992-07-16|NONE|RAIL| among the pending depos|
+134|145|6|4|47|49121.58|0.05|0.00|A|F|1992-08-16|1992-07-06|1992-08-28|NONE|REG AIR|s! carefully unusual requests boost careful|
+134|36|7|5|12|11232.36|0.05|0.02|A|F|1992-07-03|1992-06-01|1992-07-11|COLLECT COD|TRUCK|nts are quic|
+134|134|10|6|12|12409.56|0.00|0.00|A|F|1992-08-08|1992-07-07|1992-08-20|TAKE BACK RETURN|FOB|lyly regular pac|
+135|109|10|1|47|47427.70|0.06|0.08|N|O|1996-02-18|1996-01-01|1996-02-25|COLLECT COD|RAIL|ctions wake slyly abo|
+135|199|3|2|21|23082.99|0.00|0.07|N|O|1996-02-11|1996-01-12|1996-02-13|DELIVER IN PERSON|SHIP| deposits believe. furiously regular p|
+135|158|10|3|33|34918.95|0.02|0.00|N|O|1996-01-03|1995-11-21|1996-02-01|TAKE BACK RETURN|MAIL|ptotes boost slowly care|
+135|68|7|4|34|32914.04|0.02|0.03|N|O|1996-01-12|1996-01-19|1996-02-05|NONE|TRUCK|counts doze against the blithely ironi|
+135|137|8|5|20|20742.60|0.01|0.04|N|O|1996-01-25|1995-11-20|1996-02-09|NONE|MAIL|theodolites. quickly p|
+135|115|5|6|13|13196.43|0.04|0.02|N|O|1995-11-12|1995-12-22|1995-11-17|NONE|FOB|nal ideas. final instr|
+160|15|2|1|36|32940.36|0.07|0.01|N|O|1997-03-11|1997-03-11|1997-03-20|COLLECT COD|MAIL|old, ironic deposits are quickly abov|
+160|87|8|2|22|21715.76|0.00|0.04|N|O|1997-02-18|1997-03-05|1997-03-05|COLLECT COD|RAIL|ncies about the request|
+160|21|10|3|34|31314.68|0.01|0.05|N|O|1997-01-31|1997-03-13|1997-02-14|NONE|FOB|st sleep even gifts. dependencies along|
+161|103|10|1|19|19058.90|0.01|0.01|A|F|1994-12-13|1994-11-19|1994-12-26|DELIVER IN PERSON|TRUCK|, regular sheaves sleep along|
+162|190|1|1|2|2180.38|0.02|0.01|N|O|1995-09-02|1995-06-17|1995-09-08|COLLECT COD|FOB|es! final somas integrate|
+163|168|3|1|43|45930.88|0.01|0.00|N|O|1997-09-19|1997-11-19|1997-10-03|COLLECT COD|REG AIR|al, bold dependencies wake. iron|
+163|121|2|2|13|13274.56|0.01|0.04|N|O|1997-11-11|1997-10-18|1997-12-07|DELIVER IN PERSON|TRUCK|inal requests. even pinto beans hag|
+163|37|3|3|27|25299.81|0.04|0.08|N|O|1997-12-26|1997-11-28|1998-01-05|COLLECT COD|REG AIR|ously express dependen|
+163|193|5|4|5|5465.95|0.02|0.00|N|O|1997-11-17|1997-10-09|1997-12-05|DELIVER IN PERSON|TRUCK| must belie|
+163|127|2|5|12|12325.44|0.10|0.00|N|O|1997-12-18|1997-10-26|1997-12-22|COLLECT COD|TRUCK|ly blithe accounts cajole |
+163|191|4|6|20|21823.80|0.00|0.07|N|O|1997-09-27|1997-11-15|1997-10-07|TAKE BACK RETURN|FOB|tructions integrate b|
+164|92|4|1|26|25794.34|0.09|0.04|A|F|1993-01-04|1992-11-21|1993-01-07|NONE|RAIL|s. blithely special courts are blithel|
+164|19|6|2|24|22056.24|0.05|0.05|R|F|1992-12-22|1992-11-27|1993-01-06|NONE|AIR|side of the slyly unusual theodolites. f|
+164|126|9|3|38|38992.56|0.03|0.06|R|F|1992-12-04|1992-11-23|1993-01-02|TAKE BACK RETURN|AIR|counts cajole fluffily regular packages. b|
+164|18|2|4|32|29376.32|0.05|0.01|R|F|1992-12-21|1992-12-23|1992-12-28|COLLECT COD|RAIL|ts wake again|
+164|148|1|5|43|45070.02|0.06|0.01|R|F|1992-11-26|1993-01-03|1992-12-08|COLLECT COD|RAIL|y carefully regular dep|
+164|109|10|6|27|27245.70|0.10|0.04|R|F|1992-12-23|1993-01-16|1993-01-10|DELIVER IN PERSON|AIR|ayers wake carefully a|
+164|4|7|7|23|20792.00|0.09|0.04|A|F|1992-11-03|1992-12-02|1992-11-12|NONE|REG AIR|ress packages haggle ideas. blithely spec|
+165|34|5|1|3|2802.09|0.01|0.08|R|F|1993-03-29|1993-03-06|1993-04-12|DELIVER IN PERSON|REG AIR|riously requests. depos|
+165|162|7|2|43|45672.88|0.08|0.05|R|F|1993-02-27|1993-04-19|1993-03-03|DELIVER IN PERSON|TRUCK|jole slyly according |
+165|59|1|3|15|14385.75|0.00|0.05|R|F|1993-04-10|1993-03-29|1993-05-01|COLLECT COD|SHIP| bold packages mainta|
+165|140|1|4|49|50966.86|0.07|0.06|A|F|1993-02-20|1993-04-02|1993-03-10|COLLECT COD|REG AIR|uses sleep slyly ruthlessly regular a|
+165|156|4|5|27|28516.05|0.01|0.04|R|F|1993-04-27|1993-03-04|1993-05-13|NONE|MAIL|around the ironic, even orb|
+166|65|2|1|37|35707.22|0.09|0.03|N|O|1995-11-16|1995-10-17|1995-12-13|NONE|MAIL|lar frays wake blithely a|
+166|167|8|2|13|13873.08|0.09|0.05|N|O|1995-11-09|1995-11-18|1995-11-14|COLLECT COD|SHIP|fully above the blithely fina|
+166|100|2|3|41|41004.10|0.07|0.03|N|O|1995-11-13|1995-11-07|1995-12-08|COLLECT COD|FOB|hily along the blithely pending fo|
+166|46|3|4|8|7568.32|0.05|0.02|N|O|1995-12-30|1995-11-29|1996-01-29|DELIVER IN PERSON|RAIL|e carefully bold |
+167|102|3|1|28|28058.80|0.06|0.01|R|F|1993-02-19|1993-02-16|1993-03-03|DELIVER IN PERSON|TRUCK|sly during the u|
+167|172|2|2|27|28948.59|0.09|0.00|R|F|1993-05-01|1993-03-31|1993-05-31|TAKE BACK RETURN|FOB|eans affix furiously-- packages|
+192|98|1|1|23|22956.07|0.00|0.00|N|O|1998-02-05|1998-02-06|1998-03-07|TAKE BACK RETURN|AIR|ly pending theodolites haggle quickly fluf|
+192|162|7|2|20|21243.20|0.07|0.01|N|O|1998-03-13|1998-02-02|1998-03-31|TAKE BACK RETURN|REG AIR|tes. carefu|
+192|111|8|3|15|15166.65|0.09|0.01|N|O|1998-01-30|1998-02-10|1998-02-23|TAKE BACK RETURN|TRUCK|he ironic requests haggle about|
+192|197|1|4|2|2194.38|0.06|0.02|N|O|1998-03-06|1998-02-03|1998-03-24|COLLECT COD|SHIP|s. dependencies nag furiously alongside|
+192|83|4|5|25|24577.00|0.02|0.03|N|O|1998-02-15|1998-01-11|1998-03-17|COLLECT COD|TRUCK|. carefully regular|
+192|142|9|6|45|46896.30|0.00|0.05|N|O|1998-03-11|1998-01-09|1998-04-03|NONE|MAIL|equests. ideas sleep idea|
+193|93|5|1|9|8937.81|0.06|0.06|A|F|1993-09-17|1993-10-08|1993-09-30|COLLECT COD|TRUCK|against the fluffily regular d|
+193|154|6|2|15|15812.25|0.02|0.07|R|F|1993-11-22|1993-10-09|1993-12-05|TAKE BACK RETURN|SHIP|ffily. regular packages d|
+193|94|6|3|23|22864.07|0.06|0.05|A|F|1993-08-21|1993-10-11|1993-09-02|DELIVER IN PERSON|TRUCK|ly even accounts wake blithely bold|
+194|3|6|1|17|15351.00|0.05|0.04|R|F|1992-05-24|1992-05-22|1992-05-30|COLLECT COD|AIR| regular deposi|
+194|184|5|2|1|1084.18|0.04|0.06|R|F|1992-04-30|1992-05-18|1992-05-23|NONE|REG AIR| regular theodolites. regular, iron|
+194|66|1|3|13|12558.78|0.08|0.08|A|F|1992-05-07|1992-06-18|1992-05-10|NONE|AIR|about the blit|
+194|146|7|4|36|37661.04|0.00|0.05|R|F|1992-05-21|1992-05-18|1992-05-27|TAKE BACK RETURN|RAIL|pecial packages wake after the slyly r|
+194|57|2|5|8|7656.40|0.04|0.00|R|F|1992-07-06|1992-06-25|1992-07-11|COLLECT COD|FOB|uriously unusual excuses|
+194|149|6|6|16|16786.24|0.06|0.03|A|F|1992-05-14|1992-06-14|1992-05-21|TAKE BACK RETURN|TRUCK|y regular requests. furious|
+194|168|7|7|21|22431.36|0.02|0.01|R|F|1992-05-06|1992-05-20|1992-05-07|COLLECT COD|REG AIR|accounts detect quickly dogged |
+195|85|6|1|6|5910.48|0.04|0.02|A|F|1994-01-09|1994-03-27|1994-01-28|COLLECT COD|REG AIR|y, even deposits haggle carefully. bli|
+195|94|8|2|41|40757.69|0.05|0.07|A|F|1994-02-24|1994-02-11|1994-03-20|NONE|TRUCK|rts detect in place of t|
+195|86|7|3|34|33526.72|0.08|0.08|R|F|1994-01-31|1994-02-11|1994-02-12|NONE|TRUCK| cajole furiously bold i|
+195|86|7|4|41|40429.28|0.06|0.04|R|F|1994-03-14|1994-03-13|1994-04-09|COLLECT COD|RAIL|ggle fluffily foxes. fluffily ironic ex|
+196|136|7|1|19|19686.47|0.03|0.02|R|F|1993-04-17|1993-05-27|1993-04-30|NONE|SHIP|sts maintain foxes. furiously regular p|
+196|10|3|2|15|13650.15|0.03|0.04|A|F|1993-07-05|1993-05-08|1993-07-06|TAKE BACK RETURN|SHIP|s accounts. furio|
+197|99|1|1|39|38964.51|0.02|0.04|N|O|1995-07-21|1995-07-01|1995-08-14|TAKE BACK RETURN|AIR|press accounts. daringly sp|
+197|178|8|2|8|8625.36|0.09|0.02|A|F|1995-04-17|1995-07-01|1995-04-27|DELIVER IN PERSON|SHIP|y blithely even deposits. blithely fina|
+197|156|4|3|17|17954.55|0.06|0.02|N|O|1995-08-02|1995-06-23|1995-08-03|COLLECT COD|REG AIR|ts. careful|
+197|18|5|4|25|22950.25|0.04|0.01|N|F|1995-06-13|1995-05-23|1995-06-24|TAKE BACK RETURN|FOB|s-- quickly final accounts|
+197|42|9|5|14|13188.56|0.09|0.01|R|F|1995-05-08|1995-05-24|1995-05-12|TAKE BACK RETURN|RAIL|use slyly slyly silent depo|
+197|106|1|6|1|1006.10|0.07|0.05|N|O|1995-07-15|1995-06-21|1995-08-11|COLLECT COD|RAIL| even, thin dependencies sno|
+198|57|8|1|33|31582.65|0.07|0.02|N|O|1998-01-05|1998-03-20|1998-01-10|TAKE BACK RETURN|TRUCK|carefully caref|
+198|16|10|2|20|18320.20|0.03|0.00|N|O|1998-01-15|1998-03-31|1998-01-25|DELIVER IN PERSON|FOB|carefully final escapades a|
+198|149|2|3|15|15737.10|0.04|0.02|N|O|1998-04-12|1998-02-26|1998-04-15|COLLECT COD|MAIL|es. quickly pending deposits s|
+198|11|5|4|35|31885.35|0.08|0.02|N|O|1998-02-27|1998-03-23|1998-03-14|TAKE BACK RETURN|RAIL|ests nod quickly furiously sly pinto be|
+198|102|3|5|33|33069.30|0.02|0.01|N|O|1998-03-22|1998-03-12|1998-04-14|DELIVER IN PERSON|SHIP|ending foxes acr|
+199|133|9|1|50|51656.50|0.02|0.00|N|O|1996-06-12|1996-06-03|1996-07-04|DELIVER IN PERSON|MAIL|essly regular ideas boost sly|
+199|134|5|2|30|31023.90|0.08|0.05|N|O|1996-03-27|1996-05-29|1996-04-14|NONE|TRUCK|ilent packages doze quickly. thinly |
+224|151|2|1|16|16818.40|0.04|0.00|A|F|1994-08-01|1994-07-30|1994-08-27|DELIVER IN PERSON|MAIL|y unusual foxes |
+224|109|2|2|34|34309.40|0.04|0.08|R|F|1994-07-13|1994-08-25|1994-07-31|COLLECT COD|TRUCK| carefully. final platelets |
+224|190|1|3|41|44697.79|0.07|0.04|A|F|1994-09-01|1994-09-15|1994-09-02|TAKE BACK RETURN|SHIP|after the furiou|
+224|167|4|4|12|12805.92|0.08|0.06|R|F|1994-10-12|1994-08-29|1994-10-20|DELIVER IN PERSON|MAIL|uriously regular packages. slyly fina|
+224|94|7|5|45|44734.05|0.07|0.07|R|F|1994-08-14|1994-09-02|1994-08-27|COLLECT COD|AIR|leep furiously regular requests. furiousl|
+224|51|3|6|4|3804.20|0.02|0.00|R|F|1994-09-08|1994-08-24|1994-10-04|DELIVER IN PERSON|FOB|tructions |
+225|172|3|1|4|4288.68|0.09|0.07|N|O|1995-08-05|1995-08-19|1995-09-03|TAKE BACK RETURN|SHIP|ng the ironic packages. asymptotes among |
+225|131|7|2|3|3093.39|0.00|0.08|N|O|1995-07-25|1995-07-08|1995-08-17|DELIVER IN PERSON|REG AIR| fluffily about the carefully bold a|
+225|199|2|3|45|49463.55|0.06|0.01|N|O|1995-08-17|1995-08-20|1995-08-30|TAKE BACK RETURN|FOB|the slyly even platelets use aro|
+225|147|4|4|24|25131.36|0.00|0.06|N|O|1995-09-23|1995-08-05|1995-10-16|COLLECT COD|MAIL|ironic accounts are final account|
+225|8|5|5|31|28148.00|0.04|0.06|N|O|1995-06-21|1995-07-24|1995-07-04|TAKE BACK RETURN|FOB|special platelets. quickly r|
+225|132|8|6|12|12385.56|0.00|0.00|A|F|1995-06-04|1995-07-15|1995-06-08|COLLECT COD|MAIL| unusual requests. bus|
+225|142|1|7|44|45854.16|0.10|0.06|N|O|1995-09-22|1995-08-16|1995-10-22|NONE|REG AIR|leep slyly |
+226|97|9|1|4|3988.36|0.00|0.00|R|F|1993-03-31|1993-04-30|1993-04-10|NONE|TRUCK|c foxes integrate carefully against th|
+226|138|4|2|46|47753.98|0.06|0.01|A|F|1993-07-06|1993-04-24|1993-07-13|COLLECT COD|FOB|s. carefully bold accounts cajol|
+226|38|4|3|35|32831.05|0.09|0.03|A|F|1993-03-31|1993-05-18|1993-04-01|NONE|RAIL|osits cajole. final, even foxes a|
+226|41|10|4|45|42346.80|0.10|0.02|R|F|1993-04-17|1993-05-27|1993-05-11|DELIVER IN PERSON|AIR| carefully pending pi|
+226|118|8|5|2|2036.22|0.07|0.02|R|F|1993-03-26|1993-04-13|1993-04-20|TAKE BACK RETURN|SHIP|al platelets. express somas |
+226|83|4|6|48|47187.84|0.02|0.00|A|F|1993-06-11|1993-05-15|1993-06-19|NONE|REG AIR|efully silent packages. final deposit|
+226|118|8|7|14|14253.54|0.09|0.00|R|F|1993-05-20|1993-06-05|1993-05-27|COLLECT COD|MAIL|ep carefully regular accounts. ironic|
+227|166|1|1|19|20257.04|0.05|0.06|N|O|1995-12-10|1996-01-30|1995-12-26|NONE|RAIL|s cajole furiously a|
+227|175|3|2|24|25804.08|0.07|0.07|N|O|1996-02-03|1995-12-24|1996-02-12|COLLECT COD|SHIP|uses across the blithe dependencies cajol|
+228|5|8|1|3|2715.00|0.10|0.08|A|F|1993-05-20|1993-04-08|1993-05-26|DELIVER IN PERSON|SHIP|ckages. sly|
+229|84|5|1|20|19681.60|0.02|0.03|R|F|1994-01-11|1994-01-31|1994-01-26|DELIVER IN PERSON|REG AIR|le. instructions use across the quickly fin|
+229|129|10|2|29|29844.48|0.07|0.00|A|F|1994-03-15|1994-03-02|1994-03-26|COLLECT COD|SHIP|s, final request|
+229|79|10|3|28|27413.96|0.02|0.02|R|F|1994-02-10|1994-02-02|1994-03-10|DELIVER IN PERSON|FOB| final, regular requests. platel|
+229|177|6|4|3|3231.51|0.02|0.08|R|F|1994-03-22|1994-03-24|1994-04-04|DELIVER IN PERSON|REG AIR|posits. furiously regular theodol|
+229|156|1|5|33|34852.95|0.03|0.06|R|F|1994-03-25|1994-02-11|1994-04-13|NONE|FOB| deposits; bold, ruthless theodolites|
+229|106|9|6|29|29176.90|0.04|0.00|R|F|1994-01-14|1994-02-16|1994-01-22|NONE|FOB|uriously pending |
+230|186|7|1|46|49964.28|0.09|0.00|R|F|1994-02-03|1994-01-15|1994-02-23|TAKE BACK RETURN|SHIP|old packages ha|
+230|195|7|2|6|6571.14|0.03|0.08|A|F|1994-01-26|1994-01-25|1994-02-13|NONE|REG AIR| sleep furiously about the p|
+230|8|5|3|1|908.00|0.07|0.06|R|F|1994-01-22|1994-01-03|1994-02-05|TAKE BACK RETURN|RAIL|blithely unusual dolphins. bold, ex|
+230|10|3|4|44|40040.44|0.08|0.06|R|F|1994-02-09|1994-01-18|1994-03-11|NONE|MAIL|deposits integrate slyly sile|
+230|19|9|5|8|7352.08|0.09|0.06|R|F|1993-11-03|1994-01-20|1993-11-11|TAKE BACK RETURN|TRUCK|g the instructions. fluffil|
+230|34|10|6|8|7472.24|0.00|0.05|R|F|1993-11-21|1994-01-05|1993-12-19|TAKE BACK RETURN|FOB|nal ideas. silent, reg|
+231|159|10|1|16|16946.40|0.04|0.08|R|F|1994-11-20|1994-10-29|1994-12-17|TAKE BACK RETURN|AIR|e furiously ironic pinto beans.|
+231|84|5|2|46|45267.68|0.04|0.05|R|F|1994-12-13|1994-12-02|1994-12-14|DELIVER IN PERSON|SHIP|affix blithely. bold requests among the f|
+231|199|1|3|50|54959.50|0.09|0.01|A|F|1994-12-11|1994-12-14|1994-12-13|NONE|RAIL|onic packages haggle fluffily a|
+231|57|8|4|31|29668.55|0.08|0.02|A|F|1994-11-05|1994-12-27|1994-11-30|TAKE BACK RETURN|SHIP|iously special decoys wake q|
+256|89|10|1|22|21759.76|0.09|0.02|R|F|1994-01-12|1993-12-28|1994-01-26|COLLECT COD|FOB|ke quickly ironic, ironic deposits. reg|
+256|119|6|2|40|40764.40|0.10|0.01|A|F|1993-11-30|1993-12-13|1993-12-02|NONE|FOB|nal theodolites. deposits cajole s|
+256|130|9|3|45|46355.85|0.02|0.08|R|F|1994-01-14|1994-01-17|1994-02-10|COLLECT COD|SHIP| grouches. ideas wake quickly ar|
+257|147|8|1|7|7329.98|0.05|0.02|N|O|1998-06-18|1998-05-15|1998-06-27|COLLECT COD|FOB|ackages sleep bold realms. f|
+258|107|4|1|8|8056.80|0.00|0.07|R|F|1994-01-20|1994-03-21|1994-02-09|NONE|REG AIR|ully about the fluffily silent dependencies|
+258|197|1|2|40|43887.60|0.10|0.01|A|F|1994-03-13|1994-02-23|1994-04-05|DELIVER IN PERSON|FOB|silent frets nod daringly busy, bold|
+258|162|3|3|45|47797.20|0.07|0.07|R|F|1994-03-04|1994-02-13|1994-03-30|DELIVER IN PERSON|TRUCK|regular excuses-- fluffily ruthl|
+258|133|9|4|31|32027.03|0.02|0.05|A|F|1994-04-20|1994-03-20|1994-04-28|COLLECT COD|REG AIR| slyly blithely special mul|
+258|36|2|5|25|23400.75|0.08|0.02|A|F|1994-04-13|1994-02-26|1994-04-29|TAKE BACK RETURN|TRUCK|leep pending packages.|
+258|147|4|6|36|37697.04|0.09|0.04|A|F|1994-01-11|1994-03-04|1994-01-18|DELIVER IN PERSON|AIR|nic asymptotes. slyly silent r|
+259|99|10|1|14|13987.26|0.00|0.08|A|F|1993-12-17|1993-12-09|1993-12-31|COLLECT COD|SHIP|ons against the express acco|
+259|162|1|2|14|14870.24|0.03|0.05|R|F|1993-11-10|1993-11-20|1993-11-17|DELIVER IN PERSON|FOB|ully even, regul|
+259|24|5|3|42|38808.84|0.09|0.00|R|F|1993-10-20|1993-11-18|1993-11-12|NONE|TRUCK|the slyly ironic pinto beans. fi|
+259|196|10|4|3|3288.57|0.08|0.06|R|F|1993-10-04|1993-11-07|1993-10-14|TAKE BACK RETURN|SHIP|ng slyly at the accounts.|
+259|193|6|5|6|6559.14|0.00|0.05|R|F|1993-12-05|1993-12-22|1993-12-21|COLLECT COD|TRUCK| requests sleep|
+260|156|7|1|50|52807.50|0.07|0.08|N|O|1997-03-24|1997-02-09|1997-04-20|TAKE BACK RETURN|REG AIR|c deposits |
+260|183|4|2|26|28162.68|0.02|0.07|N|O|1996-12-12|1997-02-06|1996-12-15|NONE|TRUCK|ld theodolites boost fl|
+260|42|1|3|27|25435.08|0.05|0.08|N|O|1997-03-23|1997-02-15|1997-04-22|TAKE BACK RETURN|RAIL|ions according to the|
+260|6|1|4|29|26274.00|0.10|0.06|N|O|1997-03-15|1997-01-14|1997-04-13|NONE|MAIL|fluffily even asymptotes. express wa|
+260|96|9|5|44|43827.96|0.01|0.05|N|O|1997-03-26|1997-02-03|1997-04-19|DELIVER IN PERSON|MAIL|above the blithely ironic instr|
+261|2|7|1|34|30668.00|0.05|0.08|R|F|1993-08-18|1993-09-24|1993-08-20|COLLECT COD|REG AIR|c packages. asymptotes da|
+261|66|7|2|20|19321.20|0.00|0.06|R|F|1993-10-21|1993-08-02|1993-11-04|DELIVER IN PERSON|RAIL|ites hinder |
+261|174|3|3|28|30076.76|0.08|0.03|R|F|1993-07-24|1993-08-20|1993-08-05|COLLECT COD|AIR|ironic packages nag slyly. carefully fin|
+261|119|3|4|49|49936.39|0.04|0.05|R|F|1993-09-12|1993-08-31|1993-10-07|COLLECT COD|SHIP|ions. bold accounts |
+261|61|6|5|49|47091.94|0.01|0.08|A|F|1993-09-29|1993-09-08|1993-10-01|COLLECT COD|SHIP| pinto beans haggle slyly furiously pending|
+261|97|9|6|20|19941.80|0.06|0.06|A|F|1993-10-15|1993-09-05|1993-11-07|NONE|AIR|ing to the special, ironic deposi|
+262|192|3|1|39|42595.41|0.01|0.05|N|O|1996-01-15|1996-02-18|1996-01-28|COLLECT COD|RAIL|usual, regular requests|
+262|61|6|2|33|31714.98|0.09|0.03|N|O|1996-03-10|1996-01-31|1996-03-27|TAKE BACK RETURN|AIR|atelets sleep furiously. requests cajole. b|
+262|59|1|3|35|33566.75|0.05|0.08|N|O|1996-03-12|1996-02-14|1996-04-11|COLLECT COD|MAIL|lites cajole along the pending packag|
+263|24|9|1|22|20328.44|0.06|0.08|R|F|1994-08-24|1994-06-20|1994-09-09|NONE|FOB|efully express fo|
+263|85|6|2|9|8865.72|0.08|0.00|A|F|1994-07-21|1994-07-16|1994-08-08|TAKE BACK RETURN|TRUCK|lms wake bl|
+263|143|2|3|50|52157.00|0.06|0.04|R|F|1994-08-18|1994-07-31|1994-08-22|NONE|TRUCK|re the packages. special|
+288|51|3|1|31|29482.55|0.00|0.03|N|O|1997-03-17|1997-04-28|1997-04-06|TAKE BACK RETURN|AIR|instructions wa|
+288|117|1|2|49|49838.39|0.08|0.05|N|O|1997-04-19|1997-05-19|1997-05-18|TAKE BACK RETURN|TRUCK|ic excuses sleep always spe|
+288|99|10|3|36|35967.24|0.02|0.02|N|O|1997-02-22|1997-05-07|1997-03-07|TAKE BACK RETURN|TRUCK|yly pending excu|
+288|79|10|4|19|18602.33|0.07|0.07|N|O|1997-03-14|1997-04-04|1997-03-26|NONE|MAIL|deposits. blithely quick courts ar|
+288|162|9|5|31|32926.96|0.10|0.04|N|O|1997-05-29|1997-04-24|1997-06-20|TAKE BACK RETURN|RAIL|ns. fluffily|
+289|174|2|1|25|26854.25|0.07|0.05|N|O|1997-03-18|1997-05-05|1997-04-15|DELIVER IN PERSON|FOB|out the quickly bold theodol|
+289|112|2|2|6|6072.66|0.06|0.05|N|O|1997-02-18|1997-05-08|1997-03-19|DELIVER IN PERSON|SHIP|d packages use fluffily furiously|
+289|17|4|3|44|40348.44|0.10|0.08|N|O|1997-06-05|1997-04-20|1997-07-02|COLLECT COD|MAIL|ly ironic foxes. asymptotes |
+289|40|6|4|48|45121.92|0.01|0.08|N|O|1997-03-14|1997-03-30|1997-03-24|DELIVER IN PERSON|RAIL|sits cajole. bold pinto beans x-ray fl|
+289|47|4|5|13|12311.52|0.10|0.03|N|O|1997-06-08|1997-04-06|1997-06-18|TAKE BACK RETURN|REG AIR|ts. quickly bold deposits alongside|
+290|6|1|1|35|31710.00|0.01|0.02|R|F|1994-04-01|1994-02-05|1994-04-27|NONE|MAIL|ove the final foxes detect slyly fluffily|
+290|129|4|2|2|2058.24|0.05|0.04|A|F|1994-01-30|1994-02-13|1994-02-21|TAKE BACK RETURN|TRUCK|. permanently furious reques|
+290|2|5|3|5|4510.00|0.03|0.05|A|F|1994-01-19|1994-02-24|1994-01-27|NONE|MAIL|ans integrate. requests sleep. fur|
+290|124|9|4|23|23554.76|0.05|0.08|R|F|1994-03-14|1994-02-21|1994-04-09|NONE|AIR|refully unusual packages. |
+291|123|6|1|21|21485.52|0.05|0.07|A|F|1994-05-26|1994-05-10|1994-06-23|COLLECT COD|TRUCK|y quickly regular theodolites. final t|
+291|138|9|2|19|19724.47|0.08|0.02|R|F|1994-06-14|1994-04-25|1994-06-19|NONE|REG AIR|e. ruthlessly final accounts after the|
+291|61|8|3|30|28831.80|0.10|0.02|R|F|1994-03-22|1994-04-30|1994-03-24|DELIVER IN PERSON|FOB| fluffily regular deposits. quickl|
+292|154|5|1|8|8433.20|0.10|0.03|R|F|1992-02-18|1992-03-30|1992-03-18|DELIVER IN PERSON|RAIL|sily bold deposits alongside of the ex|
+292|100|1|2|24|24002.40|0.08|0.04|R|F|1992-03-24|1992-03-06|1992-04-20|COLLECT COD|TRUCK| bold, pending theodolites u|
+293|9|6|1|14|12726.00|0.02|0.05|R|F|1992-10-19|1992-12-23|1992-11-10|DELIVER IN PERSON|SHIP|es. packages above the|
+293|187|8|2|11|11958.98|0.10|0.04|R|F|1992-12-24|1992-12-01|1993-01-12|COLLECT COD|MAIL| affix carefully quickly special idea|
+293|118|8|3|13|13235.43|0.04|0.02|A|F|1992-12-17|1992-12-26|1992-12-22|COLLECT COD|RAIL| wake after the quickly even deposits. bli|
+294|60|2|1|31|29761.86|0.00|0.01|R|F|1993-08-06|1993-08-19|1993-08-13|TAKE BACK RETURN|AIR|le fluffily along the quick|
+295|198|10|1|29|31847.51|0.02|0.07|A|F|1994-11-09|1994-12-08|1994-12-07|COLLECT COD|MAIL|inst the carefully ironic pinto beans. blit|
+295|92|6|2|26|25794.34|0.04|0.03|R|F|1994-12-13|1994-11-30|1995-01-06|DELIVER IN PERSON|AIR|ts above the slyly regular requests x-ray q|
+295|16|10|3|8|7328.08|0.10|0.07|R|F|1995-01-13|1994-11-17|1995-01-25|NONE|TRUCK| final instructions h|
+295|61|10|4|26|24987.56|0.10|0.04|A|F|1995-01-12|1994-11-22|1995-01-22|DELIVER IN PERSON|MAIL| carefully iron|
+320|5|2|1|30|27150.00|0.05|0.01|N|O|1997-12-04|1998-01-21|1997-12-13|NONE|RAIL| ironic, final accounts wake quick de|
+320|193|5|2|13|14211.47|0.03|0.00|N|O|1997-12-16|1997-12-26|1997-12-17|TAKE BACK RETURN|AIR|he furiously regular pinto beans. car|
+321|1|8|1|21|18921.00|0.01|0.08|A|F|1993-07-18|1993-04-24|1993-08-13|TAKE BACK RETURN|REG AIR|hockey players sleep slyly sl|
+321|141|4|2|41|42686.74|0.08|0.07|R|F|1993-06-21|1993-06-07|1993-07-09|NONE|REG AIR|special packages shall have to doze blit|
+322|153|8|1|12|12637.80|0.08|0.07|A|F|1992-06-29|1992-05-30|1992-07-11|NONE|AIR|ular theodolites promise qu|
+322|44|5|2|48|45313.92|0.02|0.07|A|F|1992-06-11|1992-06-16|1992-06-26|COLLECT COD|RAIL|dolites detect qu|
+322|13|3|3|20|18260.20|0.04|0.01|R|F|1992-04-26|1992-05-04|1992-05-22|DELIVER IN PERSON|MAIL|ckly toward |
+322|184|5|4|10|10841.80|0.06|0.03|R|F|1992-04-12|1992-05-13|1992-04-14|DELIVER IN PERSON|AIR| deposits grow slyly according to th|
+322|12|2|5|35|31920.35|0.07|0.06|A|F|1992-07-17|1992-05-03|1992-08-14|TAKE BACK RETURN|RAIL|egular accounts cajole carefully. even d|
+322|34|5|6|3|2802.09|0.08|0.05|A|F|1992-07-03|1992-05-10|1992-07-28|NONE|AIR|ending, ironic deposits along the blith|
+322|38|4|7|5|4690.15|0.01|0.02|A|F|1992-04-15|1992-05-12|1992-04-26|COLLECT COD|REG AIR| special grouches sleep quickly instructio|
+323|164|9|1|50|53208.00|0.05|0.04|A|F|1994-04-20|1994-04-25|1994-05-12|DELIVER IN PERSON|REG AIR|cial requests |
+323|96|8|2|18|17929.62|0.06|0.07|R|F|1994-04-13|1994-06-02|1994-05-10|DELIVER IN PERSON|TRUCK|posits cajole furiously pinto beans. |
+323|143|4|3|9|9388.26|0.07|0.04|A|F|1994-06-26|1994-06-10|1994-07-13|COLLECT COD|TRUCK|nic accounts. regular, regular pack|
+324|200|3|1|26|28605.20|0.07|0.01|R|F|1992-04-19|1992-05-28|1992-05-12|DELIVER IN PERSON|RAIL|ross the slyly regular s|
+325|159|1|1|34|36011.10|0.09|0.04|A|F|1993-10-28|1993-12-13|1993-11-17|TAKE BACK RETURN|MAIL|ly bold deposits. always iron|
+325|186|7|2|5|5430.90|0.07|0.08|A|F|1994-01-02|1994-01-05|1994-01-04|TAKE BACK RETURN|MAIL| theodolites. |
+325|19|3|3|35|32165.35|0.07|0.07|A|F|1993-12-06|1994-01-03|1993-12-26|DELIVER IN PERSON|REG AIR|packages wa|
+326|180|9|1|41|44287.38|0.06|0.03|N|O|1995-08-30|1995-07-09|1995-09-12|DELIVER IN PERSON|TRUCK|ily quickly bold ideas.|
+326|20|4|2|38|34960.76|0.02|0.08|N|O|1995-09-12|1995-08-23|1995-09-14|COLLECT COD|RAIL|es sleep slyly. carefully regular inst|
+326|184|5|3|25|27104.50|0.03|0.04|N|O|1995-08-03|1995-07-27|1995-08-16|NONE|AIR|ily furiously unusual accounts. |
+326|85|6|4|5|4925.40|0.03|0.08|N|O|1995-07-29|1995-07-13|1995-08-12|NONE|REG AIR|deas sleep according to the sometimes spe|
+326|35|6|5|31|28985.93|0.04|0.08|N|O|1995-09-27|1995-07-06|1995-10-22|NONE|TRUCK|cies sleep quick|
+326|157|9|6|41|43343.15|0.02|0.00|N|O|1995-07-05|1995-07-23|1995-07-20|TAKE BACK RETURN|REG AIR|to beans wake before the furiously re|
+326|43|10|7|47|44322.88|0.04|0.04|N|O|1995-09-16|1995-07-04|1995-10-04|NONE|REG AIR| special accounts sleep |
+327|144|3|1|16|16706.24|0.03|0.01|N|O|1995-07-05|1995-06-07|1995-07-09|TAKE BACK RETURN|TRUCK|cial ideas sleep af|
+327|42|9|2|9|8478.36|0.09|0.05|A|F|1995-05-24|1995-07-11|1995-06-05|NONE|AIR| asymptotes are fu|
+352|64|5|1|17|16389.02|0.07|0.05|R|F|1994-06-02|1994-05-31|1994-06-29|NONE|FOB|pending deposits sleep furiously |
+353|120|7|1|41|41824.92|0.00|0.06|A|F|1994-03-25|1994-03-31|1994-03-30|DELIVER IN PERSON|AIR|refully final theodoli|
+353|148|9|2|29|30396.06|0.09|0.00|A|F|1994-01-11|1994-03-19|1994-02-09|COLLECT COD|FOB|ctions impr|
+353|135|1|3|12|12421.56|0.06|0.01|R|F|1994-01-02|1994-03-26|1994-01-19|DELIVER IN PERSON|RAIL|g deposits cajole |
+353|78|7|4|46|44991.22|0.00|0.04|A|F|1994-04-14|1994-01-31|1994-05-05|DELIVER IN PERSON|FOB| ironic dolphins |
+353|117|4|5|9|9153.99|0.02|0.02|A|F|1994-03-15|1994-03-20|1994-03-18|TAKE BACK RETURN|RAIL|ual accounts! carefu|
+353|103|4|6|39|39120.90|0.02|0.05|A|F|1994-01-15|1994-03-30|1994-02-01|NONE|MAIL|losely quickly even accounts. c|
+354|50|7|1|14|13300.70|0.08|0.04|N|O|1996-04-12|1996-06-03|1996-05-08|NONE|SHIP|quickly regular grouches will eat. careful|
+354|194|8|2|24|26260.56|0.01|0.01|N|O|1996-05-08|1996-05-17|1996-06-07|DELIVER IN PERSON|AIR|y silent requests. regular, even accounts|
+354|59|10|3|50|47952.50|0.08|0.05|N|O|1996-03-21|1996-05-20|1996-04-04|COLLECT COD|TRUCK|to beans s|
+354|107|4|4|7|7049.70|0.06|0.01|N|O|1996-05-07|1996-04-18|1996-05-24|NONE|MAIL|ously idly ironic accounts-- quickl|
+354|31|2|5|18|16758.54|0.04|0.08|N|O|1996-03-31|1996-05-13|1996-04-27|DELIVER IN PERSON|RAIL| about the carefully unusual |
+354|62|1|6|36|34634.16|0.03|0.02|N|O|1996-03-19|1996-05-29|1996-03-30|NONE|AIR|onic requests thrash bold g|
+354|5|10|7|14|12670.00|0.01|0.07|N|O|1996-07-06|1996-06-08|1996-07-10|TAKE BACK RETURN|MAIL|t thinly above the ironic, |
+355|114|1|1|31|31437.41|0.09|0.07|A|F|1994-07-13|1994-08-18|1994-07-18|DELIVER IN PERSON|FOB|y unusual, ironic|
+355|97|1|2|41|40880.69|0.05|0.00|A|F|1994-08-15|1994-07-19|1994-09-06|DELIVER IN PERSON|TRUCK| deposits. carefully r|
+356|46|7|1|4|3784.16|0.10|0.01|A|F|1994-07-28|1994-08-01|1994-08-04|DELIVER IN PERSON|REG AIR| the dependencies nod unusual, final ac|
+356|108|3|2|48|48388.80|0.02|0.03|R|F|1994-08-12|1994-07-31|1994-08-26|NONE|FOB|unusual packages. furiously |
+356|119|3|3|35|35668.85|0.08|0.07|R|F|1994-10-14|1994-07-31|1994-10-23|COLLECT COD|TRUCK|s. unusual, final|
+356|56|1|4|41|39198.05|0.07|0.05|A|F|1994-09-28|1994-09-20|1994-10-07|COLLECT COD|SHIP| according to the express foxes will|
+356|125|8|5|37|37929.44|0.05|0.03|A|F|1994-07-15|1994-08-24|1994-08-09|DELIVER IN PERSON|FOB|ndencies are since the packag|
+357|114|5|1|26|26366.86|0.06|0.03|N|O|1996-12-28|1996-11-26|1997-01-13|NONE|FOB| carefully pending accounts use a|
+357|186|7|2|36|39102.48|0.07|0.06|N|O|1996-12-28|1996-11-13|1997-01-24|DELIVER IN PERSON|AIR|d the carefully even requests. |
+357|165|2|3|32|34085.12|0.05|0.07|N|O|1997-01-28|1996-12-29|1997-02-14|NONE|MAIL|y above the carefully final accounts|
+358|191|3|1|41|44738.79|0.06|0.01|A|F|1993-11-18|1993-11-14|1993-11-28|NONE|TRUCK|ely frets. furious deposits sleep |
+358|190|1|2|32|34886.08|0.05|0.08|A|F|1993-10-18|1993-12-12|1993-10-31|NONE|TRUCK|y final foxes sleep blithely sl|
+358|169|6|3|40|42766.40|0.09|0.01|A|F|1993-12-05|1993-11-04|1994-01-01|COLLECT COD|MAIL|ng the ironic theo|
+358|97|10|4|15|14956.35|0.08|0.08|A|F|1993-10-04|1993-12-17|1993-10-23|TAKE BACK RETURN|MAIL|out the blithely ironic deposits slee|
+358|29|2|5|18|16722.36|0.01|0.02|R|F|1993-10-07|1993-11-01|1993-10-26|COLLECT COD|SHIP|olphins haggle ironic accounts. f|
+358|162|3|6|32|33989.12|0.03|0.05|R|F|1993-12-21|1993-11-06|1994-01-17|DELIVER IN PERSON|RAIL|lyly express deposits |
+358|83|4|7|45|44238.60|0.05|0.02|A|F|1993-12-08|1993-10-29|1993-12-30|NONE|REG AIR|to beans. regular, unusual deposits sl|
+359|166|7|1|30|31984.80|0.00|0.08|A|F|1995-01-06|1995-02-20|1995-01-20|TAKE BACK RETURN|AIR|uses detect spec|
+359|12|9|2|18|16416.18|0.00|0.03|A|F|1995-01-27|1995-03-18|1995-01-31|DELIVER IN PERSON|RAIL|unusual warthogs. ironically sp|
+359|132|8|3|17|17546.21|0.07|0.06|A|F|1995-01-31|1995-03-18|1995-02-10|COLLECT COD|SHIP|sts according to the blithely|
+359|90|1|4|38|37623.42|0.10|0.08|R|F|1995-03-30|1995-01-20|1995-04-25|DELIVER IN PERSON|RAIL|g furiously. regular, sile|
+359|168|5|5|11|11749.76|0.01|0.03|A|F|1995-02-15|1995-01-27|1995-02-18|NONE|FOB|rets wake blithely. slyly final dep|
+359|183|4|6|23|24913.14|0.04|0.07|R|F|1995-01-31|1995-03-11|1995-02-16|DELIVER IN PERSON|REG AIR|ic courts snooze quickly furiously final fo|
+384|179|8|1|38|41008.46|0.07|0.01|R|F|1992-06-02|1992-04-18|1992-06-10|DELIVER IN PERSON|TRUCK|totes cajole blithely against the even|
+384|64|3|2|49|47238.94|0.09|0.07|A|F|1992-04-01|1992-04-25|1992-04-18|COLLECT COD|AIR|refully carefully ironic instructions. bl|
+384|182|3|3|11|11903.98|0.02|0.08|A|F|1992-04-02|1992-04-21|1992-04-15|COLLECT COD|MAIL|ash carefully|
+384|93|6|4|11|10923.99|0.00|0.06|R|F|1992-06-24|1992-05-29|1992-07-22|COLLECT COD|TRUCK|nic excuses are furiously above the blith|
+384|132|8|5|14|14449.82|0.08|0.06|R|F|1992-06-14|1992-05-29|1992-07-05|DELIVER IN PERSON|TRUCK|ckages are slyly after the slyly specia|
+385|167|6|1|7|7470.12|0.05|0.06|N|O|1996-05-23|1996-05-09|1996-06-06|DELIVER IN PERSON|REG AIR| special asymptote|
+385|54|9|2|46|43886.30|0.08|0.07|N|O|1996-03-29|1996-05-17|1996-04-18|NONE|REG AIR|lthily ironic f|
+386|153|5|1|39|41072.85|0.10|0.07|A|F|1995-05-10|1995-02-28|1995-05-25|NONE|SHIP|hely. carefully regular accounts hag|
+386|69|4|2|16|15504.96|0.06|0.01|A|F|1995-04-12|1995-04-18|1995-05-11|DELIVER IN PERSON|MAIL|lithely fluffi|
+386|131|2|3|37|38151.81|0.09|0.04|A|F|1995-05-23|1995-03-01|1995-05-25|TAKE BACK RETURN|MAIL|ending pearls breach fluffily. slyly pen|
+387|137|8|1|1|1037.13|0.08|0.03|N|O|1997-05-06|1997-04-23|1997-05-10|NONE|SHIP| pinto beans wake furiously carefu|
+387|153|4|2|42|44232.30|0.07|0.05|N|O|1997-05-25|1997-02-25|1997-05-29|DELIVER IN PERSON|RAIL|lithely final theodolites.|
+387|97|10|3|40|39883.60|0.09|0.02|N|O|1997-03-08|1997-04-18|1997-03-31|COLLECT COD|TRUCK| quickly ironic platelets are slyly. fluff|
+387|56|7|4|19|18164.95|0.08|0.00|N|O|1997-03-14|1997-04-21|1997-04-04|NONE|REG AIR|gular dependencies|
+387|149|6|5|32|33572.48|0.08|0.06|N|O|1997-05-02|1997-04-11|1997-05-11|DELIVER IN PERSON|TRUCK|gle. silent, fur|
+388|33|9|1|42|39187.26|0.05|0.06|R|F|1993-02-21|1993-02-26|1993-03-15|COLLECT COD|FOB|accounts sleep furiously|
+388|128|9|2|46|47293.52|0.07|0.01|A|F|1993-03-22|1993-01-26|1993-03-24|COLLECT COD|FOB|to beans nag about the careful reque|
+388|65|2|3|40|38602.40|0.06|0.01|A|F|1992-12-24|1993-01-28|1993-01-19|TAKE BACK RETURN|REG AIR|quests against the carefully unusual epi|
+389|190|1|1|2|2180.38|0.09|0.00|R|F|1994-04-13|1994-04-10|1994-04-25|TAKE BACK RETURN|RAIL|fts. courts eat blithely even dependenc|
+390|107|10|1|10|10071.00|0.02|0.05|N|O|1998-05-26|1998-07-06|1998-06-23|TAKE BACK RETURN|SHIP| requests. final accounts x-ray beside the|
+390|124|7|2|17|17410.04|0.09|0.06|N|O|1998-06-07|1998-06-14|1998-07-07|COLLECT COD|SHIP|ending, pending pinto beans wake slyl|
+390|184|5|3|46|49872.28|0.07|0.04|N|O|1998-06-06|1998-05-20|1998-06-14|DELIVER IN PERSON|SHIP|cial excuses. bold, pending packages|
+390|142|3|4|42|43769.88|0.01|0.05|N|O|1998-06-06|1998-06-22|1998-07-05|COLLECT COD|SHIP|counts nag across the sly, sil|
+390|128|3|5|13|13365.56|0.02|0.06|N|O|1998-07-08|1998-05-10|1998-07-18|DELIVER IN PERSON|SHIP|sleep carefully idle packages. blithely |
+390|125|4|6|11|11276.32|0.09|0.06|N|O|1998-05-05|1998-05-15|1998-06-01|DELIVER IN PERSON|SHIP|according to the foxes are furiously |
+390|85|6|7|24|23641.92|0.05|0.02|N|O|1998-04-18|1998-05-19|1998-04-28|TAKE BACK RETURN|AIR|y. enticingly final depos|
+391|122|1|1|14|14309.68|0.09|0.02|R|F|1995-02-11|1995-02-03|1995-02-13|TAKE BACK RETURN|TRUCK| escapades sleep furiously about |
+416|94|6|1|25|24852.25|0.00|0.05|A|F|1993-10-11|1993-11-26|1993-10-21|DELIVER IN PERSON|TRUCK|y final theodolites about|
+416|111|1|2|22|22244.42|0.10|0.00|R|F|1993-12-27|1993-12-17|1994-01-09|COLLECT COD|RAIL|rint blithely above the pending sentim|
+416|175|5|3|25|26879.25|0.07|0.01|R|F|1993-10-16|1993-12-03|1993-10-29|NONE|AIR|ses boost after the bold requests.|
+417|40|1|1|39|36661.56|0.01|0.02|A|F|1994-05-31|1994-05-02|1994-06-06|NONE|SHIP|y regular requests wake along |
+417|70|7|2|18|17461.26|0.00|0.01|R|F|1994-03-29|1994-04-10|1994-04-26|TAKE BACK RETURN|FOB|- final requests sle|
+417|45|2|3|41|38746.64|0.10|0.01|R|F|1994-04-11|1994-03-08|1994-05-06|COLLECT COD|RAIL|tes. regular requests across the |
+417|132|3|4|2|2064.26|0.01|0.03|R|F|1994-02-13|1994-04-19|1994-03-15|DELIVER IN PERSON|SHIP|uriously bol|
+418|19|3|1|31|28489.31|0.00|0.03|N|F|1995-06-05|1995-06-18|1995-06-26|COLLECT COD|FOB|final theodolites. fluffil|
+418|2|5|2|1|902.00|0.04|0.07|N|O|1995-06-23|1995-06-16|1995-07-23|DELIVER IN PERSON|AIR|regular, silent pinto|
+418|35|1|3|3|2805.09|0.04|0.06|N|O|1995-06-29|1995-07-12|1995-07-01|COLLECT COD|AIR|ly furiously regular w|
+419|153|8|1|33|34753.95|0.05|0.02|N|O|1996-11-06|1996-12-25|1996-11-20|TAKE BACK RETURN|TRUCK|y above the bli|
+419|65|2|2|32|30881.92|0.01|0.06|N|O|1996-12-04|1996-12-04|1996-12-24|COLLECT COD|SHIP|blithely regular requests. special pinto|
+419|71|1|3|15|14566.05|0.07|0.04|N|O|1996-12-17|1996-11-28|1996-12-19|TAKE BACK RETURN|REG AIR| sleep final, regular theodolites. fluffi|
+419|9|6|4|15|13635.00|0.01|0.02|N|O|1997-01-09|1996-12-22|1997-01-25|COLLECT COD|FOB|of the careful, thin theodolites. quickly s|
+419|149|2|5|17|17835.38|0.01|0.00|N|O|1997-01-13|1996-12-20|1997-02-01|COLLECT COD|REG AIR|lar dependencies: carefully regu|
+420|101|6|1|5|5005.50|0.04|0.03|N|O|1995-11-04|1996-01-02|1995-11-30|NONE|REG AIR|cajole blit|
+420|162|7|2|22|23367.52|0.05|0.04|N|O|1996-01-25|1995-12-16|1996-02-03|TAKE BACK RETURN|AIR|ly against the blithely re|
+420|48|1|3|45|42661.80|0.09|0.08|N|O|1996-01-14|1996-01-01|1996-01-26|COLLECT COD|FOB| final accounts. furiously express forges|
+420|75|6|4|12|11700.84|0.08|0.08|N|O|1996-02-05|1996-01-03|1996-02-12|TAKE BACK RETURN|REG AIR|c instructions are |
+420|73|2|5|37|36003.59|0.02|0.00|N|O|1995-11-16|1995-12-13|1995-11-19|DELIVER IN PERSON|SHIP|rbits. bold requests along the quickl|
+420|124|7|6|40|40964.80|0.01|0.05|N|O|1995-11-26|1995-12-26|1995-12-20|TAKE BACK RETURN|FOB| after the special|
+420|16|7|7|39|35724.39|0.00|0.08|N|O|1995-12-09|1995-12-16|1995-12-31|DELIVER IN PERSON|REG AIR|s. ironic waters about the car|
+421|134|5|1|1|1034.13|0.02|0.07|R|F|1992-05-29|1992-04-27|1992-06-09|NONE|TRUCK|oldly busy deposit|
+422|152|10|1|25|26303.75|0.10|0.07|N|O|1997-07-01|1997-08-17|1997-07-09|DELIVER IN PERSON|SHIP|carefully bold theodolit|
+422|171|1|2|10|10711.70|0.02|0.03|N|O|1997-06-15|1997-08-04|1997-07-08|TAKE BACK RETURN|AIR|he furiously ironic theodolite|
+422|176|4|3|46|49503.82|0.09|0.00|N|O|1997-06-21|1997-07-14|1997-06-27|DELIVER IN PERSON|RAIL| ideas. qu|
+422|162|7|4|25|26554.00|0.10|0.04|N|O|1997-08-24|1997-07-09|1997-09-22|NONE|FOB|ep along the furiousl|
+423|132|3|1|27|27867.51|0.06|0.03|N|O|1996-08-20|1996-08-01|1996-08-23|TAKE BACK RETURN|SHIP|ccounts. blithely regular pack|
+448|126|7|1|4|4104.48|0.00|0.04|N|O|1995-11-25|1995-10-20|1995-11-26|TAKE BACK RETURN|MAIL|nts thrash quickly among the b|
+448|173|1|2|46|49365.82|0.05|0.00|N|O|1995-08-31|1995-09-30|1995-09-09|COLLECT COD|SHIP| to the fluffily ironic packages.|
+448|27|6|3|35|32445.70|0.10|0.08|N|O|1995-09-27|1995-11-19|1995-10-20|COLLECT COD|REG AIR|ses nag quickly quickly ir|
+448|170|1|4|8|8561.36|0.10|0.00|N|O|1995-11-02|1995-10-16|1995-11-15|COLLECT COD|TRUCK|ounts wake blithely. furiously pending|
+448|138|9|5|23|23876.99|0.02|0.05|N|O|1995-09-26|1995-11-02|1995-10-17|NONE|SHIP|ious, final gifts|
+449|152|7|1|12|12625.80|0.02|0.08|N|O|1995-11-06|1995-08-25|1995-11-18|TAKE BACK RETURN|SHIP|ly. blithely ironic |
+449|109|6|2|4|4036.40|0.10|0.06|N|O|1995-10-27|1995-09-14|1995-11-21|DELIVER IN PERSON|FOB|are fluffily. requests are furiously|
+449|10|1|3|3|2730.03|0.07|0.08|N|O|1995-07-28|1995-09-11|1995-08-01|NONE|RAIL| bold deposits. express theodolites haggle|
+449|158|3|4|22|23279.30|0.07|0.00|N|O|1995-08-17|1995-09-04|1995-09-10|COLLECT COD|FOB|furiously final theodolites eat careful|
+450|162|7|1|42|44610.72|0.03|0.00|N|F|1995-06-07|1995-05-29|1995-06-23|TAKE BACK RETURN|SHIP|y asymptotes. regular depen|
+450|107|8|2|5|5035.50|0.03|0.02|A|F|1995-04-02|1995-05-06|1995-04-13|TAKE BACK RETURN|TRUCK|the pinto bea|
+450|143|6|3|32|33380.48|0.06|0.03|N|O|1995-07-02|1995-04-25|1995-07-30|TAKE BACK RETURN|SHIP| accounts nod fluffily even, pending|
+450|57|9|4|40|38282.00|0.05|0.03|R|F|1995-03-20|1995-05-25|1995-04-14|NONE|RAIL|ve. asymptote|
+450|79|10|5|2|1958.14|0.09|0.00|A|F|1995-03-11|1995-05-21|1995-03-16|COLLECT COD|AIR|y even pinto beans; qui|
+450|153|1|6|33|34753.95|0.08|0.05|R|F|1995-05-18|1995-05-22|1995-05-23|TAKE BACK RETURN|REG AIR|ily carefully final depo|
+451|130|9|1|36|37084.68|0.02|0.06|N|O|1998-06-18|1998-08-14|1998-06-20|TAKE BACK RETURN|AIR|rges can haggle carefully ironic, dogged |
+451|33|4|2|42|39187.26|0.05|0.01|N|O|1998-08-01|1998-08-05|1998-08-30|DELIVER IN PERSON|TRUCK|express excuses. blithely ironic pin|
+451|87|8|3|1|987.08|0.07|0.05|N|O|1998-07-13|1998-07-03|1998-08-04|DELIVER IN PERSON|AIR| carefully ironic packages solve furiously |
+451|77|5|4|28|27357.96|0.04|0.05|N|O|1998-06-16|1998-07-09|1998-06-17|DELIVER IN PERSON|SHIP| theodolites. even cou|
+452|115|6|1|2|2030.22|0.04|0.03|N|O|1997-12-26|1998-01-03|1998-01-12|COLLECT COD|FOB|y express instru|
+453|198|1|1|45|49418.55|0.01|0.00|N|O|1997-06-30|1997-08-20|1997-07-19|COLLECT COD|REG AIR|ifts wake carefully.|
+453|176|4|2|38|40894.46|0.08|0.04|N|O|1997-06-30|1997-07-08|1997-07-16|DELIVER IN PERSON|REG AIR| furiously f|
+453|14|1|3|38|34732.38|0.10|0.01|N|O|1997-08-10|1997-07-24|1997-09-07|NONE|SHIP|sts cajole. furiously un|
+453|96|7|4|45|44824.05|0.10|0.01|N|O|1997-09-18|1997-06-29|1997-10-14|TAKE BACK RETURN|AIR|ironic foxes. slyly pending depos|
+453|26|1|5|32|29632.64|0.04|0.01|N|O|1997-07-15|1997-06-27|1997-07-18|NONE|REG AIR|s. fluffily bold packages cajole. unu|
+453|95|7|6|28|27862.52|0.07|0.07|N|O|1997-08-16|1997-08-12|1997-08-27|NONE|MAIL|final dependencies. slyly special pl|
+454|118|8|1|24|24434.64|0.06|0.01|N|O|1996-04-26|1996-03-23|1996-05-20|NONE|TRUCK|le. deposits after the ideas nag unusual pa|
+455|157|9|1|42|44400.30|0.10|0.02|N|O|1997-01-26|1997-01-10|1997-02-22|DELIVER IN PERSON|REG AIR|around the quickly blit|
+455|28|9|2|44|40832.88|0.05|0.08|N|O|1997-01-17|1997-02-22|1997-02-12|TAKE BACK RETURN|TRUCK| accounts sleep slyly ironic asymptote|
+455|49|2|3|45|42706.80|0.04|0.06|N|O|1996-12-20|1997-01-31|1997-01-07|TAKE BACK RETURN|SHIP|thrash ironically regular packages. qui|
+455|171|9|4|11|11782.87|0.01|0.02|N|O|1997-03-15|1997-02-14|1997-03-26|DELIVER IN PERSON|MAIL|g deposits against the slyly idle foxes u|
+480|53|4|1|22|20967.10|0.04|0.02|A|F|1993-06-16|1993-07-28|1993-07-09|NONE|MAIL|into beans cajole furiously. accounts s|
+481|19|9|1|17|15623.17|0.07|0.05|A|F|1992-10-21|1992-12-09|1992-11-19|DELIVER IN PERSON|MAIL|. quickly final accounts among the |
+481|21|2|2|19|17499.38|0.08|0.01|R|F|1993-01-09|1992-11-27|1993-01-14|TAKE BACK RETURN|AIR|p blithely after t|
+481|186|7|3|42|45619.56|0.08|0.08|A|F|1992-11-27|1992-11-11|1992-12-08|COLLECT COD|RAIL|mptotes are furiously among the iron|
+481|82|3|4|11|10802.88|0.05|0.06|A|F|1993-01-12|1992-11-17|1993-02-05|NONE|FOB|eful attai|
+481|112|9|5|31|31375.41|0.05|0.01|A|F|1993-01-15|1992-12-31|1993-01-21|DELIVER IN PERSON|AIR|usly final packages believe. quick|
+482|138|9|1|32|33220.16|0.00|0.02|N|O|1996-05-22|1996-05-14|1996-05-29|NONE|SHIP|usual deposits affix against |
+482|122|5|2|1|1022.12|0.05|0.08|N|O|1996-05-29|1996-05-20|1996-05-31|COLLECT COD|AIR|es. quickly ironic escapades sleep furious|
+482|62|9|3|31|29823.86|0.04|0.03|N|O|1996-06-01|1996-05-06|1996-06-17|NONE|MAIL| blithe pin|
+482|196|7|4|8|8769.52|0.02|0.05|N|O|1996-04-19|1996-05-05|1996-04-21|NONE|TRUCK|tructions near the final, regular ideas de|
+482|39|10|5|46|43195.38|0.01|0.06|N|O|1996-07-19|1996-06-05|1996-08-10|NONE|MAIL|furiously thin realms. final, fina|
+482|79|10|6|19|18602.33|0.04|0.00|N|O|1996-03-27|1996-04-25|1996-04-15|NONE|FOB|ts hinder carefully silent requests|
+483|33|9|1|8|7464.24|0.00|0.08|N|O|1995-08-22|1995-08-23|1995-09-18|COLLECT COD|RAIL|osits. carefully fin|
+483|80|1|2|23|22541.84|0.04|0.06|N|O|1995-07-20|1995-08-11|1995-08-04|DELIVER IN PERSON|MAIL|requests was quickly against th|
+483|88|9|3|9|8892.72|0.04|0.03|N|O|1995-09-10|1995-09-02|1995-09-13|NONE|AIR| carefully express ins|
+484|31|2|1|49|45620.47|0.10|0.02|N|O|1997-03-06|1997-02-28|1997-03-23|COLLECT COD|TRUCK|ven accounts|
+484|32|8|2|45|41941.35|0.06|0.07|N|O|1997-04-09|1997-03-20|1997-04-19|DELIVER IN PERSON|TRUCK|usly final excuses boost slyly blithe|
+484|184|5|3|50|54209.00|0.06|0.05|N|O|1997-01-24|1997-03-27|1997-02-22|DELIVER IN PERSON|MAIL|uctions wake. final, silent requests haggle|
+484|165|6|4|22|23433.52|0.07|0.03|N|O|1997-04-29|1997-03-26|1997-05-17|TAKE BACK RETURN|SHIP|es are pending instructions. furiously unu|
+484|77|6|5|48|46899.36|0.00|0.05|N|O|1997-03-05|1997-02-08|1997-03-22|TAKE BACK RETURN|MAIL|l, bold packages? even mult|
+484|97|9|6|10|9970.90|0.01|0.08|N|O|1997-04-06|1997-02-14|1997-04-16|COLLECT COD|FOB|x fluffily carefully regular|
+485|150|1|1|50|52507.50|0.01|0.00|N|O|1997-03-28|1997-05-26|1997-04-18|TAKE BACK RETURN|MAIL|iously quick excuses. carefully final f|
+485|28|7|2|40|37120.80|0.08|0.01|N|O|1997-04-29|1997-05-08|1997-04-30|TAKE BACK RETURN|TRUCK|al escapades|
+485|137|3|3|22|22816.86|0.00|0.05|N|O|1997-04-06|1997-04-27|1997-05-01|DELIVER IN PERSON|TRUCK|refully final notornis haggle according |
+486|76|7|1|36|35138.52|0.00|0.01|N|O|1996-06-25|1996-05-06|1996-07-07|COLLECT COD|AIR|deposits around the quickly regular packa|
+486|68|9|2|40|38722.40|0.03|0.08|N|O|1996-05-21|1996-06-06|1996-06-07|COLLECT COD|SHIP|ts nag quickly among the slyl|
+486|136|2|3|26|26939.38|0.04|0.03|N|O|1996-03-16|1996-05-25|1996-03-31|NONE|RAIL|forges along the |
+486|72|1|4|38|36938.66|0.08|0.05|N|O|1996-05-07|1996-04-26|1996-05-26|TAKE BACK RETURN|TRUCK| blithely final pinto |
+486|29|2|5|3|2787.06|0.07|0.05|N|O|1996-07-07|1996-04-20|1996-07-23|DELIVER IN PERSON|RAIL|ccounts ha|
+486|47|4|6|46|43563.84|0.00|0.03|N|O|1996-04-18|1996-05-02|1996-04-20|COLLECT COD|AIR|theodolites eat carefully furious|
+487|92|3|1|47|46628.23|0.06|0.06|R|F|1992-09-30|1992-10-08|1992-10-24|NONE|TRUCK|tions. blithely reg|
+487|83|4|2|2|1966.16|0.02|0.06|R|F|1992-10-19|1992-11-04|1992-11-11|COLLECT COD|TRUCK|oss the unusual pinto beans. reg|
+512|189|10|1|19|20694.42|0.08|0.05|N|O|1995-07-12|1995-07-11|1995-08-04|COLLECT COD|MAIL| sleep. requests alongside of the fluff|
+512|23|2|2|37|34151.74|0.01|0.04|N|O|1995-06-20|1995-07-05|1995-07-16|NONE|RAIL|nic depths cajole? blithely b|
+512|180|1|3|40|43207.20|0.05|0.02|N|O|1995-07-06|1995-07-08|1995-07-08|COLLECT COD|TRUCK|quests are da|
+512|83|4|4|10|9830.80|0.09|0.02|N|O|1995-09-16|1995-07-29|1995-10-07|NONE|AIR|xes. pinto beans cajole carefully; |
+512|65|6|5|6|5790.36|0.03|0.05|R|F|1995-06-10|1995-06-21|1995-06-16|DELIVER IN PERSON|FOB|en ideas haggle |
+512|33|9|6|12|11196.36|0.04|0.00|R|F|1995-05-21|1995-08-03|1995-06-09|COLLECT COD|FOB|old furiously express deposits. specia|
+512|51|9|7|2|1902.10|0.09|0.08|N|O|1995-06-19|1995-08-13|1995-06-24|NONE|TRUCK|e slyly silent accounts serve with|
+513|62|7|1|20|19241.20|0.09|0.07|N|O|1995-07-12|1995-05-31|1995-07-31|NONE|AIR|efully ironic ideas doze slyl|
+513|122|5|2|44|44973.28|0.01|0.01|N|O|1995-07-14|1995-07-14|1995-08-12|NONE|MAIL|kages sleep boldly ironic theodolites. acco|
+514|79|9|1|21|20560.47|0.06|0.02|N|O|1996-06-09|1996-05-15|1996-07-07|DELIVER IN PERSON|RAIL|s sleep quickly blithely|
+514|118|2|2|34|34615.74|0.08|0.02|N|O|1996-04-14|1996-06-03|1996-04-23|COLLECT COD|REG AIR|ily even patterns. bold, silent instruc|
+514|13|7|3|6|5478.06|0.06|0.01|N|O|1996-05-30|1996-06-04|1996-06-28|COLLECT COD|SHIP|as haggle blithely; quickly s|
+514|116|7|4|43|43692.73|0.00|0.08|N|O|1996-06-07|1996-05-14|1996-07-01|TAKE BACK RETURN|FOB|thely regular |
+515|105|8|1|10|10051.00|0.03|0.02|A|F|1993-10-04|1993-11-03|1993-10-08|NONE|FOB|ar deposits th|
+515|148|1|2|38|39829.32|0.10|0.07|A|F|1993-09-19|1993-11-12|1993-10-03|DELIVER IN PERSON|SHIP|ays. furiously express requests haggle furi|
+515|183|4|3|11|11914.98|0.00|0.02|R|F|1993-09-04|1993-10-02|1993-09-05|DELIVER IN PERSON|FOB|ly pending accounts haggle blithel|
+515|109|10|4|34|34309.40|0.09|0.03|R|F|1993-10-03|1993-10-26|1993-10-15|DELIVER IN PERSON|REG AIR|ic dependencie|
+515|131|7|5|32|32996.16|0.01|0.07|R|F|1993-10-10|1993-10-08|1993-11-02|TAKE BACK RETURN|FOB|r sauternes boost. final theodolites wake a|
+515|109|4|6|25|25227.50|0.04|0.08|R|F|1993-11-14|1993-11-07|1993-12-03|DELIVER IN PERSON|MAIL|e packages engag|
+516|25|4|1|11|10175.22|0.01|0.06|N|O|1998-05-02|1998-05-23|1998-05-12|DELIVER IN PERSON|FOB|ongside of the blithely final reque|
+517|45|6|1|28|26461.12|0.03|0.02|N|O|1997-04-30|1997-05-18|1997-05-17|COLLECT COD|MAIL| requests. special, fi|
+517|156|4|2|15|15842.25|0.02|0.00|N|O|1997-04-09|1997-06-26|1997-05-01|NONE|TRUCK| slyly. express requests ar|
+517|41|8|3|9|8469.36|0.04|0.00|N|O|1997-05-03|1997-06-16|1997-05-24|COLLECT COD|SHIP| slyly stealthily express instructions. |
+517|133|4|4|11|11364.43|0.06|0.02|N|O|1997-06-20|1997-06-01|1997-06-27|NONE|REG AIR|ly throughout the fu|
+517|24|3|5|23|21252.46|0.00|0.01|N|O|1997-04-19|1997-05-07|1997-05-12|COLLECT COD|RAIL| kindle. furiously bold requests mus|
+518|165|6|1|30|31954.80|0.07|0.05|N|O|1998-02-18|1998-03-27|1998-03-16|COLLECT COD|TRUCK|slyly by the packages. carefull|
+518|84|5|2|23|22633.84|0.05|0.07|N|O|1998-02-20|1998-05-05|1998-03-11|COLLECT COD|TRUCK| special requests. fluffily ironic re|
+518|134|5|3|12|12409.56|0.01|0.06|N|O|1998-03-08|1998-03-31|1998-04-06|NONE|AIR| packages thrash slyly|
+518|122|3|4|46|47017.52|0.07|0.02|N|O|1998-04-07|1998-04-17|1998-04-29|NONE|MAIL|. blithely even ideas cajole furiously. b|
+518|71|2|5|16|15537.12|0.01|0.01|N|O|1998-03-15|1998-03-24|1998-04-08|NONE|MAIL|use quickly expre|
+518|197|10|6|39|42790.41|0.09|0.08|N|O|1998-02-26|1998-03-17|1998-03-21|DELIVER IN PERSON|FOB| the bold, special deposits are carefully |
+518|186|7|7|48|52136.64|0.03|0.07|N|O|1998-03-06|1998-04-22|1998-03-14|NONE|FOB| slyly final platelets; quickly even deposi|
+519|159|4|1|1|1059.15|0.07|0.07|N|O|1997-12-01|1998-01-26|1997-12-23|COLLECT COD|REG AIR|bold requests believe furiou|
+519|3|4|2|38|34314.00|0.05|0.08|N|O|1998-02-19|1997-12-15|1998-03-19|DELIVER IN PERSON|FOB|gular excuses detect quickly furiously |
+519|106|1|3|19|19115.90|0.00|0.02|N|O|1998-01-09|1998-01-03|1998-02-06|COLLECT COD|AIR|asymptotes. p|
+519|47|6|4|27|25570.08|0.08|0.06|N|O|1997-11-20|1997-12-06|1997-12-16|DELIVER IN PERSON|REG AIR|le. even, final dependencies|
+519|10|5|5|13|11830.13|0.06|0.08|N|O|1998-02-06|1997-12-02|1998-03-03|TAKE BACK RETURN|TRUCK|c accounts wake along the ironic so|
+519|151|6|6|3|3153.45|0.04|0.00|N|O|1998-02-01|1998-01-25|1998-02-27|TAKE BACK RETURN|FOB|erve blithely blithely ironic asymp|
+544|139|10|1|47|48839.11|0.08|0.06|R|F|1993-03-14|1993-03-27|1993-03-27|COLLECT COD|SHIP|ecial pains. deposits grow foxes. |
+545|170|1|1|4|4280.68|0.02|0.00|N|O|1996-02-23|1995-12-16|1996-03-21|DELIVER IN PERSON|FOB|, ironic grouches cajole over|
+545|171|10|2|18|19281.06|0.00|0.00|N|O|1996-02-21|1996-01-17|1996-02-26|NONE|RAIL|al, final packages affix. even a|
+546|85|6|1|16|15761.28|0.08|0.02|N|O|1997-02-04|1996-12-30|1997-02-25|DELIVER IN PERSON|TRUCK|de of the orbits. sometimes regula|
+547|71|10|1|44|42727.08|0.08|0.08|N|O|1996-10-18|1996-08-17|1996-10-27|TAKE BACK RETURN|FOB|thely express dependencies. qu|
+547|137|8|2|48|49782.24|0.01|0.04|N|O|1996-10-21|1996-08-04|1996-11-20|COLLECT COD|SHIP|thely specia|
+547|182|3|3|3|3246.54|0.05|0.02|N|O|1996-09-04|1996-08-01|1996-09-21|COLLECT COD|SHIP|pinto beans. ironi|
+548|197|8|1|2|2194.38|0.06|0.05|A|F|1994-11-26|1994-11-06|1994-12-06|COLLECT COD|MAIL|ests haggle quickly eve|
+548|5|6|2|6|5430.00|0.00|0.08|A|F|1995-01-18|1994-12-08|1995-02-10|NONE|TRUCK|sits wake furiously regular|
+548|1|8|3|21|18921.00|0.03|0.08|A|F|1995-01-13|1994-12-18|1995-01-25|NONE|AIR|ideas. special accounts above the furiou|
+548|57|9|4|21|20098.05|0.08|0.03|A|F|1994-10-27|1994-12-04|1994-11-21|DELIVER IN PERSON|AIR| engage quickly. regular theo|
+548|93|7|5|19|18868.71|0.00|0.02|A|F|1994-09-24|1994-11-24|1994-10-01|DELIVER IN PERSON|MAIL|courts boost care|
+548|153|8|6|32|33700.80|0.06|0.04|A|F|1994-12-16|1994-11-20|1994-12-29|NONE|REG AIR|c instruction|
+549|196|9|1|18|19731.42|0.07|0.04|R|F|1992-10-19|1992-08-12|1992-11-13|COLLECT COD|REG AIR|furiously according to the ironic, regular |
+549|189|10|2|38|41388.84|0.07|0.05|A|F|1992-08-17|1992-08-28|1992-09-05|COLLECT COD|RAIL|the regular, furious excuses. carefu|
+549|66|7|3|36|34778.16|0.08|0.04|R|F|1992-09-11|1992-10-11|1992-09-12|DELIVER IN PERSON|AIR|ts against the ironic, even theodolites eng|
+549|21|4|4|18|16578.36|0.09|0.01|A|F|1992-07-31|1992-09-11|1992-08-08|NONE|RAIL|ely regular accounts above the |
+549|24|7|5|38|35112.76|0.06|0.02|R|F|1992-08-23|1992-08-12|1992-08-25|COLLECT COD|REG AIR|eposits. carefully regular depos|
+550|191|3|1|31|33826.89|0.04|0.02|N|O|1995-10-24|1995-09-27|1995-11-04|COLLECT COD|AIR|thely silent packages. unusual|
+551|24|9|1|8|7392.16|0.08|0.02|N|O|1995-07-29|1995-07-18|1995-08-02|NONE|REG AIR| wake quickly slyly pending platel|
+551|159|4|2|20|21183.00|0.00|0.07|N|O|1995-09-18|1995-08-25|1995-10-11|COLLECT COD|TRUCK|r ideas. final, even ideas hinder alongside|
+551|162|9|3|16|16994.56|0.07|0.06|N|O|1995-07-29|1995-08-19|1995-08-10|COLLECT COD|MAIL|y along the carefully ex|
+576|87|8|1|2|1974.16|0.07|0.01|N|O|1997-05-15|1997-06-30|1997-05-28|NONE|RAIL|ccounts along the ac|
+576|34|5|2|6|5604.18|0.06|0.05|N|O|1997-05-15|1997-07-26|1997-06-03|DELIVER IN PERSON|TRUCK|al deposits. slyly even sauternes a|
+576|37|3|3|6|5622.18|0.08|0.07|N|O|1997-08-28|1997-06-16|1997-09-25|DELIVER IN PERSON|FOB|ts. ironic multipliers |
+576|138|9|4|5|5190.65|0.03|0.07|N|O|1997-06-11|1997-06-17|1997-07-05|NONE|REG AIR|l foxes boost slyly. accounts af|
+577|26|5|1|25|23150.50|0.06|0.01|A|F|1995-04-09|1995-02-20|1995-05-09|TAKE BACK RETURN|AIR|ve slyly of the frets. careful|
+577|64|1|2|14|13496.84|0.08|0.03|R|F|1995-03-19|1995-02-25|1995-04-09|DELIVER IN PERSON|RAIL|l accounts wake deposits. ironic packa|
+578|156|7|1|40|42246.00|0.02|0.08|N|O|1997-02-10|1997-03-18|1997-02-11|NONE|SHIP|usly even platel|
+578|188|9|2|23|25028.14|0.05|0.08|N|O|1997-03-06|1997-03-03|1997-03-20|TAKE BACK RETURN|FOB|nstructions. ironic deposits|
+579|151|6|1|9|9460.35|0.00|0.05|N|O|1998-06-20|1998-04-28|1998-07-19|DELIVER IN PERSON|RAIL|e ironic, express deposits are furiously|
+579|33|4|2|39|36388.17|0.02|0.01|N|O|1998-06-21|1998-06-03|1998-06-26|COLLECT COD|REG AIR|ncies. furiously final r|
+579|60|5|3|6|5760.36|0.03|0.00|N|O|1998-04-24|1998-05-03|1998-05-08|TAKE BACK RETURN|TRUCK|ickly final requests-- bold accou|
+579|7|10|4|41|37187.00|0.04|0.05|N|O|1998-05-28|1998-05-01|1998-06-04|COLLECT COD|REG AIR|bold, express requests sublate slyly. blith|
+579|13|7|5|28|25564.28|0.00|0.03|N|O|1998-07-10|1998-05-24|1998-07-19|NONE|RAIL|ic ideas until th|
+579|167|6|6|5|5335.80|0.05|0.08|N|O|1998-05-02|1998-04-25|1998-05-05|COLLECT COD|REG AIR|refully silent ideas cajole furious|
+580|85|6|1|33|32507.64|0.03|0.05|N|O|1997-10-11|1997-09-19|1997-10-16|TAKE BACK RETURN|FOB|y express theodolites cajole carefully |
+580|174|5|2|31|33299.27|0.04|0.08|N|O|1997-10-04|1997-09-08|1997-10-15|COLLECT COD|FOB|ose alongside of the sl|
+580|185|6|3|19|20618.42|0.04|0.04|N|O|1997-07-23|1997-09-21|1997-08-15|NONE|FOB|mong the special packag|
+581|64|1|1|41|39526.46|0.09|0.07|N|O|1997-05-26|1997-04-06|1997-06-10|TAKE BACK RETURN|MAIL|nts. quickly|
+581|93|5|2|14|13903.26|0.06|0.08|N|O|1997-05-17|1997-04-14|1997-06-08|NONE|MAIL|. deposits s|
+581|101|6|3|49|49053.90|0.10|0.02|N|O|1997-02-27|1997-04-24|1997-03-10|TAKE BACK RETURN|MAIL|. slyly regular pinto beans acr|
+581|75|4|4|30|29252.10|0.10|0.08|N|O|1997-06-19|1997-05-21|1997-06-22|TAKE BACK RETURN|TRUCK| regular ideas grow furio|
+582|57|9|1|7|6699.35|0.07|0.00|N|O|1997-11-16|1997-11-29|1997-12-10|TAKE BACK RETURN|FOB|ithely unusual t|
+582|51|2|2|49|46601.45|0.05|0.03|N|O|1997-12-17|1998-01-12|1997-12-31|COLLECT COD|REG AIR|nts according to the furiously regular pin|
+582|141|4|3|42|43727.88|0.07|0.00|N|O|1997-11-15|1997-12-21|1997-12-03|COLLECT COD|SHIP|iously beside the silent de|
+582|168|9|4|36|38453.76|0.06|0.01|N|O|1997-12-09|1997-11-27|1997-12-26|TAKE BACK RETURN|SHIP|lar requests. quickly |
+583|145|6|1|1|1045.14|0.07|0.07|N|O|1997-06-17|1997-04-29|1997-06-28|NONE|TRUCK| regular, regular ideas. even, bra|
+583|120|4|2|47|47945.64|0.10|0.06|N|O|1997-07-14|1997-05-12|1997-08-11|DELIVER IN PERSON|AIR|nts are fluffily. furiously even re|
+583|130|5|3|34|35024.42|0.01|0.02|N|O|1997-05-11|1997-04-24|1997-06-03|DELIVER IN PERSON|MAIL|express req|
+583|142|1|4|33|34390.62|0.10|0.01|N|O|1997-05-28|1997-04-25|1997-06-24|NONE|AIR|kages cajole slyly across the|
+583|189|10|5|13|14159.34|0.04|0.06|N|O|1997-06-23|1997-05-29|1997-07-08|COLLECT COD|TRUCK|y sly theodolites. ironi|
+608|154|6|1|19|20028.85|0.08|0.06|N|O|1996-04-19|1996-05-02|1996-05-03|DELIVER IN PERSON|RAIL|ideas. the|
+608|198|1|2|40|43927.60|0.03|0.01|N|O|1996-05-21|1996-04-11|1996-06-02|NONE|AIR| alongside of the regular tithes. sly|
+609|66|5|1|21|20287.26|0.01|0.05|R|F|1994-08-24|1994-08-23|1994-08-27|DELIVER IN PERSON|FOB|de of the special warthogs. excu|
+610|111|8|1|49|49544.39|0.10|0.07|N|O|1995-08-29|1995-10-26|1995-09-12|TAKE BACK RETURN|SHIP|ular instruc|
+610|68|3|2|11|10648.66|0.07|0.08|N|O|1995-10-31|1995-10-25|1995-11-18|NONE|MAIL|blithely final |
+610|118|9|3|26|26470.86|0.09|0.04|N|O|1995-11-22|1995-09-09|1995-12-04|TAKE BACK RETURN|AIR|cross the furiously even theodolites sl|
+610|186|7|4|17|18465.06|0.03|0.03|N|O|1995-11-01|1995-10-30|1995-11-04|COLLECT COD|FOB|p quickly instead of the slyly pending foxe|
+610|146|7|5|39|40799.46|0.08|0.05|N|O|1995-10-30|1995-10-21|1995-11-11|TAKE BACK RETURN|REG AIR|counts. ironic warhorses are |
+610|95|7|6|5|4975.45|0.00|0.07|N|O|1995-08-11|1995-10-22|1995-08-26|TAKE BACK RETURN|FOB|n pinto beans. iro|
+610|190|1|7|27|29435.13|0.06|0.03|N|O|1995-09-02|1995-09-19|1995-09-15|NONE|REG AIR| ironic pinto beans haggle. blithe|
+611|17|7|1|39|35763.39|0.05|0.06|R|F|1993-05-06|1993-04-09|1993-05-22|TAKE BACK RETURN|SHIP|nto beans |
+611|81|2|2|1|981.08|0.08|0.07|R|F|1993-05-17|1993-02-26|1993-06-15|DELIVER IN PERSON|MAIL|ts. pending platelets aff|
+611|120|4|3|39|39784.68|0.09|0.02|A|F|1993-03-10|1993-03-10|1993-03-17|TAKE BACK RETURN|TRUCK|the evenly bold requests. furious|
+612|185|6|1|5|5425.90|0.07|0.00|R|F|1992-11-08|1992-11-20|1992-12-03|TAKE BACK RETURN|RAIL|structions. q|
+612|195|7|2|28|30665.32|0.07|0.06|R|F|1993-01-02|1992-12-11|1993-01-30|DELIVER IN PERSON|TRUCK|regular instructions affix bl|
+612|67|4|3|49|47385.94|0.00|0.08|A|F|1993-01-08|1992-11-25|1993-01-17|TAKE BACK RETURN|REG AIR|theodolite|
+612|39|5|4|28|26292.84|0.05|0.00|A|F|1992-11-12|1992-12-05|1992-12-02|TAKE BACK RETURN|REG AIR|lyly regular asym|
+612|88|9|5|1|988.08|0.08|0.04|R|F|1992-12-18|1992-12-13|1992-12-20|TAKE BACK RETURN|FOB| requests.|
+612|189|10|6|33|35942.94|0.10|0.03|R|F|1992-11-30|1992-12-01|1992-12-12|COLLECT COD|MAIL|bove the blithely even ideas. careful|
+613|91|5|1|17|16848.53|0.06|0.06|N|O|1995-09-23|1995-08-04|1995-10-15|NONE|SHIP|ar dependencie|
+613|79|7|2|6|5874.42|0.05|0.05|N|O|1995-08-05|1995-08-09|1995-08-08|TAKE BACK RETURN|MAIL|y ironic deposits eat |
+613|186|7|3|3|3258.54|0.03|0.01|N|O|1995-09-27|1995-09-11|1995-10-05|NONE|TRUCK|ccounts cajole. |
+613|159|10|4|7|7414.05|0.02|0.04|N|O|1995-09-07|1995-08-02|1995-09-16|DELIVER IN PERSON|MAIL|ously blithely final pinto beans. regula|
+614|195|8|1|21|22998.99|0.00|0.03|R|F|1993-03-29|1993-01-06|1993-04-16|TAKE BACK RETURN|TRUCK|arefully. slyly express packag|
+614|187|8|2|48|52184.64|0.07|0.07|A|F|1993-03-09|1993-01-19|1993-03-19|DELIVER IN PERSON|SHIP|riously special excuses haggle along the|
+614|167|2|3|43|45887.88|0.05|0.00|A|F|1993-03-07|1993-02-22|1993-03-18|DELIVER IN PERSON|SHIP| express accounts wake. slyly ironic ins|
+614|147|6|4|14|14659.96|0.04|0.06|A|F|1992-12-03|1993-02-14|1992-12-27|DELIVER IN PERSON|SHIP|ular packages haggle about the pack|
+614|196|8|5|30|32885.70|0.08|0.07|R|F|1993-01-16|1993-02-08|1993-02-12|TAKE BACK RETURN|FOB|tructions are f|
+614|137|8|6|48|49782.24|0.04|0.08|A|F|1992-12-14|1993-01-22|1993-01-11|NONE|TRUCK| regular platelets cajole quickly eve|
+615|105|6|1|36|36183.60|0.10|0.01|A|F|1992-06-01|1992-07-14|1992-06-27|NONE|FOB| packages. carefully final pinto bea|
+640|93|7|1|49|48661.41|0.09|0.02|R|F|1993-03-27|1993-04-17|1993-04-15|NONE|RAIL|s haggle slyly|
+640|1|4|2|40|36040.00|0.09|0.05|A|F|1993-05-11|1993-04-11|1993-05-15|COLLECT COD|TRUCK|oach according to the bol|
+640|180|8|3|22|23763.96|0.05|0.07|A|F|1993-05-07|1993-04-14|1993-05-21|TAKE BACK RETURN|TRUCK|osits across the slyly regular theodo|
+640|32|3|4|45|41941.35|0.07|0.07|R|F|1993-04-15|1993-04-23|1993-04-21|DELIVER IN PERSON|REG AIR|ong the qui|
+641|126|9|1|18|18470.16|0.01|0.08|R|F|1993-10-17|1993-10-11|1993-10-29|DELIVER IN PERSON|AIR|p blithely bold packages. quick|
+641|100|2|2|1|1000.10|0.09|0.02|R|F|1993-12-03|1993-10-28|1993-12-26|TAKE BACK RETURN|RAIL| nag across the regular foxes.|
+641|95|7|3|40|39803.60|0.05|0.06|R|F|1993-11-22|1993-10-20|1993-12-11|DELIVER IN PERSON|REG AIR|lets. furiously regular requests cajo|
+641|71|10|4|25|24276.75|0.03|0.02|A|F|1993-12-04|1993-11-18|1993-12-18|TAKE BACK RETURN|FOB|d, regular d|
+641|4|9|5|41|37064.00|0.07|0.04|R|F|1993-11-29|1993-10-27|1993-12-04|TAKE BACK RETURN|FOB| asymptotes are quickly. bol|
+642|54|5|1|26|24805.30|0.10|0.03|A|F|1994-04-16|1994-02-01|1994-04-27|COLLECT COD|REG AIR|quests according to the unu|
+643|13|3|1|28|25564.28|0.00|0.08|A|F|1995-04-13|1995-05-12|1995-04-14|TAKE BACK RETURN|TRUCK|ly regular requests nag sly|
+643|51|2|2|48|45650.40|0.01|0.02|N|O|1995-07-10|1995-06-07|1995-08-01|NONE|FOB|ly ironic accounts|
+643|163|2|3|23|24452.68|0.05|0.03|N|O|1995-07-09|1995-05-18|1995-07-31|COLLECT COD|RAIL|sits are carefully according to the e|
+643|45|4|4|39|36856.56|0.08|0.04|A|F|1995-06-08|1995-06-16|1995-06-13|COLLECT COD|RAIL| the pains. carefully s|
+643|190|1|5|47|51238.93|0.10|0.03|R|F|1995-04-05|1995-06-14|1995-04-26|DELIVER IN PERSON|RAIL|y against |
+644|134|10|1|46|47569.98|0.02|0.01|A|F|1992-05-20|1992-06-14|1992-06-14|DELIVER IN PERSON|RAIL| special requests was sometimes expre|
+644|130|3|2|11|11331.43|0.05|0.02|A|F|1992-08-20|1992-07-21|1992-09-11|TAKE BACK RETURN|TRUCK|ealthy pinto beans use carefu|
+644|101|6|3|44|44048.40|0.04|0.04|R|F|1992-08-17|1992-07-26|1992-08-20|COLLECT COD|REG AIR|iously ironic pinto beans. bold packa|
+644|80|8|4|7|6860.56|0.01|0.02|A|F|1992-05-18|1992-07-01|1992-06-07|COLLECT COD|RAIL| regular requests are blithely. slyly|
+644|50|1|5|23|21851.15|0.02|0.04|R|F|1992-07-31|1992-07-28|1992-08-13|DELIVER IN PERSON|TRUCK|uctions nag quickly alongside of t|
+644|85|6|6|33|32507.64|0.00|0.07|R|F|1992-08-26|1992-07-27|1992-08-28|NONE|AIR|ages sleep. bold, bo|
+644|51|9|7|38|36139.90|0.08|0.06|R|F|1992-05-17|1992-07-10|1992-06-06|TAKE BACK RETURN|MAIL| packages. blithely slow accounts nag quic|
+645|160|8|1|33|34985.28|0.01|0.02|A|F|1994-12-09|1995-02-21|1995-01-03|NONE|TRUCK|heodolites b|
+645|170|1|2|47|50297.99|0.07|0.05|R|F|1995-02-16|1995-02-15|1995-02-25|COLLECT COD|TRUCK|hely regular instructions alon|
+645|70|7|3|46|44623.22|0.10|0.01|A|F|1995-01-04|1995-02-21|1995-01-21|COLLECT COD|REG AIR| regular dependencies across the speci|
+645|96|9|4|49|48808.41|0.05|0.03|R|F|1995-01-24|1995-01-06|1995-02-17|NONE|TRUCK|y. slyly iron|
+645|5|8|5|43|38915.00|0.06|0.02|A|F|1995-02-12|1995-02-27|1995-03-06|TAKE BACK RETURN|REG AIR| furiously accounts. slyly|
+645|34|5|6|18|16812.54|0.10|0.08|A|F|1995-03-02|1995-02-08|1995-03-03|COLLECT COD|RAIL|ep. slyly even |
+645|28|9|7|9|8352.18|0.03|0.03|A|F|1994-12-25|1995-01-04|1995-01-15|COLLECT COD|REG AIR|special deposits. regular, final th|
+646|109|6|1|31|31282.10|0.00|0.05|R|F|1994-12-17|1995-02-16|1995-01-04|COLLECT COD|MAIL|ag furiousl|
+646|127|8|2|1|1027.12|0.07|0.01|A|F|1994-12-05|1995-01-07|1994-12-31|TAKE BACK RETURN|MAIL|t blithely regular deposits. quic|
+646|30|9|3|24|22320.72|0.06|0.02|A|F|1995-02-20|1994-12-30|1995-03-16|TAKE BACK RETURN|TRUCK|regular accounts haggle dog|
+646|99|2|4|34|33969.06|0.01|0.00|R|F|1994-12-28|1994-12-27|1994-12-31|COLLECT COD|SHIP|slow accounts. fluffily idle instructions|
+646|90|1|5|17|16831.53|0.04|0.01|A|F|1994-12-31|1994-12-26|1995-01-01|DELIVER IN PERSON|REG AIR|inal packages haggle carefully |
+646|115|2|6|40|40604.40|0.10|0.01|R|F|1995-01-01|1995-01-13|1995-01-11|COLLECT COD|TRUCK|ronic packages sleep across th|
+647|17|1|1|41|37597.41|0.08|0.08|N|O|1997-11-19|1997-09-24|1997-12-15|COLLECT COD|REG AIR|r instructions. quickly unusu|
+647|113|10|2|5|5065.55|0.10|0.00|N|O|1997-09-25|1997-09-22|1997-10-25|TAKE BACK RETURN|AIR|ly express packages haggle caref|
+647|153|8|3|15|15797.25|0.08|0.00|N|O|1997-09-23|1997-10-09|1997-10-21|NONE|MAIL|ve the even, bold foxes sleep |
+672|173|4|1|41|43999.97|0.06|0.06|R|F|1994-06-20|1994-07-03|1994-06-22|COLLECT COD|REG AIR| dependencies in|
+672|190|1|2|9|9811.71|0.03|0.04|R|F|1994-06-25|1994-06-06|1994-07-19|TAKE BACK RETURN|TRUCK|haggle carefully carefully reg|
+672|143|2|3|35|36509.90|0.02|0.01|R|F|1994-07-13|1994-06-04|1994-07-14|COLLECT COD|RAIL| dependencies haggle quickly. theo|
+673|71|10|1|22|21363.54|0.03|0.02|R|F|1994-03-15|1994-04-27|1994-03-29|TAKE BACK RETURN|TRUCK| the regular, even requests. carefully fin|
+674|102|5|1|23|23048.30|0.06|0.07|A|F|1992-10-25|1992-10-15|1992-11-03|COLLECT COD|SHIP|ve the quickly even deposits. blithe|
+674|59|4|2|4|3836.20|0.02|0.07|R|F|1992-10-05|1992-11-22|1992-10-22|NONE|RAIL|ly express pinto beans sleep car|
+675|157|9|1|1|1057.15|0.04|0.08|N|O|1997-11-27|1997-09-30|1997-12-12|DELIVER IN PERSON|REG AIR|ide of the slyly regular packages. unus|
+675|137|3|2|35|36299.55|0.08|0.07|N|O|1997-08-19|1997-10-16|1997-09-17|DELIVER IN PERSON|REG AIR|s. furiously expre|
+675|176|6|3|34|36589.78|0.10|0.04|N|O|1997-11-17|1997-10-07|1997-11-27|NONE|FOB|y final accounts unwind around the |
+675|100|4|4|15|15001.50|0.09|0.05|N|O|1997-10-18|1997-09-28|1997-11-13|COLLECT COD|TRUCK|posits after the furio|
+675|5|8|5|46|41630.00|0.09|0.05|N|O|1997-09-18|1997-10-14|1997-10-01|DELIVER IN PERSON|AIR| deposits along the express foxes |
+676|51|3|1|9|8559.45|0.09|0.02|N|O|1997-04-03|1997-02-02|1997-04-08|COLLECT COD|REG AIR|aintain sl|
+676|78|6|2|20|19561.40|0.07|0.07|N|O|1997-02-02|1997-02-01|1997-02-11|NONE|REG AIR|riously around the blithely |
+676|163|4|3|35|37210.60|0.05|0.01|N|O|1996-12-30|1997-01-13|1997-01-19|DELIVER IN PERSON|RAIL|into beans. blithe|
+676|73|1|4|24|23353.68|0.01|0.06|N|O|1997-02-05|1997-01-16|1997-03-07|TAKE BACK RETURN|TRUCK|ress, regular dep|
+676|166|1|5|31|33050.96|0.01|0.06|N|O|1997-02-06|1997-02-28|1997-03-08|COLLECT COD|TRUCK|ial deposits cajo|
+676|76|7|6|33|32210.31|0.09|0.05|N|O|1997-03-02|1997-02-22|1997-03-19|TAKE BACK RETURN|TRUCK|as wake slyly furiously close pinto b|
+676|143|6|7|11|11474.54|0.07|0.02|N|O|1997-03-09|1997-03-06|1997-03-31|TAKE BACK RETURN|MAIL|he final acco|
+677|59|7|1|32|30689.60|0.04|0.08|R|F|1994-01-06|1994-01-31|1994-02-02|NONE|RAIL|slyly final|
+677|168|9|2|39|41658.24|0.00|0.07|R|F|1993-12-19|1994-02-11|1994-01-05|TAKE BACK RETURN|SHIP|ges. furiously regular packages use |
+677|24|5|3|46|42504.92|0.01|0.02|R|F|1993-12-02|1994-02-12|1993-12-06|COLLECT COD|RAIL|ng theodolites. furiously unusual theodo|
+677|148|7|4|1|1048.14|0.06|0.05|R|F|1993-12-01|1994-01-14|1993-12-26|DELIVER IN PERSON|MAIL|ly. regular |
+677|150|9|5|25|26253.75|0.00|0.05|A|F|1994-03-12|1994-02-02|1994-03-28|DELIVER IN PERSON|AIR| packages integrate blithely|
+678|146|7|1|20|20922.80|0.05|0.08|R|F|1993-06-21|1993-04-07|1993-07-10|TAKE BACK RETURN|MAIL|furiously express excuses. foxes eat fu|
+678|37|3|2|22|20614.66|0.01|0.02|A|F|1993-05-10|1993-04-29|1993-06-08|NONE|REG AIR|de of the carefully even requests. bl|
+678|143|10|3|16|16690.24|0.06|0.02|R|F|1993-03-20|1993-04-13|1993-04-16|DELIVER IN PERSON|REG AIR|equests cajole around the carefully regular|
+678|199|10|4|48|52761.12|0.08|0.08|R|F|1993-02-28|1993-04-04|1993-03-24|NONE|REG AIR|ithely. slyly express foxes|
+678|98|9|5|16|15969.44|0.06|0.04|R|F|1993-03-09|1993-04-18|1993-04-07|NONE|AIR| about the |
+678|43|4|6|11|10373.44|0.09|0.00|R|F|1993-04-28|1993-05-16|1993-05-11|COLLECT COD|TRUCK|ess deposits dazzle f|
+679|192|3|1|9|9829.71|0.09|0.00|N|O|1995-12-20|1996-01-27|1996-01-07|COLLECT COD|REG AIR|leep slyly. entici|
+704|190|1|1|40|43607.60|0.05|0.05|N|O|1997-01-30|1997-01-10|1997-02-20|COLLECT COD|AIR|ggle quickly. r|
+704|4|5|2|14|12656.00|0.07|0.08|N|O|1997-02-02|1996-12-26|1997-02-19|DELIVER IN PERSON|REG AIR|ve the quickly final forges. furiously p|
+705|189|10|1|46|50102.28|0.05|0.06|N|O|1997-04-18|1997-05-06|1997-05-05|DELIVER IN PERSON|SHIP|ss deposits. ironic packa|
+705|117|7|2|35|35598.85|0.10|0.04|N|O|1997-03-25|1997-03-20|1997-04-23|TAKE BACK RETURN|FOB|carefully ironic accounts|
+706|197|9|1|23|25235.37|0.05|0.00|N|O|1995-12-06|1995-12-02|1995-12-16|COLLECT COD|SHIP|ckey players. requests above the|
+707|155|6|1|34|35875.10|0.01|0.02|R|F|1994-12-08|1995-01-15|1995-01-02|NONE|RAIL| dependencies|
+707|43|10|2|22|20746.88|0.00|0.06|A|F|1995-01-12|1994-12-28|1995-01-16|DELIVER IN PERSON|REG AIR| kindle ironically|
+708|124|7|1|3|3072.36|0.05|0.02|N|O|1998-10-09|1998-09-22|1998-11-07|COLLECT COD|FOB|e slyly pending foxes. |
+708|180|1|2|19|20523.42|0.06|0.00|N|O|1998-10-28|1998-09-23|1998-11-25|COLLECT COD|SHIP| requests. even, thin ideas|
+708|122|5|3|33|33729.96|0.09|0.06|N|O|1998-09-10|1998-09-20|1998-09-22|COLLECT COD|RAIL|s boost carefully ruthless theodolites. f|
+708|56|7|4|5|4780.25|0.07|0.07|N|O|1998-07-22|1998-08-15|1998-07-28|TAKE BACK RETURN|REG AIR|c pinto beans nag after the account|
+708|143|2|5|36|37553.04|0.08|0.01|N|O|1998-07-16|1998-09-04|1998-08-11|NONE|SHIP|ests. even, regular hockey p|
+708|23|6|6|7|6461.14|0.10|0.03|N|O|1998-08-16|1998-08-15|1998-09-10|COLLECT COD|REG AIR|lly express ac|
+709|87|8|1|7|6909.56|0.00|0.00|N|O|1998-06-14|1998-06-08|1998-06-18|TAKE BACK RETURN|RAIL| special orbits cajole |
+709|198|10|2|15|16472.85|0.08|0.00|N|O|1998-07-10|1998-06-26|1998-08-09|NONE|RAIL|ily regular deposits. sauternes was accor|
+709|169|8|3|10|10691.60|0.01|0.02|N|O|1998-06-04|1998-06-30|1998-06-11|NONE|REG AIR|ts cajole boldly |
+709|108|9|4|40|40324.00|0.10|0.08|N|O|1998-08-12|1998-06-20|1998-08-20|DELIVER IN PERSON|RAIL|ggle fluffily carefully ironic|
+710|163|8|1|47|49968.52|0.06|0.08|A|F|1993-01-18|1993-03-24|1993-01-24|TAKE BACK RETURN|MAIL|usual ideas into th|
+710|193|4|2|38|41541.22|0.07|0.02|R|F|1993-04-18|1993-03-12|1993-05-15|COLLECT COD|FOB|sts boost fluffily aft|
+710|139|5|3|7|7273.91|0.04|0.06|R|F|1993-01-20|1993-03-28|1993-02-15|TAKE BACK RETURN|REG AIR|xpress, special ideas. bl|
+710|90|1|4|25|24752.25|0.00|0.05|R|F|1993-03-31|1993-02-05|1993-04-22|COLLECT COD|FOB|eas detect do|
+710|186|7|5|12|13034.16|0.01|0.02|A|F|1993-02-18|1993-02-27|1993-03-07|DELIVER IN PERSON|MAIL|ions. slyly express theodolites al|
+710|114|4|6|21|21296.31|0.04|0.06|R|F|1993-03-22|1993-03-05|1993-03-27|DELIVER IN PERSON|SHIP|es. furiously p|
+710|160|2|7|46|48767.36|0.03|0.07|R|F|1993-04-16|1993-03-27|1993-05-05|COLLECT COD|MAIL|ges use; blithely pending excuses inte|
+711|146|3|1|2|2092.28|0.10|0.04|R|F|1993-12-01|1993-12-09|1993-12-16|DELIVER IN PERSON|REG AIR|ely across t|
+711|103|8|2|27|27083.70|0.00|0.08|A|F|1993-10-02|1993-10-26|1993-10-08|DELIVER IN PERSON|MAIL|slyly. ironic asy|
+711|128|7|3|46|47293.52|0.10|0.00|R|F|1993-12-26|1993-11-19|1994-01-21|TAKE BACK RETURN|MAIL|deposits. permanen|
+711|128|9|4|20|20562.40|0.09|0.00|A|F|1994-01-17|1993-11-10|1994-01-31|DELIVER IN PERSON|TRUCK|kly regular acco|
+736|158|9|1|46|48674.90|0.05|0.01|N|O|1998-07-16|1998-09-01|1998-08-09|NONE|AIR|uctions cajole|
+736|80|1|2|23|22541.84|0.02|0.05|N|O|1998-10-08|1998-08-27|1998-10-19|TAKE BACK RETURN|AIR|k accounts are carefully|
+736|57|9|3|13|12441.65|0.00|0.03|N|O|1998-08-16|1998-07-26|1998-08-19|DELIVER IN PERSON|FOB|st furiously among the |
+736|98|2|4|14|13973.26|0.06|0.04|N|O|1998-10-04|1998-08-14|1998-10-16|COLLECT COD|REG AIR|nstructions.|
+736|169|6|5|32|34213.12|0.04|0.03|N|O|1998-07-30|1998-08-22|1998-08-12|DELIVER IN PERSON|RAIL|iously final accoun|
+737|182|3|1|12|12986.16|0.01|0.01|R|F|1992-04-28|1992-06-30|1992-05-08|COLLECT COD|RAIL|posits after the slyly bold du|
+738|198|1|1|34|37338.46|0.00|0.06|R|F|1993-06-09|1993-04-15|1993-07-09|TAKE BACK RETURN|TRUCK|s against the ironic exc|
+738|188|9|2|4|4352.72|0.00|0.03|A|F|1993-06-20|1993-04-08|1993-07-09|NONE|AIR|ar packages. fluffily bo|
+738|170|1|3|23|24613.91|0.04|0.08|A|F|1993-03-17|1993-04-02|1993-04-05|TAKE BACK RETURN|SHIP|nic, final excuses promise quickly regula|
+738|141|10|4|12|12493.68|0.04|0.08|A|F|1993-06-16|1993-05-05|1993-06-22|NONE|SHIP|ove the slyly regular p|
+738|175|4|5|30|32255.10|0.02|0.00|A|F|1993-06-12|1993-05-29|1993-06-25|NONE|AIR|ecial instructions haggle blithely regula|
+739|85|6|1|28|27582.24|0.00|0.03|N|O|1998-06-03|1998-08-04|1998-06-29|TAKE BACK RETURN|RAIL|elets about the pe|
+739|4|7|2|50|45200.00|0.07|0.06|N|O|1998-08-26|1998-07-16|1998-09-02|COLLECT COD|MAIL|ndencies. blith|
+739|49|2|3|12|11388.48|0.05|0.00|N|O|1998-08-20|1998-07-24|1998-08-22|NONE|MAIL|le slyly along the close i|
+739|44|3|4|47|44369.88|0.09|0.07|N|O|1998-08-12|1998-07-09|1998-08-28|NONE|REG AIR|deas according to the theodolites sn|
+739|188|9|5|30|32645.40|0.07|0.06|N|O|1998-06-19|1998-08-26|1998-07-02|DELIVER IN PERSON|REG AIR|above the even deposits. ironic requests|
+740|2|9|1|22|19844.00|0.10|0.02|N|O|1995-07-24|1995-09-11|1995-08-11|TAKE BACK RETURN|FOB|odolites cajole ironic, pending instruc|
+740|66|1|2|35|33812.10|0.00|0.00|N|O|1995-09-06|1995-08-22|1995-10-02|NONE|TRUCK|p quickly. fu|
+740|199|10|3|29|31876.51|0.06|0.05|N|O|1995-10-26|1995-09-17|1995-10-29|DELIVER IN PERSON|FOB|ntly bold pinto beans sleep quickl|
+741|187|8|1|25|27179.50|0.03|0.06|N|O|1998-07-15|1998-08-27|1998-08-12|DELIVER IN PERSON|MAIL|accounts. blithely bold pa|
+741|91|4|2|22|21803.98|0.09|0.01|N|O|1998-09-07|1998-09-28|1998-09-12|COLLECT COD|AIR|ven deposits about the regular, ironi|
+742|102|3|1|46|46096.60|0.04|0.08|A|F|1995-03-12|1995-03-20|1995-03-16|TAKE BACK RETURN|SHIP|e slyly bold deposits cajole according to|
+742|96|8|2|15|14941.35|0.08|0.05|A|F|1995-02-26|1995-03-20|1995-03-03|NONE|SHIP|blithely unusual pinto|
+742|102|5|3|24|24050.40|0.08|0.08|A|F|1995-02-12|1995-03-12|1995-02-14|DELIVER IN PERSON|SHIP|affix slyly. furiously i|
+742|192|4|4|16|17475.04|0.01|0.05|A|F|1995-01-15|1995-02-25|1995-01-24|COLLECT COD|AIR|eodolites haggle carefully regul|
+742|101|4|5|48|48052.80|0.09|0.08|R|F|1995-03-24|1995-01-23|1995-04-08|TAKE BACK RETURN|TRUCK| platelets |
+742|192|6|6|49|53517.31|0.02|0.07|A|F|1995-01-13|1995-02-13|1995-01-26|TAKE BACK RETURN|RAIL| carefully bold foxes sle|
+743|192|5|1|21|22935.99|0.01|0.04|N|O|1996-10-26|1996-11-05|1996-11-11|COLLECT COD|MAIL|d requests. packages afte|
+768|196|7|1|39|42751.41|0.06|0.08|N|O|1996-09-25|1996-10-27|1996-10-20|NONE|SHIP|out the ironic|
+768|18|9|2|2|1836.02|0.00|0.04|N|O|1996-11-13|1996-10-03|1996-11-25|DELIVER IN PERSON|SHIP|ular courts. slyly dogged accou|
+768|6|1|3|30|27180.00|0.06|0.05|N|O|1996-09-22|1996-11-03|1996-10-13|NONE|MAIL| furiously fluffy pinto beans haggle along|
+768|25|8|4|37|34225.74|0.10|0.00|N|O|1996-10-02|1996-09-23|1996-10-14|TAKE BACK RETURN|REG AIR|ending requests across the quickly|
+768|47|10|5|47|44510.88|0.06|0.05|N|O|1996-11-28|1996-10-30|1996-12-12|NONE|TRUCK|foxes. slyly ironic deposits a|
+768|112|9|6|43|43520.73|0.10|0.06|N|O|1996-09-22|1996-11-03|1996-10-22|TAKE BACK RETURN|AIR|sual ideas wake quickly|
+768|49|10|7|33|31318.32|0.01|0.04|N|O|1996-09-06|1996-09-29|1996-10-01|COLLECT COD|RAIL|sly ironic instructions. excuses can hagg|
+769|176|6|1|36|38742.12|0.02|0.02|A|F|1993-10-01|1993-08-07|1993-10-15|NONE|AIR|es. furiously iro|
+769|160|8|2|4|4240.64|0.01|0.04|R|F|1993-06-25|1993-08-12|1993-07-15|DELIVER IN PERSON|FOB| ideas. even|
+770|181|2|1|39|42166.02|0.09|0.06|N|O|1998-07-19|1998-08-09|1998-08-04|NONE|REG AIR|osits. foxes cajole |
+770|54|2|2|25|23851.25|0.03|0.02|N|O|1998-05-26|1998-07-23|1998-06-04|TAKE BACK RETURN|AIR| deposits dazzle fluffily alongside of |
+771|7|4|1|12|10884.00|0.10|0.08|N|O|1995-07-18|1995-08-02|1995-08-07|COLLECT COD|TRUCK|carefully. pending in|
+771|161|10|2|38|40324.08|0.03|0.08|N|O|1995-07-22|1995-09-10|1995-07-29|TAKE BACK RETURN|REG AIR| quickly final requests are final packages.|
+771|7|8|3|14|12698.00|0.02|0.05|N|O|1995-07-31|1995-08-13|1995-08-07|DELIVER IN PERSON|AIR|r, final packages are slyly iro|
+771|42|3|4|7|6594.28|0.06|0.02|N|O|1995-06-18|1995-08-31|1995-06-20|NONE|REG AIR|theodolites after the fluffily express |
+771|78|6|5|13|12714.91|0.09|0.01|N|O|1995-08-10|1995-08-21|1995-08-30|NONE|FOB|packages affix slyly about the quickly |
+771|82|3|6|23|22587.84|0.08|0.03|N|O|1995-06-19|1995-09-07|1995-07-09|COLLECT COD|FOB|cajole besides the quickly ironic pin|
+772|53|5|1|35|33356.75|0.10|0.06|R|F|1993-07-05|1993-06-05|1993-08-02|NONE|SHIP|kly thin packages wake slowly|
+772|84|5|2|10|9840.80|0.05|0.01|R|F|1993-05-20|1993-05-19|1993-06-15|DELIVER IN PERSON|MAIL| deposits cajole carefully instructions. t|
+772|86|7|3|35|34512.80|0.03|0.04|R|F|1993-04-18|1993-06-13|1993-05-01|COLLECT COD|MAIL|ng ideas. special packages haggle alon|
+772|180|8|4|10|10801.80|0.08|0.02|A|F|1993-05-17|1993-06-09|1993-05-29|COLLECT COD|AIR|o the furiously final deposits. furi|
+772|54|5|5|42|40070.10|0.02|0.07|A|F|1993-06-09|1993-07-16|1993-06-12|DELIVER IN PERSON|MAIL| express foxes abo|
+773|100|1|1|5|5000.50|0.06|0.04|A|F|1993-11-21|1993-12-19|1993-12-21|COLLECT COD|MAIL|ar requests. regular, thin packages u|
+773|11|5|2|31|28241.31|0.02|0.06|A|F|1993-12-30|1993-11-02|1994-01-01|TAKE BACK RETURN|MAIL|e slyly unusual deposit|
+773|151|3|3|39|40994.85|0.06|0.05|A|F|1994-01-04|1993-12-23|1994-01-26|DELIVER IN PERSON|FOB|quickly eve|
+773|29|8|4|28|26012.56|0.10|0.06|R|F|1994-01-19|1993-11-05|1994-01-23|NONE|TRUCK|he furiously slow deposits.|
+773|134|5|5|9|9307.17|0.09|0.02|R|F|1993-10-09|1993-12-25|1993-11-04|TAKE BACK RETURN|FOB|ent orbits haggle fluffily after the |
+773|40|1|6|43|40421.72|0.07|0.03|A|F|1993-11-06|1993-11-20|1993-11-08|TAKE BACK RETURN|SHIP|furiously bold dependencies. blithel|
+774|183|4|1|49|53075.82|0.08|0.03|N|O|1995-12-06|1996-01-07|1995-12-14|DELIVER IN PERSON|SHIP|ess accounts are carefully |
+774|17|4|2|3|2751.03|0.10|0.06|N|O|1996-02-13|1996-01-14|1996-03-04|COLLECT COD|FOB| slyly even courts nag blith|
+774|148|7|3|34|35636.76|0.02|0.07|N|O|1996-03-16|1996-01-03|1996-03-22|COLLECT COD|FOB|lar excuses are furiously final instr|
+774|15|6|4|8|7320.08|0.00|0.02|N|O|1996-01-24|1996-01-15|1996-02-13|COLLECT COD|RAIL|ully ironic requests c|
+774|177|5|5|44|47395.48|0.09|0.07|N|O|1996-02-29|1996-01-16|1996-03-06|NONE|REG AIR|s according to the deposits unwind ca|
+774|120|1|6|2|2040.24|0.07|0.03|N|O|1995-12-11|1996-02-10|1995-12-14|TAKE BACK RETURN|SHIP|accounts; slyly regular|
+775|32|3|1|16|14912.48|0.10|0.06|N|F|1995-05-23|1995-05-07|1995-06-19|NONE|TRUCK|un quickly slyly|
+775|174|2|2|21|22557.57|0.01|0.06|R|F|1995-05-01|1995-06-02|1995-05-13|DELIVER IN PERSON|FOB| quickly sile|
+775|108|5|3|20|20162.00|0.01|0.08|N|F|1995-06-17|1995-05-22|1995-07-13|COLLECT COD|AIR|en dependencies nag slowly |
+800|72|1|1|38|36938.66|0.00|0.05|N|O|1998-07-21|1998-09-25|1998-08-07|TAKE BACK RETURN|TRUCK|according to the bold, final dependencies |
+800|85|6|2|21|20686.68|0.04|0.05|N|O|1998-07-23|1998-10-01|1998-08-20|TAKE BACK RETURN|RAIL|ckly even requests after the carefully r|
+800|176|5|3|26|27980.42|0.01|0.02|N|O|1998-07-23|1998-10-08|1998-07-25|DELIVER IN PERSON|FOB|bove the pending requests.|
+801|6|3|1|13|11778.00|0.10|0.02|R|F|1992-04-25|1992-04-24|1992-05-16|COLLECT COD|RAIL|s are fluffily stealthily expres|
+801|95|8|2|21|20896.89|0.05|0.02|A|F|1992-03-14|1992-04-01|1992-04-05|COLLECT COD|AIR|wake silently furiously idle deposits. |
+801|3|4|3|21|18963.00|0.05|0.03|A|F|1992-04-25|1992-03-20|1992-05-04|COLLECT COD|REG AIR|cial, special packages.|
+801|164|9|4|12|12769.92|0.08|0.04|A|F|1992-06-06|1992-04-14|1992-06-12|TAKE BACK RETURN|RAIL|s. ironic pinto b|
+801|74|2|5|45|43833.15|0.01|0.06|R|F|1992-03-22|1992-03-22|1992-03-25|COLLECT COD|REG AIR| even asymptotes|
+801|122|7|6|10|10221.20|0.08|0.01|A|F|1992-06-05|1992-05-15|1992-06-21|DELIVER IN PERSON|MAIL|al accounts. carefully regular foxes wake|
+801|26|5|7|11|10186.22|0.01|0.03|A|F|1992-05-09|1992-04-19|1992-05-15|DELIVER IN PERSON|REG AIR|y special pinto beans cajole |
+802|143|6|1|40|41725.60|0.08|0.08|A|F|1995-01-07|1995-04-03|1995-01-23|DELIVER IN PERSON|RAIL|y bold accou|
+802|133|4|2|34|35126.42|0.08|0.06|A|F|1995-03-01|1995-03-15|1995-03-12|COLLECT COD|AIR|instructions cajole carefully. quietl|
+802|131|2|3|44|45369.72|0.07|0.04|R|F|1995-01-09|1995-02-04|1995-01-18|TAKE BACK RETURN|SHIP|rmanently idly special requ|
+802|157|2|4|18|19028.70|0.09|0.02|R|F|1995-03-06|1995-02-07|1995-03-19|TAKE BACK RETURN|RAIL|y regular requests engage furiously final d|
+802|132|3|5|19|19610.47|0.08|0.06|A|F|1995-04-01|1995-02-20|1995-04-23|DELIVER IN PERSON|REG AIR|old, furious|
+803|54|9|1|8|7632.40|0.07|0.01|N|O|1997-08-04|1997-06-19|1997-08-12|NONE|SHIP|ronic theodo|
+803|99|10|2|21|20980.89|0.08|0.06|N|O|1997-08-25|1997-06-30|1997-09-10|TAKE BACK RETURN|AIR|ironic packages cajole slyly. un|
+804|126|7|1|30|30783.60|0.08|0.04|A|F|1993-03-29|1993-05-07|1993-04-14|COLLECT COD|REG AIR|ehind the quietly regular pac|
+804|199|3|2|2|2198.38|0.02|0.00|A|F|1993-06-23|1993-04-30|1993-06-25|NONE|TRUCK|slyly silent |
+804|76|5|3|44|42947.08|0.04|0.05|R|F|1993-07-06|1993-04-13|1993-07-28|DELIVER IN PERSON|TRUCK|ly final deposits? special |
+804|38|9|4|21|19698.63|0.01|0.00|A|F|1993-04-12|1993-06-06|1993-04-20|DELIVER IN PERSON|RAIL|ular, ironic foxes. quickly even accounts|
+805|198|10|1|25|27454.75|0.07|0.06|N|O|1995-08-05|1995-09-30|1995-08-06|NONE|AIR|ide of the pending, sly requests. quickly f|
+805|57|5|2|29|27754.45|0.07|0.01|N|O|1995-08-24|1995-08-15|1995-09-16|TAKE BACK RETURN|AIR|dolites according to the slyly f|
+805|47|8|3|12|11364.48|0.01|0.06|N|O|1995-07-13|1995-09-27|1995-08-02|TAKE BACK RETURN|REG AIR| regular foxes. furio|
+805|76|6|4|26|25377.82|0.08|0.07|N|O|1995-08-28|1995-09-24|1995-09-11|TAKE BACK RETURN|RAIL|. ironic deposits sleep across |
+806|105|2|1|1|1005.10|0.04|0.07|N|O|1996-07-14|1996-09-12|1996-07-25|COLLECT COD|RAIL|ar accounts? pending, pending foxes a|
+806|160|5|2|22|23323.52|0.08|0.02|N|O|1996-10-03|1996-08-11|1996-10-20|DELIVER IN PERSON|REG AIR|fily pending |
+806|91|3|3|4|3964.36|0.04|0.03|N|O|1996-08-09|1996-09-18|1996-08-13|COLLECT COD|TRUCK|eans. quickly ironic ideas |
+807|117|7|1|49|49838.39|0.00|0.00|R|F|1993-12-05|1994-01-13|1993-12-25|COLLECT COD|REG AIR| furiously according to the un|
+807|155|10|2|49|51702.35|0.01|0.06|A|F|1994-01-17|1994-01-24|1994-01-22|COLLECT COD|TRUCK|y regular requests haggle.|
+807|181|2|3|48|51896.64|0.07|0.07|A|F|1994-01-08|1994-02-02|1994-01-15|DELIVER IN PERSON|SHIP|kly across the f|
+807|80|1|4|10|9800.80|0.09|0.00|R|F|1994-01-19|1994-02-12|1994-01-28|NONE|TRUCK|furiously final depths sleep a|
+807|143|6|5|30|31294.20|0.02|0.01|R|F|1994-01-19|1994-01-09|1994-01-27|NONE|RAIL|cial accoun|
+807|12|2|6|11|10032.11|0.02|0.04|R|F|1994-03-25|1994-01-26|1994-04-14|NONE|FOB|unts above the slyly final ex|
+807|1|6|7|19|17119.00|0.08|0.05|A|F|1994-02-10|1994-02-20|1994-03-06|NONE|SHIP|ns haggle quickly across the furi|
+832|103|6|1|45|45139.50|0.01|0.02|A|F|1992-05-08|1992-06-06|1992-06-04|COLLECT COD|MAIL|foxes engage slyly alon|
+832|48|1|2|24|22752.96|0.05|0.06|A|F|1992-06-15|1992-07-14|1992-06-17|NONE|TRUCK|ully. carefully speci|
+833|54|5|1|1|954.05|0.04|0.04|R|F|1994-04-26|1994-04-05|1994-04-29|COLLECT COD|MAIL|ffily ironic theodolites|
+833|112|6|2|38|38460.18|0.05|0.05|A|F|1994-04-05|1994-04-21|1994-05-01|COLLECT COD|TRUCK| platelets promise furiously. |
+833|162|7|3|9|9559.44|0.05|0.07|A|F|1994-02-28|1994-04-26|1994-03-20|TAKE BACK RETURN|FOB|ecial, even requests. even, bold instructi|
+834|145|2|1|36|37625.04|0.06|0.04|R|F|1994-06-28|1994-07-25|1994-07-07|TAKE BACK RETURN|SHIP|ccounts haggle after the furiously |
+834|7|2|2|11|9977.00|0.03|0.00|A|F|1994-09-18|1994-08-03|1994-10-02|DELIVER IN PERSON|TRUCK|inst the regular packa|
+835|107|2|1|33|33234.30|0.09|0.06|N|O|1995-11-01|1995-12-02|1995-11-24|DELIVER IN PERSON|RAIL|onic instructions among the carefully iro|
+835|185|6|2|28|30385.04|0.02|0.02|N|O|1995-12-27|1995-12-11|1996-01-21|NONE|SHIP| fluffily furious pinto beans|
+836|188|9|1|6|6529.08|0.09|0.03|N|O|1996-12-09|1997-01-31|1996-12-29|COLLECT COD|TRUCK|fully bold theodolites are daringly across|
+836|84|5|2|18|17713.44|0.03|0.05|N|O|1997-02-27|1997-02-11|1997-03-22|NONE|REG AIR|y pending packages use alon|
+836|141|8|3|46|47892.44|0.05|0.07|N|O|1997-03-21|1997-02-06|1997-04-05|NONE|REG AIR|boldly final pinto beans haggle furiously|
+837|57|5|1|39|37324.95|0.03|0.08|A|F|1994-07-22|1994-08-10|1994-08-11|NONE|RAIL|ecial pinto bea|
+837|88|9|2|24|23713.92|0.08|0.00|R|F|1994-06-27|1994-09-02|1994-07-27|DELIVER IN PERSON|FOB|p carefully. theodolites use. bold courts a|
+838|134|10|1|20|20682.60|0.10|0.07|N|O|1998-04-11|1998-03-25|1998-04-19|COLLECT COD|TRUCK| furiously final ideas. slow, bold |
+838|29|10|2|27|25083.54|0.05|0.07|N|O|1998-02-15|1998-04-03|1998-02-20|DELIVER IN PERSON|SHIP| pending pinto beans haggle about t|
+838|95|7|3|23|22887.07|0.10|0.07|N|O|1998-03-26|1998-04-17|1998-04-02|COLLECT COD|AIR|ets haggle furiously furiously regular r|
+838|44|5|4|18|16992.72|0.09|0.00|N|O|1998-03-28|1998-04-06|1998-03-31|TAKE BACK RETURN|AIR|hely unusual foxes. furio|
+839|158|10|1|23|24337.45|0.07|0.02|N|O|1995-10-17|1995-11-03|1995-11-04|COLLECT COD|AIR|ng ideas haggle accord|
+839|189|10|2|47|51191.46|0.08|0.00|N|O|1995-10-17|1995-11-06|1995-11-10|NONE|AIR|refully final excuses about |
+864|130|5|1|34|35024.42|0.03|0.04|N|O|1997-12-16|1997-10-23|1998-01-12|TAKE BACK RETURN|SHIP|gside of the furiously special|
+864|98|1|2|7|6986.63|0.01|0.02|N|O|1997-11-13|1997-10-07|1997-12-13|TAKE BACK RETURN|MAIL|ven requests should sleep along |
+864|80|10|3|34|33322.72|0.03|0.00|N|O|1997-09-14|1997-11-04|1997-09-21|TAKE BACK RETURN|REG AIR|to the furiously ironic platelets! |
+865|198|10|1|16|17571.04|0.07|0.03|R|F|1993-08-24|1993-06-26|1993-08-28|TAKE BACK RETURN|TRUCK|y even accounts. quickly bold decoys|
+865|20|7|2|3|2760.06|0.02|0.05|A|F|1993-07-17|1993-07-14|1993-08-01|NONE|MAIL|fully regular the|
+865|87|8|3|15|14806.20|0.00|0.06|R|F|1993-07-05|1993-06-25|1993-07-26|NONE|SHIP| deposits sleep quickl|
+865|169|4|4|34|36351.44|0.09|0.06|A|F|1993-05-09|1993-07-28|1993-05-18|DELIVER IN PERSON|REG AIR|furiously fluffily unusual account|
+866|136|7|1|5|5180.65|0.08|0.00|R|F|1993-01-22|1993-01-14|1993-02-07|TAKE BACK RETURN|AIR|tegrate fluffily. carefully f|
+867|139|10|1|7|7273.91|0.04|0.07|A|F|1994-02-19|1993-12-25|1994-02-25|DELIVER IN PERSON|TRUCK|pendencies-- slyly unusual packages hagg|
+868|168|9|1|8|8545.28|0.06|0.03|R|F|1992-10-07|1992-08-01|1992-10-16|NONE|MAIL|l deposits. blithely regular pint|
+868|29|8|2|13|12077.26|0.05|0.07|R|F|1992-07-25|1992-08-26|1992-08-04|NONE|AIR|gged instructi|
+868|68|5|3|19|18393.14|0.09|0.06|R|F|1992-06-22|1992-08-27|1992-07-04|COLLECT COD|SHIP|lyly ironic platelets wake. rut|
+868|122|1|4|43|43951.16|0.02|0.04|A|F|1992-07-02|1992-07-22|1992-07-21|COLLECT COD|SHIP|kly silent deposits wake dar|
+868|25|8|5|27|24975.54|0.04|0.01|R|F|1992-08-01|1992-08-25|1992-08-12|TAKE BACK RETURN|RAIL|oss the fluffily unusual pinto |
+868|125|6|6|19|19477.28|0.02|0.05|R|F|1992-09-20|1992-07-18|1992-10-04|NONE|FOB|ely even deposits lose blithe|
+869|63|2|1|27|26002.62|0.07|0.07|N|O|1997-01-30|1997-02-17|1997-02-26|TAKE BACK RETURN|TRUCK|uffily even excuses? slyly even deposits |
+869|47|4|2|36|34093.44|0.04|0.01|N|O|1997-05-03|1997-03-17|1997-05-24|NONE|RAIL|ong the furiously bold instructi|
+870|50|9|1|36|34201.80|0.04|0.07|A|F|1993-10-18|1993-09-16|1993-11-15|COLLECT COD|MAIL|fily. furiously final accounts are |
+870|186|7|2|5|5430.90|0.06|0.05|A|F|1993-08-13|1993-09-11|1993-08-24|COLLECT COD|FOB|e slyly excuses. ironi|
+871|97|8|1|48|47860.32|0.10|0.03|N|O|1996-02-25|1996-02-09|1996-03-18|NONE|AIR|coys dazzle slyly slow notornis. f|
+871|55|10|2|47|44887.35|0.07|0.03|N|O|1995-12-25|1996-02-01|1996-01-24|TAKE BACK RETURN|RAIL|ss, final dep|
+871|108|5|3|13|13105.30|0.09|0.01|N|O|1996-01-25|1996-01-24|1996-02-03|NONE|REG AIR| haggle furiou|
+871|190|1|4|29|31615.51|0.06|0.07|N|O|1995-11-16|1996-01-27|1995-12-16|DELIVER IN PERSON|RAIL|ests are carefu|
+871|128|7|5|8|8224.96|0.00|0.01|N|O|1995-11-25|1996-01-12|1995-12-12|DELIVER IN PERSON|AIR|lar ideas-- slyly even accou|
+871|143|2|6|26|27121.64|0.00|0.06|N|O|1996-02-07|1996-01-05|1996-02-25|COLLECT COD|AIR|symptotes use quickly near the |
+871|174|3|7|4|4296.68|0.00|0.07|N|O|1996-03-09|1996-01-20|1996-03-26|COLLECT COD|FOB|l, regular dependencies w|
+896|39|10|1|47|44134.41|0.07|0.08|R|F|1993-05-28|1993-05-15|1993-06-15|DELIVER IN PERSON|TRUCK|ly even pinto beans integrate. b|
+896|198|2|2|10|10981.90|0.03|0.07|A|F|1993-07-07|1993-06-03|1993-07-24|COLLECT COD|SHIP| quickly even theodolites. carefully regu|
+896|2|9|3|7|6314.00|0.09|0.02|A|F|1993-05-02|1993-05-24|1993-05-31|DELIVER IN PERSON|MAIL| requests |
+896|152|3|4|11|11573.65|0.08|0.04|A|F|1993-05-19|1993-05-22|1993-06-08|COLLECT COD|MAIL|the multipliers sleep|
+896|188|9|5|34|36998.12|0.00|0.05|R|F|1993-05-21|1993-06-01|1993-05-23|NONE|TRUCK|ular, close requests cajo|
+896|177|6|6|44|47395.48|0.09|0.08|R|F|1993-05-19|1993-04-14|1993-06-02|DELIVER IN PERSON|FOB|lar, pending packages. deposits are q|
+896|109|2|7|11|11100.10|0.01|0.07|A|F|1993-05-01|1993-04-09|1993-05-06|TAKE BACK RETURN|FOB|rding to the pinto beans wa|
+897|91|4|1|15|14866.35|0.07|0.04|R|F|1995-05-25|1995-05-09|1995-06-07|COLLECT COD|REG AIR|r ideas. slyly spec|
+897|184|5|2|26|28188.68|0.05|0.08|N|O|1995-07-01|1995-06-10|1995-07-14|COLLECT COD|MAIL|tions sleep according to the special|
+897|126|1|3|13|13339.56|0.07|0.00|A|F|1995-03-30|1995-05-17|1995-04-21|TAKE BACK RETURN|MAIL|bold accounts mold carefully! braids|
+897|102|7|4|2|2004.20|0.08|0.08|R|F|1995-05-22|1995-05-07|1995-06-16|COLLECT COD|RAIL|into beans. slyly special fox|
+898|161|2|1|9|9550.44|0.07|0.08|A|F|1993-07-04|1993-07-09|1993-07-25|NONE|AIR|e slyly across the blithe|
+898|179|7|2|37|39929.29|0.03|0.05|A|F|1993-08-17|1993-08-04|1993-09-01|DELIVER IN PERSON|REG AIR|packages sleep furiously|
+898|49|8|3|11|10439.44|0.01|0.00|A|F|1993-09-13|1993-08-31|1993-09-25|TAKE BACK RETURN|MAIL|etly bold accounts |
+898|193|6|4|36|39354.84|0.04|0.07|R|F|1993-08-04|1993-07-25|1993-08-23|DELIVER IN PERSON|REG AIR| after the carefully |
+899|61|10|1|18|17299.08|0.04|0.05|N|O|1998-08-06|1998-05-09|1998-09-05|DELIVER IN PERSON|AIR|re daring, pending deposits. blit|
+899|47|4|2|25|23676.00|0.00|0.07|N|O|1998-07-21|1998-05-12|1998-08-16|NONE|REG AIR|rly final sentiments. bold pinto beans |
+899|85|6|3|4|3940.32|0.09|0.05|N|O|1998-06-02|1998-06-28|1998-06-14|TAKE BACK RETURN|REG AIR|ter the carefully regular deposits are agai|
+899|180|9|4|14|15122.52|0.05|0.03|N|O|1998-05-21|1998-05-28|1998-06-03|TAKE BACK RETURN|FOB|ades impress carefully|
+899|71|10|5|4|3884.28|0.06|0.02|N|O|1998-04-11|1998-05-14|1998-04-27|NONE|TRUCK|ges. blithe, ironic waters cajole care|
+899|120|4|6|47|47945.64|0.00|0.04|N|O|1998-04-14|1998-05-30|1998-05-13|DELIVER IN PERSON|TRUCK|furiously final foxes after the s|
+899|14|1|7|11|10054.11|0.02|0.08|N|O|1998-06-03|1998-06-15|1998-06-20|COLLECT COD|REG AIR|t the ironic|
+900|199|1|1|44|48364.36|0.01|0.06|R|F|1994-12-15|1994-12-03|1994-12-27|COLLECT COD|MAIL| detect quick|
+900|115|6|2|48|48725.28|0.08|0.04|A|F|1994-12-22|1994-11-08|1995-01-19|COLLECT COD|TRUCK|cial pinto beans nag |
+900|75|6|3|24|23401.68|0.03|0.00|R|F|1994-10-21|1994-12-25|1994-10-22|TAKE BACK RETURN|SHIP|-ray furiously un|
+901|22|7|1|36|33192.72|0.01|0.01|N|O|1998-08-11|1998-10-09|1998-08-27|DELIVER IN PERSON|REG AIR|. accounts are care|
+901|46|7|2|2|1892.08|0.09|0.02|N|O|1998-10-25|1998-09-27|1998-11-01|DELIVER IN PERSON|AIR|d foxes use slyly|
+901|43|10|3|37|34892.48|0.04|0.08|N|O|1998-11-01|1998-09-13|1998-11-05|NONE|AIR|ickly final deposits |
+901|18|9|4|11|10098.11|0.00|0.06|N|O|1998-11-13|1998-10-19|1998-11-14|TAKE BACK RETURN|TRUCK|ourts among the quickly expre|
+902|111|2|1|3|3033.33|0.06|0.00|R|F|1994-10-01|1994-10-25|1994-10-28|COLLECT COD|MAIL|into beans thrash blithely about the flu|
+902|118|2|2|8|8144.88|0.06|0.07|R|F|1994-10-25|1994-09-20|1994-11-07|COLLECT COD|RAIL| orbits al|
+902|165|2|3|24|25563.84|0.02|0.05|R|F|1994-11-08|1994-10-12|1994-11-26|NONE|FOB|. blithely even accounts poach furiously i|
+903|65|10|1|27|26056.62|0.04|0.03|N|O|1995-09-18|1995-09-20|1995-10-02|TAKE BACK RETURN|SHIP|lly pending foxes. furiously|
+903|9|2|2|35|31815.00|0.06|0.05|N|O|1995-09-18|1995-08-21|1995-10-12|TAKE BACK RETURN|TRUCK|rets wake fin|
+903|9|2|3|33|29997.00|0.02|0.03|N|O|1995-09-24|1995-09-01|1995-10-12|COLLECT COD|MAIL|ely ironic packages wake blithely|
+903|56|1|4|9|8604.45|0.09|0.00|N|O|1995-10-06|1995-09-14|1995-10-24|NONE|TRUCK|he slyly ev|
+903|42|3|5|1|942.04|0.04|0.00|N|O|1995-10-22|1995-09-13|1995-11-03|NONE|AIR|y final platelets sublate among the |
+903|168|9|6|13|13886.08|0.07|0.02|N|O|1995-09-11|1995-10-04|1995-10-03|COLLECT COD|SHIP|sleep along the final|
+928|169|10|1|29|31005.64|0.07|0.02|R|F|1995-05-17|1995-05-12|1995-05-21|NONE|REG AIR|ly alongside of the s|
+928|48|7|2|24|22752.96|0.05|0.08|A|F|1995-04-06|1995-05-08|1995-04-24|DELIVER IN PERSON|AIR|s the furiously regular warthogs im|
+928|152|10|3|46|48398.90|0.08|0.00|A|F|1995-05-09|1995-04-09|1995-06-01|DELIVER IN PERSON|REG AIR| beans sleep against the carefully ir|
+928|52|4|4|43|40938.15|0.10|0.05|A|F|1995-04-14|1995-04-21|1995-05-09|NONE|REG AIR|blithely. express, silent requests doze at|
+928|12|3|5|38|34656.38|0.02|0.08|N|F|1995-06-08|1995-04-15|1995-06-30|TAKE BACK RETURN|SHIP|xpress grouc|
+928|55|6|6|50|47752.50|0.05|0.00|N|F|1995-06-07|1995-04-15|1995-07-01|DELIVER IN PERSON|TRUCK| slyly slyly special request|
+928|11|5|7|11|10021.11|0.00|0.01|A|F|1995-04-29|1995-04-16|1995-04-30|NONE|AIR|longside of|
+929|129|8|1|45|46310.40|0.09|0.01|R|F|1993-01-24|1992-12-06|1993-02-16|DELIVER IN PERSON|REG AIR|ges haggle careful|
+929|175|5|2|44|47307.48|0.02|0.00|A|F|1992-10-09|1992-11-20|1992-10-22|DELIVER IN PERSON|SHIP|s. excuses cajole. carefully regu|
+929|74|5|3|14|13636.98|0.06|0.07|A|F|1992-10-21|1992-11-17|1992-11-15|NONE|FOB|gainst the|
+929|102|5|4|7|7014.70|0.06|0.01|A|F|1992-12-24|1992-12-19|1993-01-08|TAKE BACK RETURN|TRUCK|ithely. slyly c|
+930|45|4|1|36|34021.44|0.10|0.04|R|F|1994-12-21|1995-02-20|1994-12-24|COLLECT COD|RAIL|quickly regular pinto beans sle|
+930|18|8|2|47|43146.47|0.08|0.00|A|F|1995-03-20|1995-02-04|1995-04-04|DELIVER IN PERSON|AIR|ackages. fluffily e|
+930|65|10|3|10|9650.60|0.07|0.08|A|F|1994-12-18|1995-01-27|1995-01-16|COLLECT COD|AIR|ckly regular requests: regular instructions|
+930|100|2|4|21|21002.10|0.06|0.02|A|F|1995-02-16|1995-03-03|1995-03-13|DELIVER IN PERSON|SHIP|foxes. regular deposits integrate carefu|
+930|164|9|5|50|53208.00|0.03|0.06|A|F|1995-04-03|1995-01-29|1995-04-22|COLLECT COD|MAIL| excuses among the furiously express ideas |
+930|145|4|6|10|10451.40|0.00|0.04|A|F|1995-02-09|1995-02-17|1995-02-16|NONE|SHIP|blithely bold i|
+930|167|4|7|30|32014.80|0.07|0.08|R|F|1995-01-20|1995-02-28|1995-02-04|TAKE BACK RETURN|RAIL|g accounts sleep along the platelets.|
+931|40|1|1|18|16920.72|0.00|0.05|A|F|1993-04-04|1993-01-11|1993-04-13|NONE|RAIL|slyly ironic re|
+931|17|7|2|10|9170.10|0.05|0.07|A|F|1993-03-01|1993-01-09|1993-03-07|TAKE BACK RETURN|SHIP|ajole quickly. slyly sil|
+931|147|6|3|48|50262.72|0.01|0.08|A|F|1993-02-03|1993-03-02|1993-02-09|TAKE BACK RETURN|FOB|ep alongside of the fluffy |
+931|82|3|4|38|37319.04|0.08|0.08|A|F|1993-03-06|1993-02-24|1993-03-27|DELIVER IN PERSON|RAIL|usly final packages integrate carefully|
+932|44|1|1|41|38705.64|0.01|0.05|N|O|1997-06-05|1997-07-22|1997-06-26|COLLECT COD|RAIL|foxes. ironic pl|
+933|49|8|1|23|21827.92|0.02|0.04|R|F|1992-08-13|1992-09-18|1992-08-25|DELIVER IN PERSON|MAIL| the furiously bold dinos. sly|
+933|13|4|2|27|24651.27|0.02|0.01|R|F|1992-10-03|1992-10-02|1992-10-26|DELIVER IN PERSON|RAIL|ests. express|
+933|100|2|3|26|26002.60|0.05|0.00|A|F|1992-11-09|1992-11-03|1992-11-16|DELIVER IN PERSON|AIR| the deposits affix slyly after t|
+934|118|5|1|18|18325.98|0.07|0.01|N|O|1996-09-10|1996-09-20|1996-09-25|COLLECT COD|RAIL|y unusual requests dazzle above t|
+935|28|3|1|23|21344.46|0.05|0.00|N|O|1997-11-11|1997-11-22|1997-11-29|COLLECT COD|REG AIR|ular accounts about|
+935|65|10|2|23|22196.38|0.02|0.01|N|O|1998-01-11|1997-11-25|1998-02-05|COLLECT COD|TRUCK|hes haggle furiously dolphins. qu|
+935|135|1|3|36|37264.68|0.06|0.00|N|O|1997-11-05|1997-12-05|1997-11-25|TAKE BACK RETURN|AIR|leep about the exp|
+935|58|3|4|13|12454.65|0.08|0.04|N|O|1998-01-13|1997-11-30|1998-02-08|DELIVER IN PERSON|TRUCK|ld platelet|
+935|13|7|5|8|7304.08|0.02|0.05|N|O|1998-01-12|1997-11-02|1998-02-05|NONE|TRUCK|cept the quickly regular p|
+935|59|1|6|1|959.05|0.01|0.08|N|O|1997-12-14|1997-11-22|1998-01-08|DELIVER IN PERSON|TRUCK| instructions. ironic acc|
+960|107|10|1|1|1007.10|0.07|0.00|A|F|1994-12-24|1994-10-26|1995-01-20|DELIVER IN PERSON|AIR|y ironic packages. quickly even |
+960|117|7|2|25|25427.75|0.06|0.08|R|F|1994-12-01|1994-10-29|1994-12-27|DELIVER IN PERSON|RAIL|ts. fluffily regular requests |
+960|175|3|3|32|34405.44|0.01|0.08|R|F|1995-01-19|1994-12-17|1995-02-04|DELIVER IN PERSON|FOB|around the blithe, even pl|
+961|118|5|1|7|7126.77|0.10|0.00|N|O|1995-07-23|1995-07-20|1995-08-11|TAKE BACK RETURN|RAIL|usual dolphins. ironic pearls sleep blit|
+961|91|2|2|18|17839.62|0.09|0.05|N|O|1995-07-01|1995-08-14|1995-07-04|DELIVER IN PERSON|AIR|rmanent foxes haggle speci|
+961|97|8|3|42|41877.78|0.06|0.01|N|O|1995-08-24|1995-08-21|1995-09-10|TAKE BACK RETURN|SHIP|ests do cajole blithely. furiously bo|
+961|34|10|4|29|27086.87|0.00|0.07|N|F|1995-06-10|1995-08-20|1995-06-26|TAKE BACK RETURN|TRUCK|l accounts use blithely against the|
+961|26|7|5|38|35188.76|0.03|0.05|N|O|1995-08-21|1995-07-19|1995-08-27|NONE|RAIL|he blithely special requests. furiousl|
+961|197|8|6|30|32915.70|0.09|0.03|N|O|1995-07-06|1995-07-20|1995-07-26|DELIVER IN PERSON|MAIL|warhorses slee|
+962|57|8|1|36|34453.80|0.01|0.03|R|F|1994-08-09|1994-07-10|1994-09-02|COLLECT COD|TRUCK|al foxes. iron|
+962|36|2|2|27|25272.81|0.09|0.02|A|F|1994-05-11|1994-07-10|1994-06-03|TAKE BACK RETURN|SHIP|y slyly express deposits. final i|
+962|80|1|3|3|2940.24|0.07|0.08|A|F|1994-05-08|1994-07-06|1994-06-02|DELIVER IN PERSON|FOB|ag furiously. even pa|
+962|57|5|4|20|19141.00|0.04|0.02|R|F|1994-08-26|1994-06-27|1994-09-11|DELIVER IN PERSON|SHIP| deposits use fluffily according to |
+962|152|7|5|12|12625.80|0.02|0.00|A|F|1994-06-09|1994-06-07|1994-06-11|COLLECT COD|TRUCK|across the furiously regular escapades daz|
+962|188|9|6|5|5440.90|0.02|0.05|A|F|1994-08-29|1994-07-15|1994-09-19|COLLECT COD|TRUCK|efully bold packages run slyly caref|
+963|194|8|1|7|7659.33|0.01|0.00|R|F|1994-09-12|1994-07-18|1994-09-17|DELIVER IN PERSON|REG AIR|s. slyly regular depe|
+963|98|10|2|48|47908.32|0.10|0.06|R|F|1994-08-25|1994-08-12|1994-09-21|DELIVER IN PERSON|RAIL|ages. quickly express deposits cajole pe|
+964|199|10|1|39|42868.41|0.04|0.01|N|O|1995-06-21|1995-07-24|1995-06-24|NONE|AIR|se furiously regular instructions. blith|
+964|113|4|2|1|1013.11|0.02|0.05|N|O|1995-08-20|1995-07-29|1995-09-10|DELIVER IN PERSON|REG AIR|unts. quickly even platelets s|
+964|57|5|3|49|46895.45|0.01|0.03|N|O|1995-09-06|1995-08-10|1995-10-05|NONE|MAIL|ounts. blithely regular packag|
+964|55|3|4|44|42022.20|0.05|0.02|N|O|1995-09-18|1995-08-02|1995-10-17|TAKE BACK RETURN|TRUCK|ronic deposit|
+965|108|1|1|20|20162.00|0.04|0.05|N|F|1995-06-16|1995-07-20|1995-07-06|COLLECT COD|MAIL|kly. carefully pending requ|
+965|18|5|2|23|21114.23|0.06|0.08|N|O|1995-07-12|1995-07-08|1995-08-11|COLLECT COD|MAIL|ld kindle carefully across th|
+966|180|8|1|19|20523.42|0.07|0.01|N|O|1998-05-26|1998-07-15|1998-05-29|COLLECT COD|FOB|efully final pinto beans. quickly |
+966|117|4|2|42|42718.62|0.02|0.06|N|O|1998-06-28|1998-06-20|1998-07-05|NONE|TRUCK|tions boost furiously car|
+966|22|1|3|42|38724.84|0.06|0.08|N|O|1998-06-15|1998-06-08|1998-07-05|NONE|RAIL|sly ironic asymptotes hagg|
+966|5|2|4|20|18100.00|0.04|0.01|N|O|1998-07-19|1998-07-15|1998-07-27|NONE|TRUCK|pecial ins|
+967|59|4|1|41|39321.05|0.05|0.05|R|F|1992-09-21|1992-08-15|1992-10-21|NONE|MAIL|ld foxes wake closely special|
+967|85|6|2|4|3940.32|0.01|0.02|A|F|1992-07-15|1992-07-27|1992-07-18|DELIVER IN PERSON|REG AIR|platelets hang carefully along |
+967|132|8|3|10|10321.30|0.00|0.02|A|F|1992-09-18|1992-08-06|1992-09-19|DELIVER IN PERSON|MAIL|old pinto beans alongside of the exp|
+967|148|7|4|49|51358.86|0.01|0.04|A|F|1992-09-28|1992-09-15|1992-10-14|NONE|SHIP|the slyly even ideas. carefully even|
+967|17|1|5|41|37597.41|0.08|0.04|A|F|1992-07-23|1992-08-07|1992-08-13|TAKE BACK RETURN|FOB|efully special ide|
+967|106|9|6|17|17103.70|0.05|0.06|A|F|1992-10-02|1992-08-19|1992-10-25|NONE|MAIL|y ironic foxes caj|
+967|161|8|7|18|19100.88|0.00|0.02|A|F|1992-10-06|1992-08-05|1992-10-15|DELIVER IN PERSON|RAIL|ngage blith|
+992|60|2|1|14|13440.84|0.10|0.03|N|O|1998-01-29|1997-12-29|1998-02-18|TAKE BACK RETURN|MAIL|the unusual, even dependencies affix fluff|
+992|38|9|2|34|31893.02|0.02|0.00|N|O|1997-11-29|1998-01-21|1997-11-30|NONE|RAIL|s use silently. blithely regular ideas b|
+992|105|6|3|30|30153.00|0.10|0.00|N|O|1997-12-15|1998-02-02|1998-01-12|NONE|SHIP|nic instructions n|
+992|48|5|4|21|19908.84|0.06|0.06|N|O|1997-11-13|1997-12-28|1997-12-10|NONE|TRUCK|fily. quickly special deposit|
+992|92|4|5|7|6944.63|0.09|0.05|N|O|1997-11-30|1997-12-24|1997-12-16|DELIVER IN PERSON|TRUCK|ideas haggle. special theodolit|
+992|75|3|6|41|39977.87|0.10|0.01|N|O|1997-11-14|1998-02-04|1997-11-23|TAKE BACK RETURN|AIR|eodolites cajole across the accounts.|
+993|175|5|1|33|35480.61|0.01|0.05|N|O|1996-01-03|1995-11-28|1996-01-23|DELIVER IN PERSON|AIR| the deposits affix agains|
+993|3|6|2|28|25284.00|0.06|0.08|N|O|1995-10-24|1995-11-20|1995-11-06|DELIVER IN PERSON|RAIL|lites. even theodolite|
+993|40|1|3|10|9400.40|0.03|0.08|N|O|1995-12-17|1995-11-13|1995-12-20|NONE|RAIL|encies wake fur|
+993|191|4|4|40|43647.60|0.01|0.01|N|O|1995-11-16|1995-11-01|1995-12-05|TAKE BACK RETURN|RAIL|gle above the furiously |
+993|146|7|5|33|34522.62|0.09|0.08|N|O|1995-09-28|1995-10-24|1995-10-03|COLLECT COD|RAIL|fluffily. quiet excuses sleep furiously sly|
+993|137|3|6|35|36299.55|0.04|0.02|N|O|1995-10-26|1995-10-20|1995-11-05|DELIVER IN PERSON|FOB|es. ironic, ironic requests|
+993|5|2|7|15|13575.00|0.09|0.03|N|O|1995-09-27|1995-10-21|1995-10-17|DELIVER IN PERSON|MAIL|sits. pending pinto beans haggle? ca|
+994|65|6|1|4|3860.24|0.07|0.03|R|F|1994-07-05|1994-05-21|1994-07-20|COLLECT COD|SHIP|aggle carefully acc|
+994|10|3|2|11|10010.11|0.01|0.00|R|F|1994-05-03|1994-06-10|1994-05-22|NONE|AIR|ular accounts sleep |
+994|31|7|3|5|4655.15|0.08|0.08|A|F|1994-06-24|1994-06-14|1994-06-26|NONE|MAIL|ainst the pending requests. packages sl|
+994|131|7|4|25|25778.25|0.10|0.00|A|F|1994-06-03|1994-06-02|1994-06-06|COLLECT COD|RAIL|usual pinto beans.|
+995|173|1|1|15|16097.55|0.08|0.05|N|O|1995-06-30|1995-08-04|1995-07-27|NONE|REG AIR|uses. fluffily fina|
+995|129|4|2|28|28815.36|0.08|0.03|N|F|1995-06-12|1995-07-20|1995-06-19|DELIVER IN PERSON|SHIP|pades. quick, final frays use flu|
+995|166|3|3|45|47977.20|0.00|0.05|N|O|1995-08-02|1995-07-21|1995-08-03|DELIVER IN PERSON|SHIP|lar packages detect blithely above t|
+995|66|3|4|25|24151.50|0.01|0.08|N|O|1995-09-08|1995-08-05|1995-09-28|NONE|TRUCK|lyly even |
+995|24|5|5|18|16632.36|0.06|0.03|N|O|1995-07-03|1995-07-29|1995-07-22|TAKE BACK RETURN|AIR| even accounts unwind c|
+996|173|2|1|43|46146.31|0.03|0.06|N|O|1998-03-27|1998-03-25|1998-04-06|COLLECT COD|SHIP| the blithely ironic foxes. slyly silent d|
+997|163|4|1|11|11694.76|0.00|0.02|N|O|1997-06-16|1997-07-21|1997-07-14|DELIVER IN PERSON|TRUCK|p furiously according to t|
+997|48|9|2|17|16116.68|0.03|0.00|N|O|1997-07-28|1997-07-26|1997-08-20|DELIVER IN PERSON|SHIP|aggle quickly furiously|
+998|10|7|1|22|20020.22|0.04|0.05|A|F|1994-12-03|1995-02-17|1994-12-19|TAKE BACK RETURN|RAIL|lites. qui|
+998|181|2|2|7|7568.26|0.10|0.05|R|F|1995-03-24|1995-01-18|1995-04-03|NONE|MAIL|nic deposits. even asym|
+998|142|9|3|30|31264.20|0.05|0.07|A|F|1994-12-02|1995-01-23|1994-12-23|NONE|SHIP|lyly idle Tir|
+998|11|8|4|6|5466.06|0.09|0.05|R|F|1995-03-20|1994-12-27|1995-04-13|DELIVER IN PERSON|MAIL|refully accounts. carefully express ac|
+998|73|2|5|1|973.07|0.04|0.00|R|F|1995-01-05|1995-01-06|1995-01-13|NONE|SHIP|es sleep. regular dependencies use bl|
+999|61|6|1|34|32676.04|0.00|0.08|R|F|1993-10-30|1993-10-17|1993-10-31|TAKE BACK RETURN|SHIP|its. daringly final instruc|
+999|199|1|2|41|45066.79|0.08|0.01|A|F|1993-10-16|1993-12-04|1993-11-03|DELIVER IN PERSON|REG AIR|us depths. carefully ironic instruc|
+999|118|5|3|15|15271.65|0.07|0.06|A|F|1993-12-12|1993-10-18|1994-01-08|COLLECT COD|REG AIR|y ironic requests. carefully regu|
+999|3|4|4|10|9030.00|0.05|0.02|A|F|1993-11-23|1993-12-02|1993-11-29|NONE|MAIL|efully pending|
+999|19|10|5|3|2757.03|0.03|0.00|R|F|1993-09-17|1993-10-22|1993-10-13|NONE|FOB|nic, pending ideas. bl|
+999|181|2|6|37|40003.66|0.00|0.04|R|F|1994-01-03|1993-10-28|1994-01-12|DELIVER IN PERSON|TRUCK|ckly slyly unusual packages: packages hagg|
+1024|199|2|1|49|53860.31|0.03|0.05|N|O|1998-03-06|1998-01-26|1998-03-29|TAKE BACK RETURN|FOB|ts. asymptotes nag fur|
+1024|126|5|2|34|34888.08|0.00|0.01|N|O|1998-01-06|1998-02-05|1998-01-26|COLLECT COD|SHIP|des the slyly even|
+1024|44|3|3|28|26433.12|0.04|0.01|N|O|1998-03-04|1998-03-12|1998-03-15|TAKE BACK RETURN|TRUCK|e blithely regular pi|
+1024|184|5|4|13|14094.34|0.02|0.04|N|O|1998-04-11|1998-02-26|1998-04-18|NONE|FOB|e slyly around the slyly special instructi|
+1024|21|4|5|49|45129.98|0.02|0.04|N|O|1998-02-27|1998-03-10|1998-03-27|COLLECT COD|FOB| carefully bold |
+1025|150|1|1|36|37805.40|0.03|0.04|A|F|1995-05-15|1995-07-05|1995-06-10|COLLECT COD|FOB|e unusual, regular instr|
+1025|69|10|2|23|22288.38|0.08|0.03|N|F|1995-06-02|1995-07-29|1995-06-23|COLLECT COD|RAIL| regular platelets nag carefu|
+1025|23|2|3|25|23075.50|0.06|0.05|R|F|1995-05-29|1995-06-21|1995-06-13|DELIVER IN PERSON|REG AIR|xpress foxes. furiousl|
+1026|38|4|1|36|33769.08|0.10|0.02|N|O|1997-06-14|1997-07-20|1997-06-23|NONE|SHIP|st the ide|
+1026|37|8|2|6|5622.18|0.10|0.08|N|O|1997-07-07|1997-08-16|1997-07-14|TAKE BACK RETURN|TRUCK|to beans. special, regular packages hagg|
+1027|156|1|1|43|45414.45|0.07|0.08|R|F|1992-06-17|1992-08-28|1992-07-10|DELIVER IN PERSON|MAIL|oxes. carefully regular deposits|
+1027|113|10|2|20|20262.20|0.01|0.02|A|F|1992-06-08|1992-08-29|1992-06-14|NONE|TRUCK|ar excuses eat f|
+1027|126|9|3|2|2052.24|0.01|0.02|R|F|1992-08-28|1992-07-09|1992-09-10|NONE|FOB|s. quickly unusual waters inside |
+1027|100|4|4|13|13001.30|0.08|0.01|R|F|1992-08-22|1992-07-10|1992-09-12|DELIVER IN PERSON|RAIL|ily ironic ideas use|
+1027|136|2|5|22|22794.86|0.02|0.00|A|F|1992-09-03|1992-08-14|1992-10-01|DELIVER IN PERSON|FOB|the furiously express ex|
+1027|105|8|6|10|10051.00|0.06|0.08|R|F|1992-08-28|1992-08-06|1992-09-03|COLLECT COD|REG AIR|ilent, express foxes near the blithely sp|
+1028|128|3|1|2|2056.24|0.09|0.03|A|F|1994-01-10|1994-03-22|1994-01-26|COLLECT COD|FOB|s alongside of the regular asymptotes sleep|
+1028|112|9|2|39|39472.29|0.06|0.05|R|F|1994-02-18|1994-03-22|1994-03-06|TAKE BACK RETURN|MAIL| final dependencies affix a|
+1028|100|3|3|8|8000.80|0.03|0.07|A|F|1994-02-14|1994-03-28|1994-02-22|NONE|AIR|e carefully final packages. furiously fi|
+1028|32|8|4|26|24232.78|0.07|0.02|A|F|1994-03-18|1994-02-08|1994-03-19|TAKE BACK RETURN|RAIL|ronic platelets. carefully f|
+1028|29|2|5|27|25083.54|0.00|0.04|A|F|1994-04-03|1994-02-07|1994-04-26|NONE|REG AIR|ial accounts nag. slyly|
+1028|26|1|6|39|36114.78|0.03|0.02|A|F|1994-02-27|1994-02-16|1994-03-02|DELIVER IN PERSON|AIR|c theodoli|
+1028|31|2|7|22|20482.66|0.03|0.00|R|F|1994-04-24|1994-02-27|1994-05-08|NONE|REG AIR| Tiresias alongside of the carefully spec|
+1029|137|3|1|45|46670.85|0.05|0.07|R|F|1994-07-21|1994-08-30|1994-07-29|TAKE BACK RETURN|FOB|sits boost blithely|
+1030|65|10|1|17|16406.02|0.06|0.06|R|F|1994-10-13|1994-08-01|1994-11-10|DELIVER IN PERSON|RAIL|ly. carefully even packages dazz|
+1031|46|7|1|15|14190.60|0.10|0.08|A|F|1994-11-07|1994-10-29|1994-11-09|TAKE BACK RETURN|FOB|about the carefully bold a|
+1031|165|4|2|28|29824.48|0.05|0.01|A|F|1994-12-10|1994-10-29|1994-12-18|COLLECT COD|FOB|ly ironic accounts across the q|
+1031|187|8|3|27|29353.86|0.07|0.02|R|F|1994-09-20|1994-10-18|1994-10-10|DELIVER IN PERSON|SHIP|gular deposits cajole. blithely unus|
+1031|88|9|4|7|6916.56|0.03|0.03|R|F|1994-12-07|1994-11-11|1994-12-30|COLLECT COD|FOB|r instructions. car|
+1031|191|5|5|44|48012.36|0.01|0.07|R|F|1994-11-20|1994-11-24|1994-12-11|NONE|AIR|re slyly above the furio|
+1056|121|6|1|37|37781.44|0.04|0.06|R|F|1995-02-18|1995-04-01|1995-03-20|NONE|TRUCK| special packages. qui|
+1057|193|5|1|29|31702.51|0.10|0.01|A|F|1992-05-05|1992-05-05|1992-06-03|TAKE BACK RETURN|SHIP|es wake according to the q|
+1057|169|8|2|11|11760.76|0.00|0.02|R|F|1992-03-31|1992-04-18|1992-04-18|COLLECT COD|AIR|yly final theodolites. furi|
+1057|85|6|3|21|20686.68|0.03|0.04|A|F|1992-02-28|1992-05-01|1992-03-10|NONE|REG AIR|ar orbits boost bli|
+1057|182|3|4|20|21643.60|0.06|0.03|R|F|1992-03-02|1992-05-19|1992-03-13|DELIVER IN PERSON|TRUCK|s wake bol|
+1057|97|1|5|7|6979.63|0.06|0.05|R|F|1992-06-05|1992-04-30|1992-06-20|NONE|TRUCK|y slyly express theodolites. slyly bo|
+1057|52|7|6|19|18088.95|0.04|0.07|A|F|1992-05-31|1992-05-09|1992-06-02|DELIVER IN PERSON|FOB|r-- packages haggle alon|
+1058|140|6|1|24|24963.36|0.08|0.04|A|F|1993-07-09|1993-05-28|1993-07-22|DELIVER IN PERSON|TRUCK|fully ironic accounts. express accou|
+1058|89|10|2|5|4945.40|0.04|0.07|R|F|1993-05-11|1993-05-29|1993-05-27|COLLECT COD|TRUCK|refully even requests boost along|
+1058|90|1|3|44|43563.96|0.10|0.01|R|F|1993-06-26|1993-06-21|1993-07-20|COLLECT COD|TRUCK|uriously f|
+1058|5|2|4|25|22625.00|0.09|0.01|A|F|1993-05-27|1993-06-10|1993-06-20|TAKE BACK RETURN|MAIL| the final requests believe carefully |
+1059|178|9|1|16|17250.72|0.07|0.02|A|F|1994-04-24|1994-03-31|1994-04-28|DELIVER IN PERSON|SHIP|y ironic pinto |
+1059|29|2|2|7|6503.14|0.07|0.06|R|F|1994-03-30|1994-04-01|1994-04-24|DELIVER IN PERSON|MAIL|the furiously silent excuses are e|
+1059|88|9|3|45|44463.60|0.00|0.02|R|F|1994-06-10|1994-05-08|1994-06-21|COLLECT COD|RAIL|riously even theodolites. slyly regula|
+1059|110|7|4|26|26262.86|0.09|0.01|A|F|1994-03-17|1994-04-18|1994-03-26|DELIVER IN PERSON|TRUCK|ar pinto beans at the furiously |
+1059|139|5|5|37|38447.81|0.09|0.04|R|F|1994-03-31|1994-05-08|1994-04-06|COLLECT COD|RAIL| packages lose in place of the slyly unusu|
+1059|190|1|6|50|54509.50|0.00|0.03|A|F|1994-06-15|1994-05-11|1994-06-29|NONE|MAIL|s impress furiously about|
+1059|123|4|7|13|13300.56|0.01|0.03|R|F|1994-06-12|1994-05-11|1994-07-02|COLLECT COD|TRUCK|usly regular theodo|
+1060|196|10|1|8|8769.52|0.07|0.04|R|F|1993-05-21|1993-05-06|1993-06-10|DELIVER IN PERSON|FOB|iously. furiously regular in|
+1060|8|5|2|26|23608.00|0.06|0.08|R|F|1993-04-12|1993-04-01|1993-04-20|DELIVER IN PERSON|TRUCK|counts; even deposits are carefull|
+1060|164|3|3|11|11705.76|0.01|0.07|A|F|1993-05-13|1993-05-08|1993-05-17|TAKE BACK RETURN|MAIL|e regular deposits: re|
+1060|110|7|4|16|16161.76|0.03|0.06|A|F|1993-06-15|1993-04-18|1993-07-05|COLLECT COD|SHIP|ccounts. foxes maintain care|
+1060|53|8|5|1|953.05|0.04|0.06|A|F|1993-06-19|1993-05-10|1993-06-21|COLLECT COD|RAIL|posits detect carefully abo|
+1060|72|2|6|26|25273.82|0.01|0.03|A|F|1993-02-28|1993-04-01|1993-03-09|TAKE BACK RETURN|FOB|quickly abo|
+1060|121|10|7|36|36760.32|0.09|0.01|R|F|1993-03-14|1993-03-24|1993-04-02|TAKE BACK RETURN|FOB|r the quickly|
+1061|151|6|1|7|7358.05|0.04|0.04|N|O|1998-08-09|1998-08-12|1998-08-16|COLLECT COD|FOB|es are slyly expr|
+1061|119|3|2|2|2038.22|0.06|0.02|N|O|1998-08-15|1998-08-05|1998-08-24|COLLECT COD|MAIL|. regular accounts impre|
+1061|111|8|3|26|26288.86|0.08|0.02|N|O|1998-06-18|1998-07-25|1998-06-24|TAKE BACK RETURN|AIR|ave to slee|
+1061|136|7|4|41|42481.33|0.00|0.05|N|O|1998-06-29|1998-07-02|1998-07-27|NONE|MAIL|s are. ironic theodolites cajole. dep|
+1061|131|2|5|50|51556.50|0.04|0.08|N|O|1998-05-25|1998-07-22|1998-06-22|COLLECT COD|AIR|nding excuses are around the e|
+1061|144|1|6|35|36544.90|0.09|0.05|N|O|1998-07-05|1998-07-07|1998-07-30|TAKE BACK RETURN|MAIL|ending requests nag careful|
+1062|137|8|1|38|39410.94|0.00|0.01|N|O|1997-01-27|1997-03-07|1997-02-16|DELIVER IN PERSON|TRUCK|deas. pending acc|
+1063|96|9|1|42|41835.78|0.03|0.02|A|F|1994-07-10|1994-05-25|1994-07-26|NONE|RAIL|tructions about the blithely ex|
+1088|107|8|1|30|30213.00|0.07|0.03|A|F|1992-05-22|1992-06-25|1992-06-11|TAKE BACK RETURN|SHIP|long the packages snooze careful|
+1088|37|3|2|11|10307.33|0.06|0.00|A|F|1992-08-30|1992-07-25|1992-09-10|TAKE BACK RETURN|AIR|inal requests. fluffily express theod|
+1088|181|2|3|5|5405.90|0.03|0.07|A|F|1992-07-01|1992-07-25|1992-07-02|NONE|AIR|refully ironic packages. r|
+1088|124|5|4|3|3072.36|0.09|0.03|A|F|1992-06-15|1992-08-02|1992-06-18|DELIVER IN PERSON|MAIL|pecial theodolites |
+1089|151|3|1|47|49404.05|0.05|0.06|N|O|1996-06-26|1996-06-25|1996-07-11|NONE|TRUCK|aggle furiously among the bravely eve|
+1089|50|7|2|35|33251.75|0.03|0.00|N|O|1996-08-14|1996-07-10|1996-08-26|NONE|TRUCK|ly express deposits haggle|
+1089|26|7|3|23|21298.46|0.10|0.05|N|O|1996-06-24|1996-07-25|1996-07-20|DELIVER IN PERSON|AIR|g dolphins. deposits integrate. s|
+1089|141|10|4|1|1041.14|0.01|0.03|N|O|1996-07-08|1996-07-07|1996-07-17|COLLECT COD|RAIL|n courts among the caref|
+1090|22|3|1|5|4610.10|0.02|0.05|N|O|1998-02-19|1997-12-25|1998-02-24|DELIVER IN PERSON|AIR|s above the |
+1090|113|10|2|28|28367.08|0.08|0.08|N|O|1998-02-20|1998-01-03|1998-03-19|NONE|FOB|s cajole above the regular|
+1091|38|9|1|40|37521.20|0.10|0.06|N|O|1996-12-17|1996-10-14|1996-12-24|TAKE BACK RETURN|REG AIR|platelets. regular packag|
+1092|184|5|1|48|52040.64|0.04|0.04|N|O|1995-06-25|1995-04-06|1995-07-18|DELIVER IN PERSON|AIR|unusual accounts. fluffi|
+1092|153|5|2|1|1053.15|0.01|0.06|A|F|1995-03-10|1995-04-21|1995-04-06|COLLECT COD|RAIL|lent, pending requests-- requests nag accor|
+1092|161|8|3|28|29712.48|0.05|0.08|R|F|1995-04-08|1995-05-01|1995-05-02|DELIVER IN PERSON|FOB|affix carefully. u|
+1092|86|7|4|2|1972.16|0.05|0.07|R|F|1995-04-09|1995-05-12|1995-05-03|TAKE BACK RETURN|TRUCK|ans. slyly eve|
+1093|87|8|1|7|6909.56|0.04|0.02|N|O|1997-11-24|1997-09-23|1997-11-25|TAKE BACK RETURN|SHIP|bold deposits. blithely ironic depos|
+1093|177|5|2|37|39855.29|0.08|0.04|N|O|1997-11-06|1997-10-08|1997-11-22|COLLECT COD|FOB|le furiously across the carefully sp|
+1093|61|2|3|34|32676.04|0.01|0.06|N|O|1997-11-07|1997-09-06|1997-11-28|TAKE BACK RETURN|REG AIR|sits. express accounts play carefully. bol|
+1094|115|6|1|9|9135.99|0.07|0.06|N|O|1997-12-28|1998-03-16|1998-01-18|DELIVER IN PERSON|AIR|as. slyly pe|
+1095|137|3|1|33|34225.29|0.01|0.02|N|O|1995-10-03|1995-09-22|1995-10-13|NONE|MAIL|slyly around the iron|
+1095|136|2|2|24|24867.12|0.04|0.06|N|O|1995-08-24|1995-10-20|1995-09-09|COLLECT COD|TRUCK|packages nod furiously above the carefully |
+1095|156|4|3|13|13729.95|0.06|0.01|N|O|1995-08-24|1995-10-19|1995-09-02|TAKE BACK RETURN|REG AIR|ously even accounts. slyly bold a|
+1095|135|1|4|28|28983.64|0.08|0.03|N|O|1995-09-20|1995-11-18|1995-10-02|DELIVER IN PERSON|SHIP| regular pac|
+1095|112|2|5|40|40484.40|0.09|0.03|N|O|1995-10-18|1995-11-14|1995-11-09|NONE|MAIL| bold accounts haggle slyly furiously even|
+1095|181|2|6|37|40003.66|0.07|0.08|N|O|1995-10-04|1995-11-13|1995-10-12|NONE|SHIP|. quickly even dolphins sle|
+1120|178|8|1|10|10781.70|0.08|0.05|N|O|1997-12-17|1998-01-21|1997-12-23|DELIVER IN PERSON|MAIL|dependencies. blithel|
+1120|20|1|2|49|45080.98|0.01|0.07|N|O|1998-01-03|1998-02-02|1998-01-09|TAKE BACK RETURN|RAIL|heodolites. quick re|
+1120|76|6|3|21|20497.47|0.06|0.01|N|O|1998-01-11|1998-02-04|1998-01-19|COLLECT COD|REG AIR|s: fluffily even packages c|
+1120|46|9|4|22|20812.88|0.09|0.08|N|O|1997-11-15|1998-01-25|1997-12-07|TAKE BACK RETURN|REG AIR|ons. slyly silent requests sleep silent|
+1120|83|4|5|10|9830.80|0.07|0.08|N|O|1997-11-10|1998-02-01|1997-11-28|TAKE BACK RETURN|AIR|ages haggle furiously |
+1121|168|3|1|42|44862.72|0.04|0.05|N|O|1997-03-05|1997-03-18|1997-03-14|DELIVER IN PERSON|SHIP|nts are slyly special packages. f|
+1121|161|10|2|27|28651.32|0.08|0.00|N|O|1997-05-08|1997-03-28|1997-05-14|NONE|MAIL|ly ironic accounts cajole slyly abou|
+1121|157|5|3|10|10571.50|0.00|0.04|N|O|1997-04-17|1997-03-18|1997-05-02|TAKE BACK RETURN|RAIL|dencies. quickly regular theodolites n|
+1121|166|1|4|29|30918.64|0.02|0.01|N|O|1997-03-07|1997-04-02|1997-04-01|DELIVER IN PERSON|REG AIR| use furiously. quickly silent package|
+1121|30|9|5|47|43711.41|0.09|0.03|N|O|1997-04-27|1997-03-28|1997-05-14|COLLECT COD|FOB|ly idle, i|
+1121|200|1|6|50|55010.00|0.06|0.03|N|O|1997-04-21|1997-02-16|1997-04-25|NONE|TRUCK|odolites. slyly even accounts|
+1121|80|8|7|37|36262.96|0.06|0.01|N|O|1997-02-27|1997-03-04|1997-03-02|COLLECT COD|RAIL|special packages. fluffily final requests s|
+1122|92|6|1|8|7936.72|0.10|0.06|N|O|1997-02-02|1997-04-03|1997-02-22|TAKE BACK RETURN|RAIL|c foxes are along the slyly r|
+1122|182|3|2|29|31383.22|0.05|0.04|N|O|1997-05-07|1997-04-07|1997-05-15|COLLECT COD|SHIP|ptotes. quickl|
+1122|147|6|3|25|26178.50|0.09|0.01|N|O|1997-03-21|1997-03-03|1997-04-07|TAKE BACK RETURN|RAIL|d furiously. pinto |
+1122|106|9|4|40|40244.00|0.08|0.08|N|O|1997-02-07|1997-03-25|1997-02-25|NONE|REG AIR|packages sleep after the asym|
+1122|151|2|5|15|15767.25|0.05|0.03|N|O|1997-04-15|1997-03-15|1997-05-07|COLLECT COD|SHIP|olve blithely regular, |
+1122|162|7|6|24|25491.84|0.04|0.01|N|O|1997-03-08|1997-02-20|1997-04-05|NONE|RAIL|blithely requests. slyly pending r|
+1122|1|6|7|38|34238.00|0.00|0.08|N|O|1997-01-23|1997-04-02|1997-02-16|NONE|TRUCK|t theodolites sleep. even, ironic|
+1123|12|2|1|10|9120.10|0.05|0.08|N|O|1996-11-12|1996-10-04|1996-11-30|NONE|MAIL|ckages are above the depths. slyly ir|
+1123|178|8|2|39|42048.63|0.03|0.08|N|O|1996-08-25|1996-10-21|1996-09-04|DELIVER IN PERSON|REG AIR|rding to the furiously ironic requests: r|
+1123|101|4|3|38|38041.80|0.03|0.08|N|O|1996-09-23|1996-10-04|1996-09-27|DELIVER IN PERSON|FOB| blithely carefully unusual reques|
+1124|198|2|1|1|1098.19|0.09|0.08|N|O|1998-10-06|1998-10-02|1998-10-30|NONE|REG AIR| instructions cajole qu|
+1124|6|1|2|13|11778.00|0.05|0.04|N|O|1998-09-05|1998-10-03|1998-09-30|DELIVER IN PERSON|SHIP|t the slyly |
+1124|93|5|3|35|34758.15|0.10|0.05|N|O|1998-11-25|1998-10-08|1998-12-25|TAKE BACK RETURN|AIR|ut the slyly bold pinto beans; fi|
+1124|50|1|4|25|23751.25|0.08|0.05|N|O|1998-08-05|1998-10-14|1998-08-11|NONE|MAIL|ggle slyly according|
+1124|75|5|5|33|32177.31|0.05|0.04|N|O|1998-10-19|1998-09-17|1998-10-26|TAKE BACK RETURN|SHIP|eposits sleep slyly. stealthily f|
+1124|27|6|6|43|39861.86|0.01|0.03|N|O|1998-09-19|1998-10-28|1998-10-10|COLLECT COD|MAIL|across the |
+1124|95|6|7|1|995.09|0.09|0.01|N|O|1998-10-07|1998-08-31|1998-10-12|NONE|TRUCK|ly bold accou|
+1125|133|4|1|4|4132.52|0.08|0.02|A|F|1994-12-10|1994-12-28|1994-12-30|NONE|MAIL| quickly express packages a|
+1125|138|9|2|24|24915.12|0.10|0.03|R|F|1995-01-31|1994-12-02|1995-02-20|COLLECT COD|AIR|es about the slyly s|
+1125|122|7|3|26|26575.12|0.05|0.04|A|F|1995-02-24|1995-01-18|1995-03-05|COLLECT COD|TRUCK|l instruction|
+1125|98|1|4|29|28944.61|0.06|0.00|A|F|1994-11-29|1994-12-20|1994-12-10|DELIVER IN PERSON|RAIL| platelets wake against the carefully i|
+1126|36|2|1|44|41185.32|0.08|0.03|N|O|1998-05-07|1998-04-02|1998-05-29|NONE|TRUCK|es. carefully special|
+1126|58|3|2|7|6706.35|0.06|0.01|N|O|1998-05-02|1998-03-22|1998-05-21|COLLECT COD|MAIL|ons. final, unusual|
+1126|147|10|3|14|14659.96|0.07|0.07|N|O|1998-04-17|1998-04-15|1998-05-12|DELIVER IN PERSON|TRUCK|nstructions. blithe|
+1127|43|10|1|35|33006.40|0.02|0.03|N|O|1995-11-25|1995-11-03|1995-12-17|NONE|TRUCK|l instructions boost blithely according |
+1127|110|5|2|38|38384.18|0.09|0.05|N|O|1995-11-07|1995-11-11|1995-11-26|DELIVER IN PERSON|RAIL|. never final packages boost acro|
+1127|20|1|3|29|26680.58|0.09|0.07|N|O|1995-09-20|1995-11-21|1995-10-11|DELIVER IN PERSON|REG AIR|y. blithely r|
+1127|175|6|4|7|7526.19|0.07|0.05|N|O|1995-11-05|1995-11-02|1995-11-11|DELIVER IN PERSON|FOB| idly pending pains |
+1152|9|10|1|23|20907.00|0.06|0.04|A|F|1994-10-14|1994-10-22|1994-10-21|DELIVER IN PERSON|MAIL|equests alongside of the unusual |
+1152|100|2|2|25|25002.50|0.04|0.08|R|F|1994-10-20|1994-09-18|1994-10-28|DELIVER IN PERSON|REG AIR|efully ironic accounts. sly instructions wa|
+1152|42|9|3|6|5652.24|0.07|0.03|A|F|1994-12-07|1994-11-05|1994-12-25|DELIVER IN PERSON|FOB|p furiously; packages above th|
+1153|86|7|1|15|14791.20|0.00|0.08|N|O|1996-04-24|1996-07-17|1996-04-29|TAKE BACK RETURN|SHIP|uctions boost fluffily according to|
+1153|169|8|2|50|53458.00|0.00|0.07|N|O|1996-06-27|1996-07-13|1996-07-05|COLLECT COD|REG AIR|ronic asymptotes nag slyly. |
+1153|44|5|3|25|23601.00|0.00|0.05|N|O|1996-06-18|1996-06-28|1996-07-09|NONE|TRUCK| theodolites|
+1153|92|3|4|43|42659.87|0.01|0.00|N|O|1996-06-09|1996-06-01|1996-07-04|DELIVER IN PERSON|MAIL|special instructions are. unusual, final du|
+1153|142|5|5|45|46896.30|0.00|0.02|N|O|1996-06-18|1996-06-20|1996-07-03|TAKE BACK RETURN|AIR|oss the ex|
+1153|136|7|6|26|26939.38|0.02|0.03|N|O|1996-08-16|1996-07-12|1996-09-08|NONE|MAIL|kages haggle carefully. f|
+1153|192|4|7|5|5460.95|0.02|0.03|N|O|1996-05-03|1996-06-12|1996-05-28|TAKE BACK RETURN|FOB|special excuses promi|
+1154|143|10|1|31|32337.34|0.06|0.06|A|F|1992-04-17|1992-04-26|1992-05-17|COLLECT COD|AIR|ithely. final, blithe |
+1154|148|7|2|50|52407.00|0.07|0.06|A|F|1992-04-22|1992-04-21|1992-05-01|NONE|TRUCK|ove the furiously bold Tires|
+1154|97|1|3|5|4985.45|0.09|0.04|A|F|1992-06-07|1992-05-07|1992-07-05|DELIVER IN PERSON|MAIL|the furiously |
+1154|1|2|4|35|31535.00|0.00|0.07|A|F|1992-03-30|1992-04-02|1992-04-21|DELIVER IN PERSON|TRUCK|the carefully regular pinto beans boost|
+1154|36|2|5|18|16848.54|0.02|0.03|A|F|1992-02-26|1992-03-24|1992-03-20|TAKE BACK RETURN|REG AIR|y regular excuses cajole blithely. fi|
+1154|196|8|6|50|54809.50|0.06|0.03|A|F|1992-03-04|1992-04-01|1992-04-01|TAKE BACK RETURN|TRUCK| even, special |
+1155|70|1|1|4|3880.28|0.09|0.05|N|O|1997-10-19|1997-12-09|1997-11-02|DELIVER IN PERSON|SHIP|ic foxes according to the carefully final |
+1155|196|9|2|39|42751.41|0.08|0.05|N|O|1998-01-29|1998-01-03|1998-02-01|COLLECT COD|TRUCK|ckly final pinto beans was.|
+1155|147|4|3|23|24084.22|0.08|0.03|N|O|1997-11-24|1997-11-28|1997-12-06|DELIVER IN PERSON|FOB|ly unusual packages. iro|
+1155|140|1|4|12|12481.68|0.01|0.06|N|O|1997-11-01|1998-01-03|1997-11-19|DELIVER IN PERSON|RAIL|packages do|
+1155|5|2|5|49|44345.00|0.04|0.08|N|O|1997-12-07|1997-12-30|1997-12-08|NONE|AIR|ccounts are alongside of t|
+1156|87|8|1|15|14806.20|0.07|0.06|N|O|1996-12-21|1997-01-03|1997-01-10|TAKE BACK RETURN|AIR|the furiously pen|
+1156|33|4|2|21|19593.63|0.02|0.08|N|O|1996-11-07|1997-01-14|1996-12-03|NONE|AIR|dolphins. fluffily ironic packages sleep re|
+1156|12|2|3|29|26448.29|0.09|0.06|N|O|1997-01-24|1996-12-26|1997-02-04|DELIVER IN PERSON|TRUCK|ts sleep sly|
+1156|172|3|4|42|45031.14|0.02|0.00|N|O|1997-01-18|1997-01-12|1997-02-13|NONE|REG AIR|s. quickly bold pains are|
+1156|74|4|5|49|47729.43|0.04|0.01|N|O|1996-11-16|1996-12-02|1996-12-05|COLLECT COD|AIR|ithely unusual in|
+1156|195|9|6|42|45997.98|0.02|0.06|N|O|1997-01-27|1997-01-09|1997-01-28|DELIVER IN PERSON|MAIL|even requests boost ironic deposits. pe|
+1156|47|6|7|20|18940.80|0.08|0.07|N|O|1997-01-01|1997-01-06|1997-01-16|COLLECT COD|MAIL|deposits sleep bravel|
+1157|49|2|1|16|15184.64|0.06|0.00|N|O|1998-04-12|1998-03-09|1998-04-23|DELIVER IN PERSON|MAIL|tions hang|
+1157|83|4|2|4|3932.32|0.10|0.05|N|O|1998-02-24|1998-03-30|1998-03-24|DELIVER IN PERSON|SHIP|ounts. ironic deposits|
+1157|48|7|3|8|7584.32|0.02|0.00|N|O|1998-03-25|1998-03-16|1998-03-29|NONE|REG AIR|blithely even pa|
+1157|77|8|4|46|44945.22|0.07|0.08|N|O|1998-04-19|1998-03-13|1998-04-23|NONE|FOB|slyly regular excuses. accounts|
+1157|160|5|5|14|14842.24|0.03|0.03|N|O|1998-04-17|1998-03-03|1998-05-01|NONE|FOB|theodolites. fluffily re|
+1158|45|2|1|5|4725.20|0.02|0.04|N|O|1996-10-20|1996-07-30|1996-11-14|COLLECT COD|AIR|symptotes along the care|
+1158|157|9|2|23|24314.45|0.00|0.08|N|O|1996-10-21|1996-08-19|1996-10-31|COLLECT COD|MAIL|ularly ironic requests use care|
+1159|109|10|1|39|39354.90|0.01|0.00|A|F|1992-11-20|1992-10-28|1992-12-18|TAKE BACK RETURN|FOB| blithely express reques|
+1159|96|9|2|7|6972.63|0.08|0.00|A|F|1992-11-25|1992-10-27|1992-12-20|NONE|AIR|olve somet|
+1159|98|10|3|11|10978.99|0.10|0.03|R|F|1992-12-09|1992-12-07|1992-12-18|DELIVER IN PERSON|MAIL|h furiousl|
+1184|47|4|1|27|25570.08|0.01|0.00|N|O|1998-01-10|1997-12-02|1998-02-06|TAKE BACK RETURN|REG AIR|s wake fluffily. fl|
+1184|147|10|2|4|4188.56|0.04|0.03|N|O|1997-12-25|1998-01-24|1998-01-18|DELIVER IN PERSON|RAIL| express packages. slyly expres|
+1184|164|5|3|7|7449.12|0.05|0.00|N|O|1998-02-14|1998-01-06|1998-03-11|COLLECT COD|TRUCK|ckly warthogs. blithely bold foxes hag|
+1184|126|9|4|3|3078.36|0.02|0.05|N|O|1998-01-15|1997-12-19|1998-02-02|NONE|REG AIR|ar packages. final packages cajol|
+1185|72|1|1|8|7776.56|0.01|0.06|A|F|1992-12-05|1992-10-05|1992-12-28|DELIVER IN PERSON|MAIL|ely according to the furiously regular r|
+1185|31|2|2|28|26068.84|0.07|0.06|A|F|1992-09-24|1992-10-07|1992-10-10|DELIVER IN PERSON|REG AIR|ke. slyly regular t|
+1185|190|1|3|12|13082.28|0.05|0.06|R|F|1992-10-12|1992-09-26|1992-11-11|NONE|REG AIR|instructions. daringly pend|
+1186|3|4|1|28|25284.00|0.08|0.07|N|O|1996-12-08|1996-10-17|1996-12-15|TAKE BACK RETURN|TRUCK|ffily spec|
+1186|92|5|2|11|10912.99|0.07|0.05|N|O|1996-10-03|1996-10-21|1996-10-17|DELIVER IN PERSON|AIR|s haggle furiously; slyl|
+1186|101|2|3|20|20022.00|0.07|0.07|N|O|1996-08-20|1996-10-23|1996-09-05|COLLECT COD|FOB|ely alongside of the blithel|
+1186|106|7|4|27|27164.70|0.06|0.04|N|O|1996-10-08|1996-11-06|1996-10-09|TAKE BACK RETURN|SHIP|accounts. express, e|
+1187|178|6|1|29|31266.93|0.01|0.04|R|F|1992-12-10|1993-02-09|1992-12-29|TAKE BACK RETURN|RAIL|riously express ac|
+1187|131|7|2|15|15466.95|0.03|0.04|A|F|1992-12-22|1993-01-13|1993-01-01|NONE|TRUCK|ests. foxes wake. carefu|
+1187|78|8|3|40|39122.80|0.08|0.06|R|F|1993-03-05|1992-12-31|1993-03-12|NONE|TRUCK|ar, brave deposits nag blithe|
+1188|115|9|1|2|2030.22|0.00|0.04|N|O|1996-05-22|1996-05-23|1996-06-06|COLLECT COD|RAIL|its breach blit|
+1188|113|4|2|9|9117.99|0.01|0.08|N|O|1996-08-04|1996-06-04|1996-08-19|NONE|REG AIR|ow carefully ironic d|
+1188|179|10|3|41|44245.97|0.07|0.04|N|O|1996-06-29|1996-05-21|1996-07-21|TAKE BACK RETURN|TRUCK|althy packages. fluffily unusual ideas h|
+1189|51|2|1|23|21874.15|0.06|0.00|R|F|1994-07-25|1994-06-07|1994-08-02|COLLECT COD|FOB|s. fluffy Tiresias run quickly. bra|
+1189|105|2|2|32|32163.20|0.09|0.02|R|F|1994-05-06|1994-07-03|1994-05-15|TAKE BACK RETURN|FOB|e regular deposits. quickly quiet deposi|
+1189|57|5|3|22|21055.10|0.05|0.03|R|F|1994-06-09|1994-06-29|1994-06-23|DELIVER IN PERSON|TRUCK|quickly unusual platelets lose forges. ca|
+1190|84|5|1|32|31490.56|0.07|0.06|N|O|1997-05-08|1997-04-17|1997-06-01|COLLECT COD|FOB|y final packages? slyly even|
+1191|49|6|1|29|27522.16|0.00|0.04|N|O|1996-01-24|1996-01-28|1996-02-17|COLLECT COD|AIR| regular pin|
+1216|97|1|1|8|7976.72|0.03|0.04|R|F|1993-02-01|1993-03-06|1993-02-08|TAKE BACK RETURN|TRUCK| of the carefully express|
+1216|75|3|2|48|46803.36|0.10|0.01|R|F|1993-01-17|1993-02-01|1993-02-13|COLLECT COD|SHIP|symptotes use against th|
+1216|42|3|3|18|16956.72|0.00|0.03|A|F|1993-01-20|1993-01-28|1993-02-02|COLLECT COD|MAIL|y final packages nod |
+1217|60|5|1|45|43202.70|0.07|0.02|A|F|1992-07-01|1992-06-23|1992-07-06|COLLECT COD|AIR|riously close ideas|
+1218|140|6|1|16|16642.24|0.04|0.07|A|F|1994-06-26|1994-08-07|1994-06-30|TAKE BACK RETURN|FOB|ven realms be|
+1218|94|6|2|41|40757.69|0.06|0.06|R|F|1994-08-04|1994-08-05|1994-08-11|TAKE BACK RETURN|SHIP|dolphins. theodolites beyond th|
+1218|48|7|3|44|41713.76|0.07|0.06|A|F|1994-10-05|1994-09-03|1994-10-30|COLLECT COD|TRUCK|thely ironic accounts wake slyly|
+1218|42|9|4|1|942.04|0.01|0.08|R|F|1994-09-15|1994-09-07|1994-10-03|COLLECT COD|TRUCK|press furio|
+1219|132|3|1|6|6192.78|0.08|0.04|N|O|1995-11-13|1995-12-24|1995-11-18|NONE|MAIL|pecial, ironic requ|
+1219|129|4|2|4|4116.48|0.01|0.04|N|O|1995-11-24|1995-11-22|1995-12-07|TAKE BACK RETURN|SHIP|lly quick requests. blithely even h|
+1220|169|4|1|25|26729.00|0.10|0.03|N|O|1996-10-15|1996-11-07|1996-11-06|COLLECT COD|REG AIR| regular orbi|
+1220|160|5|2|36|38165.76|0.01|0.02|N|O|1996-12-10|1996-11-14|1997-01-07|COLLECT COD|SHIP|ar packages. blithely final acc|
+1220|37|8|3|3|2811.09|0.08|0.06|N|O|1996-09-06|1996-11-03|1996-09-10|COLLECT COD|REG AIR| final theodolites. blithely silent |
+1220|6|1|4|36|32616.00|0.07|0.03|N|O|1996-12-12|1996-10-03|1996-12-15|TAKE BACK RETURN|TRUCK|unusual, silent pinto beans aga|
+1220|49|2|5|25|23726.00|0.03|0.08|N|O|1996-09-11|1996-10-09|1996-09-25|DELIVER IN PERSON|RAIL|packages affi|
+1221|81|2|1|43|42186.44|0.05|0.05|R|F|1992-06-22|1992-07-15|1992-07-20|DELIVER IN PERSON|FOB|y slyly above the slyly unusual ideas|
+1221|170|1|2|12|12842.04|0.00|0.08|R|F|1992-08-07|1992-06-24|1992-08-13|COLLECT COD|AIR|yly ironic |
+1221|69|6|3|3|2907.18|0.10|0.08|R|F|1992-07-01|1992-06-04|1992-07-27|COLLECT COD|TRUCK|ing to the fluffily|
+1221|120|10|4|41|41824.92|0.06|0.02|A|F|1992-04-28|1992-07-02|1992-05-19|NONE|RAIL|ns. bold deposit|
+1221|108|1|5|13|13105.30|0.10|0.00|R|F|1992-08-01|1992-06-29|1992-08-27|TAKE BACK RETURN|AIR|ajole furiously. blithely expres|
+1221|85|6|6|7|6895.56|0.08|0.06|A|F|1992-06-27|1992-06-16|1992-07-23|TAKE BACK RETURN|RAIL|xpress accounts |
+1222|72|10|1|12|11664.84|0.09|0.02|A|F|1993-02-12|1993-03-14|1993-03-12|TAKE BACK RETURN|RAIL|s print permanently unusual packages. |
+1222|159|7|2|12|12709.80|0.08|0.01|A|F|1993-05-05|1993-03-27|1993-05-18|TAKE BACK RETURN|REG AIR| furiously bold instructions|
+1222|8|1|3|26|23608.00|0.02|0.08|R|F|1993-02-13|1993-03-20|1993-02-22|TAKE BACK RETURN|MAIL|, even accounts are ironic|
+1223|100|1|1|28|28002.80|0.10|0.06|N|O|1996-08-07|1996-07-24|1996-08-13|TAKE BACK RETURN|MAIL| quickly ironic requests. furious|
+1248|164|5|1|45|47887.20|0.00|0.08|A|F|1992-04-17|1992-03-31|1992-05-13|NONE|RAIL|ter the pending pl|
+1248|151|9|2|37|38892.55|0.06|0.06|R|F|1992-01-26|1992-02-05|1992-02-13|COLLECT COD|TRUCK|. final requests integrate quickly. blit|
+1248|56|8|3|26|24857.30|0.09|0.06|A|F|1992-01-16|1992-03-01|1992-02-06|TAKE BACK RETURN|AIR| ironic dependen|
+1248|156|7|4|49|51751.35|0.02|0.01|A|F|1992-04-24|1992-02-18|1992-05-03|TAKE BACK RETURN|AIR|beans run quickly according to the carefu|
+1248|122|7|5|20|20442.40|0.08|0.00|A|F|1992-03-12|1992-03-23|1992-04-07|TAKE BACK RETURN|AIR|nal foxes cajole carefully slyl|
+1248|62|9|6|30|28861.80|0.10|0.01|R|F|1992-02-01|1992-03-24|1992-02-08|TAKE BACK RETURN|MAIL|fily special foxes kindle am|
+1249|59|4|1|49|46993.45|0.07|0.05|A|F|1994-03-03|1994-02-28|1994-03-08|NONE|RAIL|ffily express theodo|
+1250|2|3|1|15|13530.00|0.10|0.06|A|F|1992-11-05|1992-12-17|1992-12-03|TAKE BACK RETURN|SHIP| regular, i|
+1251|4|5|1|37|33448.00|0.08|0.08|N|O|1997-12-21|1998-01-12|1997-12-26|COLLECT COD|AIR|. furiously|
+1251|78|9|2|36|35210.52|0.07|0.04|N|O|1997-11-29|1998-01-07|1997-12-03|TAKE BACK RETURN|RAIL|y ironic Tiresias are slyly furio|
+1251|99|3|3|37|36966.33|0.09|0.02|N|O|1998-01-11|1997-12-01|1998-01-23|DELIVER IN PERSON|RAIL|finally bold requests|
+1251|150|9|4|7|7351.05|0.07|0.00|N|O|1998-01-08|1997-12-27|1998-01-18|COLLECT COD|MAIL|riously pe|
+1251|188|9|5|1|1088.18|0.02|0.03|N|O|1997-12-08|1998-01-06|1998-01-01|DELIVER IN PERSON|REG AIR| use quickly final packages. iron|
+1252|87|8|1|13|12832.04|0.10|0.01|N|O|1997-09-07|1997-09-12|1997-10-01|COLLECT COD|REG AIR|sts dazzle|
+1252|111|8|2|27|27299.97|0.00|0.08|N|O|1997-10-22|1997-10-10|1997-11-10|TAKE BACK RETURN|REG AIR|packages hag|
+1252|40|1|3|19|17860.76|0.07|0.02|N|O|1997-10-13|1997-10-23|1997-10-18|NONE|AIR|ts wake carefully-- packages sleep. quick |
+1252|92|4|4|11|10912.99|0.10|0.01|N|O|1997-10-16|1997-09-22|1997-10-28|COLLECT COD|AIR|s are. slyly final requests among the|
+1252|79|10|5|26|25455.82|0.05|0.05|N|O|1997-08-05|1997-10-24|1997-08-07|DELIVER IN PERSON|SHIP|onic pinto beans haggle furiously |
+1253|180|8|1|14|15122.52|0.00|0.06|R|F|1993-04-03|1993-04-16|1993-04-27|TAKE BACK RETURN|MAIL|lar foxes sleep furiously final, final pack|
+1253|54|9|2|13|12402.65|0.01|0.06|A|F|1993-03-05|1993-04-26|1993-03-08|DELIVER IN PERSON|FOB|al packages|
+1253|70|1|3|22|21341.54|0.05|0.06|A|F|1993-02-23|1993-04-06|1993-03-07|TAKE BACK RETURN|SHIP|telets cajole alongside of the final reques|
+1253|176|5|4|23|24751.91|0.09|0.02|R|F|1993-04-18|1993-04-18|1993-05-07|COLLECT COD|FOB| the slyly silent re|
+1253|114|8|5|19|19268.09|0.05|0.05|A|F|1993-04-01|1993-04-22|1993-04-14|TAKE BACK RETURN|AIR|al pinto bea|
+1254|193|5|1|6|6559.14|0.08|0.01|N|O|1996-02-02|1996-03-21|1996-02-29|NONE|REG AIR|lithely even deposits eat!|
+1254|200|3|2|47|51709.40|0.05|0.06|N|O|1996-03-07|1996-02-20|1996-04-05|COLLECT COD|MAIL| platelets cajol|
+1254|135|6|3|35|36229.55|0.05|0.06|N|O|1996-04-08|1996-02-29|1996-04-18|DELIVER IN PERSON|FOB|ckages boost. furious warhorses cajole|
+1255|192|4|1|12|13106.28|0.00|0.02|A|F|1994-08-17|1994-06-29|1994-09-04|TAKE BACK RETURN|REG AIR| regular, express accounts are |
+1255|194|8|2|46|50332.74|0.07|0.05|R|F|1994-07-06|1994-07-14|1994-08-05|NONE|MAIL|ons nag qui|
+1280|129|8|1|17|17495.04|0.01|0.01|A|F|1993-02-04|1993-04-10|1993-02-07|NONE|FOB|ructions integrate across the th|
+1280|189|10|2|6|6535.08|0.05|0.06|R|F|1993-03-30|1993-02-16|1993-04-18|DELIVER IN PERSON|AIR|gular deposits |
+1280|33|4|3|13|12129.39|0.03|0.02|R|F|1993-03-06|1993-03-11|1993-03-18|DELIVER IN PERSON|TRUCK|blithely final accounts use evenly |
+1280|175|3|4|5|5375.85|0.06|0.03|R|F|1993-02-03|1993-02-11|1993-02-23|DELIVER IN PERSON|AIR|beans haggle. quickly bold instructions h|
+1280|52|10|5|24|22849.20|0.07|0.02|R|F|1993-03-20|1993-03-01|1993-04-09|COLLECT COD|RAIL|y pending orbits boost after the slyly|
+1280|66|3|6|9|8694.54|0.00|0.05|R|F|1993-04-18|1993-03-28|1993-05-04|DELIVER IN PERSON|FOB|usual accou|
+1280|92|6|7|19|18849.71|0.02|0.06|A|F|1993-02-07|1993-02-28|1993-02-12|NONE|TRUCK|lyly along the furiously regular |
+1281|138|4|1|33|34258.29|0.07|0.08|R|F|1995-02-01|1995-01-18|1995-03-03|NONE|REG AIR|dencies. thinly final pinto beans wake|
+1281|7|2|2|37|33559.00|0.08|0.03|A|F|1995-03-19|1995-02-02|1995-03-27|NONE|AIR|ounts detect|
+1281|94|7|3|2|1988.18|0.05|0.06|A|F|1994-12-27|1995-01-26|1995-01-21|TAKE BACK RETURN|FOB|ly unusual requests. final reques|
+1281|154|2|4|38|40057.70|0.04|0.06|R|F|1995-03-28|1995-01-11|1995-04-14|TAKE BACK RETURN|MAIL| ideas-- blithely regular|
+1281|152|10|5|13|13677.95|0.03|0.07|A|F|1995-02-06|1995-02-13|1995-02-18|DELIVER IN PERSON|TRUCK|fully final platelets wa|
+1281|50|9|6|4|3800.20|0.07|0.04|R|F|1995-03-15|1995-02-21|1995-03-20|NONE|SHIP|ggle against the even requests. requests |
+1281|78|6|7|43|42057.01|0.10|0.02|R|F|1995-01-28|1995-02-08|1995-02-10|DELIVER IN PERSON|AIR|final accounts. final packages slee|
+1282|23|4|1|14|12922.28|0.04|0.02|R|F|1992-06-29|1992-04-05|1992-07-21|TAKE BACK RETURN|REG AIR|ecial deposit|
+1282|30|9|2|10|9300.30|0.09|0.06|R|F|1992-04-10|1992-04-16|1992-05-01|DELIVER IN PERSON|SHIP|r theodolite|
+1282|160|1|3|19|20143.04|0.01|0.03|R|F|1992-05-07|1992-04-07|1992-05-13|NONE|RAIL|ts x-ray across the furi|
+1282|59|10|4|19|18221.95|0.00|0.05|A|F|1992-06-20|1992-04-17|1992-07-05|DELIVER IN PERSON|REG AIR|nto beans. carefully close theodo|
+1283|93|7|1|47|46675.23|0.05|0.03|N|O|1996-10-21|1996-10-29|1996-11-12|DELIVER IN PERSON|TRUCK|even instructions boost slyly blithely |
+1283|106|1|2|1|1006.10|0.00|0.08|N|O|1996-10-07|1996-10-12|1996-10-08|NONE|RAIL|d the sauternes. slyly ev|
+1283|138|4|3|18|18686.34|0.02|0.01|N|O|1996-10-14|1996-11-07|1996-10-22|DELIVER IN PERSON|AIR|equests use along the fluff|
+1283|192|4|4|40|43687.60|0.07|0.03|N|O|1996-11-09|1996-11-23|1996-11-28|NONE|MAIL|riously. even, ironic instructions after|
+1283|124|9|5|43|44037.16|0.01|0.04|N|O|1996-09-29|1996-11-19|1996-10-26|TAKE BACK RETURN|RAIL|requests sleep slyly about the |
+1283|8|5|6|30|27240.00|0.06|0.07|N|O|1996-11-22|1996-11-22|1996-12-15|COLLECT COD|TRUCK|t the fluffily|
+1283|197|8|7|21|23040.99|0.04|0.03|N|O|1996-09-12|1996-10-02|1996-10-12|NONE|REG AIR|fully regular |
+1284|178|7|1|49|52830.33|0.00|0.06|N|O|1996-04-11|1996-03-04|1996-04-16|NONE|MAIL|lar packages. special packages ac|
+1284|6|7|2|4|3624.00|0.07|0.06|N|O|1996-02-29|1996-02-11|1996-03-01|TAKE BACK RETURN|TRUCK| regular asymptotes. |
+1284|133|4|3|39|40292.07|0.08|0.00|N|O|1996-01-11|1996-02-07|1996-02-05|COLLECT COD|MAIL|even accoun|
+1284|59|10|4|1|959.05|0.01|0.07|N|O|1996-04-28|1996-04-02|1996-05-08|DELIVER IN PERSON|SHIP|al packages use carefully express de|
+1284|34|5|5|9|8406.27|0.05|0.06|N|O|1996-03-03|1996-03-19|1996-04-01|DELIVER IN PERSON|REG AIR|after the pending|
+1285|22|3|1|12|11064.24|0.00|0.06|A|F|1992-06-21|1992-08-16|1992-07-12|COLLECT COD|MAIL|ss foxes. blithe theodolites cajole slyly|
+1285|143|10|2|45|46941.30|0.01|0.02|R|F|1992-09-05|1992-08-08|1992-10-02|COLLECT COD|REG AIR| special requests haggle blithely.|
+1285|189|10|3|4|4356.72|0.09|0.06|A|F|1992-07-20|1992-08-17|1992-07-26|DELIVER IN PERSON|FOB|l packages sleep slyly quiet i|
+1285|188|9|4|39|42439.02|0.05|0.01|A|F|1992-09-15|1992-08-05|1992-10-05|DELIVER IN PERSON|TRUCK|uctions. car|
+1285|84|5|5|33|32474.64|0.00|0.08|R|F|1992-09-08|1992-08-25|1992-09-16|NONE|SHIP|ites affix|
+1286|178|9|1|49|52830.33|0.08|0.01|R|F|1993-06-24|1993-08-12|1993-06-26|DELIVER IN PERSON|SHIP|gged accoun|
+1286|49|6|2|48|45553.92|0.01|0.04|A|F|1993-07-11|1993-07-11|1993-08-01|COLLECT COD|TRUCK|unts alongs|
+1286|189|10|3|11|11980.98|0.03|0.04|R|F|1993-08-08|1993-07-30|1993-09-05|DELIVER IN PERSON|FOB| slyly even packages. requ|
+1286|184|5|4|37|40114.66|0.00|0.02|R|F|1993-05-27|1993-07-11|1993-06-01|COLLECT COD|SHIP|lyly ironic pinto beans cajole furiously s|
+1286|165|10|5|14|14912.24|0.00|0.01|R|F|1993-05-23|1993-08-09|1993-06-01|NONE|REG AIR|blithely bo|
+1286|146|5|6|41|42891.74|0.04|0.05|R|F|1993-08-02|1993-08-06|1993-08-07|TAKE BACK RETURN|FOB| the furiously expre|
+1287|174|3|1|35|37595.95|0.09|0.06|A|F|1994-09-07|1994-09-12|1994-09-30|TAKE BACK RETURN|FOB|s wake unusual grou|
+1287|95|8|2|10|9950.90|0.08|0.03|R|F|1994-07-08|1994-08-28|1994-07-10|TAKE BACK RETURN|RAIL|thely alongside of the unusual, ironic pa|
+1287|1|2|3|30|27030.00|0.00|0.07|R|F|1994-07-12|1994-09-23|1994-08-07|NONE|RAIL|ar packages. even, even|
+1287|62|7|4|10|9620.60|0.01|0.05|A|F|1994-09-03|1994-08-12|1994-09-16|TAKE BACK RETURN|REG AIR|ding, regular accounts|
+1287|179|8|5|21|22662.57|0.06|0.02|A|F|1994-10-06|1994-09-25|1994-10-16|TAKE BACK RETURN|TRUCK|y quickly bold theodoli|
+1287|21|10|6|26|23946.52|0.03|0.08|R|F|1994-10-03|1994-09-27|1994-10-30|DELIVER IN PERSON|RAIL|egular foxes. theodolites nag along t|
+1312|81|2|1|9|8829.72|0.04|0.08|R|F|1994-07-19|1994-06-29|1994-07-24|TAKE BACK RETURN|MAIL|. furiously |
+1312|136|7|2|28|29011.64|0.06|0.06|A|F|1994-09-09|1994-08-01|1994-10-02|TAKE BACK RETURN|FOB|uriously final frays should use quick|
+1312|173|1|3|18|19317.06|0.03|0.07|A|F|1994-09-13|1994-07-08|1994-09-22|TAKE BACK RETURN|MAIL|. slyly ironic|
+1313|52|4|1|48|45698.40|0.01|0.03|A|F|1994-12-20|1994-10-29|1995-01-07|COLLECT COD|MAIL|s are quick|
+1314|198|10|1|5|5490.95|0.03|0.01|A|F|1994-05-26|1994-08-06|1994-05-31|TAKE BACK RETURN|AIR|equests nag across the furious|
+1314|110|5|2|39|39394.29|0.01|0.03|R|F|1994-08-09|1994-06-14|1994-08-31|TAKE BACK RETURN|TRUCK| unusual accounts slee|
+1314|41|2|3|11|10351.44|0.01|0.04|A|F|1994-05-16|1994-07-30|1994-05-31|COLLECT COD|REG AIR|tegrate furious|
+1315|96|8|1|27|26894.43|0.01|0.03|N|O|1998-07-04|1998-06-13|1998-07-28|NONE|SHIP|latelets. fluffily ironic account|
+1315|16|6|2|15|13740.15|0.05|0.01|N|O|1998-07-12|1998-06-10|1998-08-07|COLLECT COD|AIR|. foxes integrate carefully special|
+1315|168|3|3|25|26704.00|0.01|0.08|N|O|1998-06-26|1998-06-10|1998-07-06|TAKE BACK RETURN|FOB|lites. unusual foxes affi|
+1315|161|6|4|19|20162.04|0.02|0.05|N|O|1998-07-05|1998-05-23|1998-08-04|TAKE BACK RETURN|SHIP|nal, regular warhorses about the fu|
+1315|159|7|5|32|33892.80|0.10|0.05|N|O|1998-03-30|1998-06-12|1998-04-25|NONE|SHIP|neath the final p|
+1316|127|6|1|46|47247.52|0.05|0.04|A|F|1994-01-13|1994-01-24|1994-02-03|COLLECT COD|TRUCK|ges haggle of the|
+1316|79|9|2|15|14686.05|0.02|0.01|R|F|1994-03-12|1994-03-02|1994-03-14|COLLECT COD|FOB|se. furiously final depo|
+1316|198|9|3|33|36240.27|0.10|0.06|R|F|1994-03-31|1994-01-23|1994-04-20|TAKE BACK RETURN|AIR|manently; blithely special deposits|
+1316|66|3|4|15|14490.90|0.00|0.06|R|F|1993-12-17|1994-02-04|1993-12-20|NONE|RAIL|fully express dugouts. furiously silent ide|
+1316|41|2|5|40|37641.60|0.01|0.03|R|F|1994-02-04|1994-02-09|1994-02-27|NONE|REG AIR|l dugouts. co|
+1316|4|7|6|7|6328.00|0.05|0.04|A|F|1993-12-09|1994-01-12|1993-12-30|TAKE BACK RETURN|MAIL|. furiously even accounts a|
+1316|163|8|7|8|8505.28|0.10|0.04|A|F|1994-03-26|1994-02-08|1994-04-19|NONE|SHIP|packages against the express requests wa|
+1317|134|5|1|34|35160.42|0.08|0.04|N|O|1995-08-13|1995-08-08|1995-09-10|COLLECT COD|RAIL|deposits boost thinly blithely final id|
+1317|160|2|2|7|7421.12|0.05|0.01|A|F|1995-06-08|1995-08-03|1995-06-16|TAKE BACK RETURN|SHIP| pinto beans according to the final, pend|
+1317|158|9|3|26|27511.90|0.01|0.02|N|O|1995-07-13|1995-06-26|1995-08-06|COLLECT COD|RAIL|leep along th|
+1317|106|3|4|35|35213.50|0.05|0.02|N|O|1995-07-16|1995-07-07|1995-07-22|TAKE BACK RETURN|FOB|r packages impress blithely car|
+1317|150|9|5|36|37805.40|0.02|0.00|N|O|1995-09-03|1995-07-06|1995-09-04|DELIVER IN PERSON|AIR| deposits. quic|
+1318|114|4|1|24|24338.64|0.08|0.06|N|O|1998-09-27|1998-09-15|1998-10-12|TAKE BACK RETURN|AIR|ual, unusual packages. fluffy, iro|
+1318|46|3|2|26|24597.04|0.01|0.03|N|O|1998-09-26|1998-08-09|1998-10-07|DELIVER IN PERSON|FOB|ly. regular, u|
+1318|129|4|3|31|31902.72|0.01|0.04|N|O|1998-08-25|1998-07-31|1998-08-31|COLLECT COD|AIR|ve the carefully expr|
+1319|61|8|1|21|20182.26|0.03|0.04|N|O|1996-10-05|1996-12-02|1996-10-28|COLLECT COD|FOB|s: carefully express |
+1319|37|8|2|12|11244.36|0.09|0.05|N|O|1996-11-05|1996-12-12|1996-11-29|DELIVER IN PERSON|TRUCK|packages integrate furiously. expres|
+1344|141|4|1|15|15617.10|0.10|0.07|A|F|1992-06-22|1992-06-24|1992-06-23|TAKE BACK RETURN|MAIL|rding to the blithely ironic theodolite|
+1344|190|1|2|29|31615.51|0.09|0.00|A|F|1992-07-17|1992-06-07|1992-07-21|NONE|REG AIR|ffily quiet foxes wake blithely. slyly |
+1345|198|9|1|49|53811.31|0.08|0.00|A|F|1992-12-27|1993-01-23|1993-01-06|NONE|FOB|sly. furiously final accounts are blithely |
+1345|12|9|2|37|33744.37|0.10|0.07|A|F|1992-11-27|1992-12-11|1992-12-07|COLLECT COD|FOB|e slyly express requests. ironic accounts c|
+1345|57|8|3|31|29668.55|0.08|0.07|R|F|1992-12-02|1992-12-29|1992-12-14|COLLECT COD|REG AIR|. slyly silent accounts sublat|
+1346|160|8|1|29|30744.64|0.07|0.05|A|F|1992-08-18|1992-09-15|1992-09-17|TAKE BACK RETURN|REG AIR|the pinto |
+1346|125|6|2|48|49205.76|0.06|0.03|A|F|1992-09-28|1992-07-22|1992-10-13|TAKE BACK RETURN|REG AIR| along the carefully spec|
+1346|54|5|3|13|12402.65|0.10|0.04|A|F|1992-07-22|1992-08-10|1992-08-06|NONE|SHIP|arefully brave deposits into the slyly iro|
+1346|124|5|4|6|6144.72|0.02|0.02|R|F|1992-09-13|1992-07-21|1992-09-27|TAKE BACK RETURN|AIR|inst the furiously final theodolites. caref|
+1346|187|8|5|30|32615.40|0.01|0.07|R|F|1992-10-01|1992-07-22|1992-10-24|NONE|SHIP| nag blithely. unusual, ru|
+1346|16|6|6|45|41220.45|0.02|0.04|A|F|1992-09-11|1992-08-06|1992-09-12|COLLECT COD|FOB|press deposits.|
+1347|81|2|1|45|44148.60|0.02|0.05|N|O|1997-08-24|1997-09-03|1997-09-08|COLLECT COD|AIR|ages wake around t|
+1347|143|6|2|34|35466.76|0.07|0.04|N|O|1997-06-25|1997-09-08|1997-07-24|COLLECT COD|FOB|r packages. f|
+1347|185|6|3|23|24959.14|0.03|0.04|N|O|1997-07-31|1997-08-25|1997-08-21|COLLECT COD|SHIP|ronic pinto beans. express reques|
+1347|113|7|4|28|28367.08|0.01|0.00|N|O|1997-07-30|1997-07-22|1997-08-18|TAKE BACK RETURN|FOB|foxes after the blithely special i|
+1347|65|6|5|9|8685.54|0.01|0.03|N|O|1997-08-28|1997-09-16|1997-09-26|DELIVER IN PERSON|AIR| detect blithely above the fina|
+1347|153|8|6|21|22116.15|0.06|0.04|N|O|1997-10-10|1997-08-16|1997-11-02|NONE|FOB|g pinto beans affix car|
+1347|51|3|7|10|9510.50|0.02|0.07|N|O|1997-07-04|1997-07-23|1997-07-05|DELIVER IN PERSON|SHIP|y ironic pin|
+1348|95|7|1|13|12936.17|0.01|0.01|N|O|1998-04-28|1998-06-05|1998-05-12|TAKE BACK RETURN|SHIP| blithely r|
+1348|22|5|2|41|37802.82|0.07|0.03|N|O|1998-05-02|1998-05-26|1998-05-09|COLLECT COD|RAIL|kages. platelets about the ca|
+1348|199|10|3|40|43967.60|0.07|0.05|N|O|1998-08-14|1998-07-10|1998-08-27|COLLECT COD|AIR|fter the regu|
+1348|98|1|4|2|1996.18|0.01|0.04|N|O|1998-05-30|1998-06-20|1998-06-05|COLLECT COD|MAIL|lly final packages use fluffily express ac|
+1349|181|2|1|1|1081.18|0.06|0.03|N|O|1998-01-07|1998-01-14|1998-02-03|COLLECT COD|REG AIR| express inst|
+1349|118|2|2|45|45814.95|0.03|0.02|N|O|1997-12-24|1998-01-17|1997-12-28|NONE|AIR| ironic, unusual deposits wake carefu|
+1350|54|9|1|21|20035.05|0.04|0.04|A|F|1993-12-17|1993-10-17|1993-12-25|COLLECT COD|REG AIR|lyly above the evenly |
+1350|44|5|2|32|30209.28|0.03|0.00|R|F|1993-11-18|1993-09-30|1993-12-16|COLLECT COD|MAIL|ic, final |
+1351|108|9|1|25|25202.50|0.06|0.04|N|O|1998-06-02|1998-05-25|1998-06-22|COLLECT COD|SHIP|iously regul|
+1376|169|8|1|22|23521.52|0.01|0.03|N|O|1997-08-05|1997-07-08|1997-09-03|NONE|REG AIR|inst the final, pending |
+1377|154|6|1|5|5270.75|0.06|0.05|N|O|1998-05-06|1998-07-08|1998-06-01|TAKE BACK RETURN|FOB| final, final grouches. accoun|
+1377|33|9|2|3|2799.09|0.10|0.04|N|O|1998-04-30|1998-07-02|1998-05-14|DELIVER IN PERSON|REG AIR|yly enticing requ|
+1377|84|5|3|26|25586.08|0.07|0.07|N|O|1998-05-28|1998-06-11|1998-06-25|COLLECT COD|SHIP|egular deposits. quickly regular acco|
+1377|121|4|4|39|39823.68|0.00|0.03|N|O|1998-07-27|1998-07-18|1998-08-13|DELIVER IN PERSON|SHIP|e ironic, regular requests. carefully |
+1377|33|9|5|19|17727.57|0.10|0.00|N|O|1998-06-20|1998-06-27|1998-07-20|NONE|AIR|ught to are bold foxes|
+1377|154|6|6|17|17920.55|0.03|0.04|N|O|1998-06-19|1998-07-20|1998-07-14|NONE|REG AIR|s must have to mold b|
+1378|197|10|1|34|37304.46|0.09|0.07|N|O|1996-07-08|1996-04-23|1996-07-09|COLLECT COD|RAIL|le furiously slyly final accounts. careful|
+1378|124|9|2|18|18434.16|0.05|0.02|N|O|1996-06-19|1996-05-16|1996-06-21|DELIVER IN PERSON|RAIL| theodolites. i|
+1378|73|4|3|11|10703.77|0.10|0.03|N|O|1996-06-07|1996-05-09|1996-07-05|COLLECT COD|TRUCK| blithely express hoc|
+1378|171|2|4|12|12854.04|0.02|0.06|N|O|1996-06-16|1996-05-23|1996-07-09|COLLECT COD|SHIP|notornis. b|
+1378|156|7|5|9|9505.35|0.06|0.05|N|O|1996-04-20|1996-04-13|1996-05-09|COLLECT COD|REG AIR|e carefully. carefully iron|
+1378|194|6|6|29|31731.51|0.05|0.05|N|O|1996-04-15|1996-04-23|1996-05-14|NONE|REG AIR|ual packages are furiously blith|
+1379|73|3|1|13|12649.91|0.04|0.01|N|O|1998-06-08|1998-07-13|1998-06-16|NONE|AIR|ully across the furiously iron|
+1379|118|2|2|50|50905.50|0.07|0.08|N|O|1998-08-31|1998-07-13|1998-09-02|TAKE BACK RETURN|FOB|olphins. ca|
+1379|13|7|3|24|21912.24|0.05|0.02|N|O|1998-07-06|1998-07-09|1998-07-29|DELIVER IN PERSON|MAIL|ages cajole carefully idly express re|
+1380|149|2|1|6|6294.84|0.00|0.04|N|O|1996-08-06|1996-10-01|1996-08-14|NONE|RAIL|e foxes. slyly specia|
+1380|141|4|2|40|41645.60|0.02|0.02|N|O|1996-10-01|1996-08-14|1996-10-20|COLLECT COD|RAIL|ly final frets. ironic,|
+1380|78|9|3|15|14671.05|0.05|0.02|N|O|1996-07-14|1996-08-12|1996-08-03|NONE|FOB|riously ironic foxes aff|
+1380|61|10|4|33|31714.98|0.04|0.07|N|O|1996-08-23|1996-10-01|1996-09-18|TAKE BACK RETURN|SHIP|e ironic, even excuses haggle |
+1381|144|1|1|47|49074.58|0.08|0.04|N|O|1998-09-22|1998-08-12|1998-10-12|DELIVER IN PERSON|AIR|ly ironic deposits|
+1381|34|10|2|12|11208.36|0.07|0.08|N|O|1998-08-13|1998-08-12|1998-08-28|TAKE BACK RETURN|AIR| furiously regular package|
+1382|162|3|1|18|19118.88|0.08|0.03|R|F|1993-08-30|1993-10-19|1993-09-03|DELIVER IN PERSON|AIR|hely regular deposits. fluffy s|
+1382|181|2|2|29|31354.22|0.08|0.04|A|F|1993-10-08|1993-11-11|1993-10-10|COLLECT COD|FOB| haggle: closely even asymptot|
+1382|178|7|3|43|46361.31|0.10|0.04|A|F|1993-09-02|1993-10-06|1993-09-15|DELIVER IN PERSON|AIR|ress deposits. slyly ironic foxes are blit|
+1382|181|2|4|11|11892.98|0.04|0.04|R|F|1993-09-17|1993-09-29|1993-09-21|NONE|SHIP|furiously unusual packages play quickly |
+1382|157|8|5|31|32771.65|0.07|0.03|R|F|1993-10-26|1993-10-15|1993-11-09|TAKE BACK RETURN|FOB|hely regular dependencies. f|
+1382|10|5|6|38|34580.38|0.07|0.07|R|F|1993-11-17|1993-09-28|1993-11-20|COLLECT COD|SHIP|ake pending pinto beans. s|
+1382|23|4|7|5|4615.10|0.07|0.01|R|F|1993-10-02|1993-09-29|1993-10-12|DELIVER IN PERSON|REG AIR|ter the carefully final excuses. blit|
+1383|193|7|1|14|15304.66|0.07|0.06|A|F|1993-08-25|1993-07-09|1993-09-12|DELIVER IN PERSON|RAIL|ole carefully silent requests. car|
+1383|161|10|2|19|20162.04|0.06|0.04|R|F|1993-05-24|1993-07-07|1993-06-14|NONE|AIR|lyly unusual accounts sle|
+1408|148|7|1|29|30396.06|0.03|0.04|N|O|1998-03-12|1998-02-14|1998-03-17|COLLECT COD|MAIL|en accounts grow furiousl|
+1408|173|2|2|7|7512.19|0.05|0.06|N|O|1998-01-14|1998-03-21|1998-01-29|COLLECT COD|AIR|fully final instructions. theodolites ca|
+1408|76|6|3|11|10736.77|0.00|0.03|N|O|1998-04-04|1998-01-29|1998-04-18|NONE|REG AIR|y even accounts thrash care|
+1408|148|5|4|20|20962.80|0.06|0.00|N|O|1998-04-21|1998-01-25|1998-05-12|DELIVER IN PERSON|TRUCK| blithely fluffi|
+1408|170|1|5|41|43876.97|0.02|0.06|N|O|1998-02-25|1998-02-03|1998-03-13|COLLECT COD|REG AIR|ep along the fina|
+1408|134|10|6|42|43433.46|0.05|0.08|N|O|1998-01-30|1998-02-07|1998-02-18|TAKE BACK RETURN|REG AIR|even packages. even accounts cajole|
+1408|55|6|7|26|24831.30|0.00|0.00|N|O|1998-03-19|1998-03-14|1998-04-01|COLLECT COD|RAIL|ic foxes ca|
+1409|99|1|1|23|22979.07|0.01|0.03|A|F|1993-04-18|1993-02-25|1993-05-06|DELIVER IN PERSON|FOB|ions. slyly ironic packages wake quick|
+1409|65|2|2|36|34742.16|0.09|0.02|A|F|1993-01-27|1993-01-31|1993-02-07|COLLECT COD|FOB|ncies sleep carefully r|
+1409|160|1|3|17|18022.72|0.07|0.00|R|F|1993-04-15|1993-03-01|1993-04-29|NONE|REG AIR|pending accounts poach. care|
+1410|121|10|1|15|15316.80|0.06|0.05|N|O|1997-05-25|1997-07-08|1997-06-15|NONE|SHIP| bold packages are fluf|
+1410|179|9|2|18|19425.06|0.03|0.00|N|O|1997-06-03|1997-05-17|1997-06-07|TAKE BACK RETURN|RAIL|gle furiously fluffily regular requests|
+1410|109|4|3|37|37336.70|0.02|0.01|N|O|1997-04-17|1997-06-18|1997-04-19|COLLECT COD|TRUCK|to beans b|
+1410|188|9|4|22|23939.96|0.10|0.00|N|O|1997-07-31|1997-05-17|1997-08-19|TAKE BACK RETURN|RAIL|gular account|
+1410|66|1|5|25|24151.50|0.09|0.02|N|O|1997-05-07|1997-07-10|1997-05-16|NONE|REG AIR|unts haggle against the furiously fina|
+1411|17|7|1|9|8253.09|0.06|0.04|A|F|1995-03-08|1995-03-04|1995-03-11|DELIVER IN PERSON|AIR|accounts. furiou|
+1411|107|8|2|26|26184.60|0.02|0.02|A|F|1995-04-12|1995-01-24|1995-05-03|TAKE BACK RETURN|TRUCK|c packages. |
+1411|27|6|3|37|34299.74|0.00|0.06|A|F|1995-02-27|1995-03-02|1995-03-24|NONE|MAIL|d excuses. furiously final pear|
+1411|200|3|4|20|22004.00|0.01|0.03|R|F|1995-04-06|1995-03-16|1995-04-17|COLLECT COD|FOB|s against the|
+1411|83|4|5|46|45221.68|0.08|0.05|A|F|1995-04-03|1995-01-20|1995-04-05|DELIVER IN PERSON|REG AIR|ly daring instructions|
+1411|77|6|6|30|29312.10|0.09|0.04|A|F|1995-01-12|1995-02-01|1995-01-23|DELIVER IN PERSON|MAIL|ious foxes wake courts. caref|
+1412|58|3|1|37|35447.85|0.06|0.01|A|F|1993-04-10|1993-04-19|1993-04-12|DELIVER IN PERSON|RAIL|hely express excuses are |
+1412|156|1|2|20|21123.00|0.10|0.05|A|F|1993-07-04|1993-05-18|1993-07-22|DELIVER IN PERSON|REG AIR|odolites sleep ironically|
+1412|23|2|3|2|1846.04|0.10|0.07|R|F|1993-04-01|1993-05-03|1993-04-12|DELIVER IN PERSON|REG AIR|s among the requests are a|
+1412|167|8|4|11|11738.76|0.05|0.07|R|F|1993-05-27|1993-05-30|1993-06-07|DELIVER IN PERSON|MAIL|en packages. regular packages dete|
+1412|158|6|5|11|11639.65|0.08|0.06|A|F|1993-03-30|1993-05-25|1993-04-21|NONE|FOB|se slyly. special, unusual accounts nag bl|
+1413|178|9|1|18|19407.06|0.08|0.05|N|O|1997-10-11|1997-08-17|1997-10-25|NONE|FOB|yly bold packages haggle quickly acr|
+1413|165|10|2|49|52192.84|0.07|0.06|N|O|1997-08-28|1997-08-23|1997-09-12|DELIVER IN PERSON|MAIL|nstructions br|
+1413|42|9|3|6|5652.24|0.04|0.02|N|O|1997-09-07|1997-07-30|1997-09-21|TAKE BACK RETURN|MAIL|lithely excuses. f|
+1414|38|4|1|39|36583.17|0.10|0.03|N|O|1995-09-22|1995-09-30|1995-10-07|NONE|MAIL|quickly aro|
+1414|107|8|2|4|4028.40|0.02|0.05|N|O|1995-09-16|1995-11-01|1995-10-02|COLLECT COD|AIR| haggle quickly|
+1415|149|10|1|25|26228.50|0.06|0.00|A|F|1994-09-03|1994-07-12|1994-09-13|DELIVER IN PERSON|RAIL|ect never fluff|
+1440|193|6|1|3|3279.57|0.06|0.01|N|O|1995-10-30|1995-10-17|1995-11-08|COLLECT COD|SHIP|instructions boost. fluffily regul|
+1440|114|4|2|46|46649.06|0.02|0.03|N|O|1995-09-21|1995-10-19|1995-10-19|NONE|RAIL|blithely even instructions. |
+1441|144|7|1|5|5220.70|0.04|0.01|N|O|1997-05-17|1997-05-11|1997-05-30|COLLECT COD|MAIL|egular courts. fluffily even grouches |
+1441|177|7|2|5|5385.85|0.02|0.05|N|O|1997-04-25|1997-04-16|1997-05-23|COLLECT COD|FOB|he quickly enticing pac|
+1441|118|5|3|14|14253.54|0.01|0.03|N|O|1997-06-30|1997-04-29|1997-07-24|DELIVER IN PERSON|REG AIR|special requests ha|
+1441|160|8|4|37|39225.92|0.01|0.00|N|O|1997-04-26|1997-04-27|1997-04-29|NONE|REG AIR|accounts. slyly special dolphins b|
+1441|72|10|5|34|33050.38|0.09|0.00|N|O|1997-06-12|1997-05-11|1997-06-29|TAKE BACK RETURN|RAIL|e carefully. blithely ironic dep|
+1441|25|4|6|15|13875.30|0.09|0.08|N|O|1997-05-21|1997-05-06|1997-06-04|NONE|REG AIR| dependencies-- cour|
+1441|96|10|7|50|49804.50|0.03|0.01|N|O|1997-06-07|1997-05-12|1997-06-08|NONE|SHIP| requests. blithely e|
+1442|26|5|1|8|7408.16|0.05|0.01|A|F|1994-10-31|1994-09-04|1994-11-25|COLLECT COD|AIR|c deposits haggle after the even|
+1443|34|10|1|47|43899.41|0.04|0.06|N|O|1997-02-05|1997-02-02|1997-03-03|NONE|RAIL|carefully ironic requests sl|
+1444|170|5|1|42|44947.14|0.01|0.02|R|F|1994-12-22|1995-03-03|1994-12-31|NONE|SHIP|ly bold packages boost regular ideas. spe|
+1444|57|2|2|34|32539.70|0.04|0.08|A|F|1995-02-22|1995-02-15|1995-03-19|TAKE BACK RETURN|AIR|y. doggedly pend|
+1444|155|3|3|34|35875.10|0.02|0.07|R|F|1994-12-17|1995-01-12|1995-01-03|COLLECT COD|AIR|ular accounts |
+1444|119|6|4|6|6114.66|0.06|0.03|A|F|1995-01-07|1995-03-05|1995-01-17|COLLECT COD|RAIL|al accounts. br|
+1444|20|1|5|35|32200.70|0.02|0.05|A|F|1995-02-25|1995-03-05|1995-03-24|DELIVER IN PERSON|SHIP|aggle furiou|
+1444|33|4|6|42|39187.26|0.00|0.02|A|F|1994-12-16|1995-02-18|1994-12-22|DELIVER IN PERSON|RAIL|ss requests. ironic ideas wake above|
+1444|82|3|7|12|11784.96|0.00|0.03|R|F|1994-12-23|1995-01-15|1995-01-13|COLLECT COD|TRUCK|ly among the bol|
+1445|100|1|1|24|24002.40|0.01|0.00|A|F|1995-02-21|1995-02-22|1995-03-18|DELIVER IN PERSON|SHIP|al accounts use furiously a|
+1445|67|8|2|48|46418.88|0.10|0.02|A|F|1995-02-28|1995-03-16|1995-03-12|COLLECT COD|MAIL|. final ideas are carefully dar|
+1445|192|4|3|7|7645.33|0.10|0.04|A|F|1995-04-25|1995-02-25|1995-05-10|NONE|SHIP|structions: slyly regular re|
+1445|28|1|4|17|15776.34|0.04|0.07|A|F|1995-04-02|1995-04-04|1995-05-01|COLLECT COD|FOB|ges. furiously regular pint|
+1445|135|1|5|24|24843.12|0.10|0.06|R|F|1995-04-23|1995-02-16|1995-05-18|NONE|REG AIR|rate after the carefully reg|
+1445|168|9|6|39|41658.24|0.03|0.02|A|F|1995-02-05|1995-02-20|1995-02-06|NONE|MAIL|ully unusual reques|
+1446|72|3|1|31|30134.17|0.10|0.02|N|O|1998-05-01|1998-05-17|1998-05-30|NONE|REG AIR|. slyly reg|
+1447|167|4|1|19|20276.04|0.06|0.04|A|F|1993-01-31|1992-12-07|1993-02-04|COLLECT COD|MAIL|. quickly ironic |
+1447|32|3|2|6|5592.18|0.01|0.05|A|F|1992-10-24|1992-12-10|1992-11-05|DELIVER IN PERSON|AIR|as! regular packages poach above the|
+1447|39|5|3|9|8451.27|0.04|0.00|R|F|1992-11-15|1993-01-07|1992-11-29|DELIVER IN PERSON|MAIL|counts wake s|
+1447|22|5|4|8|7376.16|0.09|0.08|R|F|1992-11-20|1993-01-12|1992-12-14|COLLECT COD|FOB|ost carefully |
+1447|130|1|5|23|23692.99|0.02|0.07|A|F|1992-12-07|1992-12-25|1993-01-06|TAKE BACK RETURN|AIR| dazzle quickly deposits. f|
+1447|200|3|6|41|45108.20|0.08|0.02|R|F|1993-01-06|1993-01-05|1993-01-13|TAKE BACK RETURN|MAIL|rts boost s|
+1472|8|5|1|36|32688.00|0.04|0.05|N|O|1996-11-06|1996-11-13|1996-11-12|COLLECT COD|SHIP|riously silent deposits to the pending d|
+1472|133|4|2|26|26861.38|0.03|0.05|N|O|1996-11-08|1996-11-13|1996-12-02|DELIVER IN PERSON|FOB|ic packages w|
+1472|1|8|3|6|5406.00|0.08|0.01|N|O|1996-10-24|1996-11-19|1996-11-23|COLLECT COD|FOB|onic theodolites hinder slyly slyly r|
+1473|54|9|1|50|47702.50|0.04|0.03|N|O|1997-05-05|1997-05-20|1997-05-09|NONE|TRUCK|requests wake express deposits. special, ir|
+1473|68|3|2|32|30977.92|0.00|0.08|N|O|1997-04-18|1997-05-12|1997-05-10|DELIVER IN PERSON|REG AIR|out the packages lose furiously ab|
+1474|15|5|1|5|4575.05|0.05|0.04|A|F|1995-04-22|1995-02-20|1995-05-06|COLLECT COD|SHIP|ully final a|
+1474|123|8|2|30|30693.60|0.04|0.02|A|F|1995-03-23|1995-02-11|1995-04-17|DELIVER IN PERSON|TRUCK|usly. evenly express |
+1474|92|5|3|18|17857.62|0.06|0.02|A|F|1995-01-23|1995-03-28|1995-02-03|NONE|RAIL|after the special|
+1475|168|3|1|15|16022.40|0.08|0.06|N|O|1998-02-12|1997-12-17|1998-03-02|TAKE BACK RETURN|SHIP|xpress requests haggle after the final, fi|
+1475|118|9|2|18|18325.98|0.07|0.00|N|O|1998-03-08|1998-01-18|1998-03-10|TAKE BACK RETURN|AIR|al deposits use. ironic packages along the |
+1475|144|1|3|30|31324.20|0.03|0.02|N|O|1998-03-11|1997-12-30|1998-03-15|COLLECT COD|REG AIR| regular theodolites mold across th|
+1475|187|8|4|50|54359.00|0.03|0.05|N|O|1997-12-14|1997-12-13|1997-12-21|COLLECT COD|AIR|. slyly bold re|
+1475|32|3|5|33|30756.99|0.01|0.06|N|O|1998-01-02|1998-01-27|1998-01-11|NONE|FOB|quickly fluffy|
+1475|50|7|6|12|11400.60|0.04|0.04|N|O|1998-01-09|1997-12-30|1998-01-23|NONE|TRUCK|arefully-- excuses sublate|
+1475|112|3|7|23|23278.53|0.02|0.00|N|O|1998-02-13|1998-02-05|1998-03-08|NONE|TRUCK|hely regular hocke|
+1476|31|7|1|20|18620.60|0.02|0.03|N|O|1996-08-11|1996-09-18|1996-08-26|TAKE BACK RETURN|AIR|. bold deposits are carefully amo|
+1477|72|1|1|31|30134.17|0.00|0.06|N|O|1997-12-16|1997-09-30|1997-12-17|COLLECT COD|RAIL| requests. fluffily final |
+1477|110|7|2|8|8080.88|0.09|0.05|N|O|1997-10-25|1997-10-18|1997-11-16|COLLECT COD|MAIL|ironic realms wake unusual, even ac|
+1477|125|6|3|42|43055.04|0.06|0.00|N|O|1997-11-02|1997-11-02|1997-11-20|DELIVER IN PERSON|SHIP|lithely after the ir|
+1477|107|8|4|32|32227.20|0.05|0.08|N|O|1997-09-12|1997-10-26|1997-10-12|TAKE BACK RETURN|AIR|; quickly regula|
+1477|115|6|5|41|41619.51|0.04|0.06|N|O|1997-12-16|1997-10-31|1998-01-12|DELIVER IN PERSON|REG AIR|y. final pearls kindle. accounts |
+1477|69|6|6|49|47483.94|0.06|0.00|N|O|1997-11-18|1997-11-06|1997-11-27|COLLECT COD|FOB|ise according to the sly, bold p|
+1477|120|4|7|33|33663.96|0.06|0.00|N|O|1997-11-12|1997-11-06|1997-11-24|DELIVER IN PERSON|TRUCK|yly regular p|
+1478|34|5|1|21|19614.63|0.00|0.06|N|O|1997-09-20|1997-10-25|1997-10-06|TAKE BACK RETURN|MAIL| fluffily pending acc|
+1479|149|6|1|33|34621.62|0.10|0.01|N|O|1996-03-12|1996-02-28|1996-03-31|DELIVER IN PERSON|FOB| carefully special courts affix. fluff|
+1504|82|3|1|42|41247.36|0.02|0.03|R|F|1992-10-18|1992-10-14|1992-11-10|TAKE BACK RETURN|FOB|ep. carefully ironic excuses haggle quickl|
+1504|103|10|2|22|22068.20|0.04|0.03|A|F|1992-09-09|1992-10-29|1992-09-10|NONE|REG AIR| accounts sleep. furiou|
+1504|178|8|3|9|9703.53|0.07|0.02|R|F|1992-11-02|1992-10-12|1992-11-15|TAKE BACK RETURN|RAIL|y slyly regular courts.|
+1504|115|2|4|10|10151.10|0.04|0.07|A|F|1992-09-22|1992-10-22|1992-10-13|TAKE BACK RETURN|TRUCK|final theodolites. furiously e|
+1504|20|10|5|7|6440.14|0.02|0.00|R|F|1992-11-20|1992-11-23|1992-12-13|COLLECT COD|MAIL|y final packa|
+1505|120|7|1|4|4080.48|0.09|0.00|A|F|1992-12-14|1992-11-11|1993-01-02|COLLECT COD|SHIP|side of the s|
+1505|123|8|2|50|51156.00|0.00|0.02|R|F|1992-11-22|1992-09-24|1992-11-26|TAKE BACK RETURN|FOB|lyly special platelets. requests ar|
+1506|133|4|1|46|47523.98|0.04|0.05|R|F|1993-01-18|1992-11-11|1993-02-09|COLLECT COD|REG AIR|sits whithout the blithely ironic packages|
+1506|114|4|2|30|30423.30|0.07|0.02|A|F|1992-11-22|1992-10-25|1992-12-04|DELIVER IN PERSON|FOB|deposits cajole |
+1506|191|3|3|28|30553.32|0.10|0.06|A|F|1992-09-22|1992-11-19|1992-10-09|TAKE BACK RETURN|AIR| unwind carefully: theodolit|
+1506|28|7|4|37|34336.74|0.00|0.03|R|F|1992-11-04|1992-12-01|1992-11-23|TAKE BACK RETURN|TRUCK|carefully bold dolphins. accounts su|
+1506|195|8|5|15|16427.85|0.05|0.00|R|F|1992-09-24|1992-11-11|1992-10-05|NONE|REG AIR| carefully fluffy packages-- caref|
+1506|50|3|6|38|36101.90|0.05|0.02|R|F|1992-12-02|1992-12-19|1992-12-29|NONE|REG AIR|xpress, regular excuse|
+1506|169|6|7|4|4276.64|0.07|0.00|R|F|1993-01-03|1992-12-06|1993-01-05|COLLECT COD|REG AIR|posits. furiou|
+1507|68|5|1|25|24201.50|0.01|0.08|R|F|1994-01-07|1994-01-06|1994-01-11|NONE|RAIL|xes. slyly busy de|
+1507|40|6|2|33|31021.32|0.04|0.02|A|F|1993-10-29|1993-12-23|1993-11-14|DELIVER IN PERSON|REG AIR| asymptotes nag furiously above t|
+1507|86|7|3|39|38457.12|0.03|0.07|R|F|1993-11-04|1993-12-16|1993-12-03|TAKE BACK RETURN|REG AIR|ly even instructions.|
+1508|51|3|1|16|15216.80|0.02|0.06|N|O|1998-06-21|1998-05-30|1998-07-11|COLLECT COD|MAIL|riously across the ironic, unusua|
+1508|25|4|2|20|18500.40|0.06|0.01|N|O|1998-04-17|1998-06-11|1998-05-17|DELIVER IN PERSON|MAIL|nic platelets. carefully final fra|
+1508|93|7|3|43|42702.87|0.01|0.02|N|O|1998-06-01|1998-06-24|1998-06-03|TAKE BACK RETURN|TRUCK|ndencies h|
+1508|148|7|4|1|1048.14|0.02|0.02|N|O|1998-07-13|1998-06-03|1998-07-17|TAKE BACK RETURN|AIR|s the blithely bold instruction|
+1508|135|6|5|29|30018.77|0.02|0.00|N|O|1998-08-03|1998-07-08|1998-08-22|COLLECT COD|RAIL|r instructions. carefully|
+1508|3|10|6|5|4515.00|0.06|0.08|N|O|1998-05-22|1998-07-06|1998-06-04|COLLECT COD|REG AIR|cording to the furiously ironic depe|
+1508|117|8|7|38|38650.18|0.03|0.06|N|O|1998-04-30|1998-06-23|1998-05-18|DELIVER IN PERSON|RAIL|tes wake furiously regular w|
+1509|28|7|1|14|12992.28|0.04|0.01|A|F|1993-10-04|1993-09-25|1993-10-21|NONE|TRUCK|nal realms|
+1509|11|2|2|46|41906.46|0.08|0.02|A|F|1993-10-15|1993-10-04|1993-11-01|TAKE BACK RETURN|FOB|uriously regula|
+1509|107|8|3|17|17120.70|0.06|0.05|A|F|1993-07-25|1993-08-28|1993-08-19|DELIVER IN PERSON|AIR| furiously. blithely regular ideas haggle c|
+1509|20|4|4|11|10120.22|0.03|0.08|R|F|1993-11-04|1993-10-03|1993-11-14|TAKE BACK RETURN|FOB|ily ironic packages nod carefully.|
+1509|90|1|5|37|36633.33|0.01|0.08|A|F|1993-08-31|1993-09-10|1993-09-24|NONE|FOB|he slyly even deposits wake a|
+1509|187|8|6|31|33702.58|0.04|0.03|A|F|1993-07-14|1993-08-21|1993-08-06|COLLECT COD|SHIP|ic deposits cajole carefully. quickly bold |
+1509|157|2|7|27|28543.05|0.01|0.01|A|F|1993-09-29|1993-09-08|1993-10-04|TAKE BACK RETURN|FOB|lithely after the |
+1510|98|2|1|11|10978.99|0.09|0.04|N|O|1996-09-23|1996-12-03|1996-10-01|DELIVER IN PERSON|RAIL|e of the unusual accounts. stealthy deposit|
+1510|84|5|2|24|23617.92|0.05|0.04|N|O|1996-10-07|1996-10-22|1996-11-03|DELIVER IN PERSON|REG AIR|yly brave theod|
+1510|190|1|3|36|39246.84|0.07|0.02|N|O|1996-10-02|1996-11-23|1996-10-05|NONE|SHIP|old deposits along the carefully|
+1510|182|3|4|8|8657.44|0.01|0.08|N|O|1996-10-26|1996-11-07|1996-10-30|TAKE BACK RETURN|RAIL|blithely express|
+1510|59|10|5|27|25894.35|0.08|0.06|N|O|1996-10-20|1996-12-05|1996-11-02|NONE|MAIL|he blithely regular req|
+1510|14|5|6|3|2742.03|0.05|0.02|N|O|1996-10-31|1996-12-03|1996-11-13|COLLECT COD|RAIL|along the slyly regular pin|
+1510|22|1|7|50|46101.00|0.04|0.05|N|O|1996-11-01|1996-10-17|1996-11-28|NONE|MAIL|even packages. carefully regular fo|
+1511|98|2|1|29|28944.61|0.01|0.04|N|O|1997-03-17|1997-02-11|1997-03-27|DELIVER IN PERSON|AIR|s cajole furiously against |
+1511|62|9|2|32|30785.92|0.04|0.01|N|O|1997-01-06|1997-03-21|1997-01-26|TAKE BACK RETURN|REG AIR| deposits. carefully ironi|
+1536|194|5|1|5|5470.95|0.08|0.03|N|O|1997-02-08|1997-03-11|1997-03-02|COLLECT COD|MAIL|requests sleep pe|
+1537|18|2|1|17|15606.17|0.01|0.03|A|F|1992-04-12|1992-04-19|1992-04-13|NONE|TRUCK|he regular pack|
+1537|179|8|2|50|53958.50|0.08|0.00|R|F|1992-05-30|1992-05-14|1992-06-23|TAKE BACK RETURN|MAIL|special packages haggle slyly at the silent|
+1537|13|4|3|44|40172.44|0.05|0.04|R|F|1992-04-01|1992-03-31|1992-04-21|NONE|TRUCK|lar courts.|
+1537|140|6|4|3|3120.42|0.08|0.07|R|F|1992-03-20|1992-04-14|1992-03-21|TAKE BACK RETURN|SHIP|s, final ideas detect sl|
+1538|102|5|1|32|32067.20|0.05|0.05|N|O|1995-07-08|1995-07-29|1995-08-01|TAKE BACK RETURN|RAIL|uses maintain blithely. fluffily|
+1538|192|3|2|27|29489.13|0.05|0.01|N|O|1995-09-19|1995-08-03|1995-09-24|DELIVER IN PERSON|TRUCK|ngly even packag|
+1538|130|3|3|36|37084.68|0.08|0.04|N|O|1995-07-11|1995-09-10|1995-07-26|DELIVER IN PERSON|MAIL|al deposits mo|
+1538|104|1|4|28|28114.80|0.10|0.04|N|O|1995-09-19|1995-08-27|1995-10-10|COLLECT COD|RAIL|bout the fluffily unusual|
+1538|178|7|5|13|14016.21|0.01|0.05|N|O|1995-06-26|1995-07-30|1995-07-25|NONE|SHIP|ly. packages sleep f|
+1538|128|3|6|42|43181.04|0.08|0.08|N|O|1995-10-10|1995-09-12|1995-11-08|DELIVER IN PERSON|TRUCK|equests cajole blithely |
+1539|196|9|1|21|23019.99|0.08|0.02|R|F|1995-04-19|1995-05-10|1995-04-27|COLLECT COD|TRUCK|ounts haggle. busy|
+1539|86|7|2|11|10846.88|0.01|0.08|A|F|1995-05-27|1995-04-13|1995-06-10|TAKE BACK RETURN|TRUCK|ly express requests. furiously |
+1539|68|5|3|7|6776.42|0.09|0.04|R|F|1995-05-14|1995-04-16|1995-05-30|DELIVER IN PERSON|AIR|. fluffily reg|
+1540|173|1|1|38|40780.46|0.03|0.01|R|F|1992-09-30|1992-10-27|1992-10-12|TAKE BACK RETURN|SHIP| final grouches bo|
+1540|60|2|2|35|33602.10|0.02|0.07|R|F|1992-10-31|1992-09-04|1992-11-05|TAKE BACK RETURN|SHIP|e blithely a|
+1540|8|3|3|25|22700.00|0.08|0.04|R|F|1992-11-15|1992-10-24|1992-12-14|DELIVER IN PERSON|SHIP|ironic deposits amo|
+1540|25|8|4|6|5550.12|0.09|0.03|R|F|1992-08-28|1992-09-17|1992-09-14|COLLECT COD|MAIL|ing to the slyly express asymptote|
+1540|87|8|5|27|26651.16|0.10|0.08|R|F|1992-12-02|1992-10-18|1992-12-31|NONE|SHIP|carefully final packages; b|
+1541|64|3|1|44|42418.64|0.10|0.05|N|O|1995-08-24|1995-07-13|1995-08-26|TAKE BACK RETURN|MAIL|o beans boost fluffily abou|
+1541|26|7|2|8|7408.16|0.10|0.08|N|F|1995-06-05|1995-08-07|1995-06-21|TAKE BACK RETURN|TRUCK|y pending packages. blithely fi|
+1542|58|9|1|37|35447.85|0.07|0.06|A|F|1993-12-15|1993-10-17|1994-01-07|TAKE BACK RETURN|REG AIR|e blithely unusual accounts. quic|
+1542|3|6|2|12|10836.00|0.09|0.06|R|F|1993-10-29|1993-11-02|1993-11-09|TAKE BACK RETURN|RAIL|carefully |
+1542|6|7|3|18|16308.00|0.05|0.05|R|F|1993-10-17|1993-11-15|1993-10-26|TAKE BACK RETURN|FOB|pending instr|
+1542|143|10|4|21|21905.94|0.01|0.05|R|F|1993-10-13|1993-12-13|1993-11-12|NONE|RAIL|y pending foxes nag blithely |
+1542|155|7|5|46|48536.90|0.00|0.00|R|F|1993-09-28|1993-11-03|1993-10-15|COLLECT COD|FOB|ial instructions. ironically|
+1543|71|10|1|34|33016.38|0.02|0.08|N|O|1997-05-25|1997-03-30|1997-06-04|NONE|AIR|ic requests are ac|
+1543|115|9|2|6|6090.66|0.09|0.01|N|O|1997-04-16|1997-05-20|1997-05-16|DELIVER IN PERSON|MAIL| among the carefully bold or|
+1543|67|8|3|42|40616.52|0.06|0.01|N|O|1997-05-26|1997-03-30|1997-06-12|DELIVER IN PERSON|FOB|its sleep until the fur|
+1543|189|10|4|42|45745.56|0.05|0.06|N|O|1997-04-11|1997-04-11|1997-04-23|TAKE BACK RETURN|MAIL|xpress instructions. regular acc|
+1543|40|1|5|9|8460.36|0.08|0.06|N|O|1997-03-14|1997-05-19|1997-03-26|DELIVER IN PERSON|FOB|ravely special requests |
+1543|49|8|6|3|2847.12|0.10|0.04|N|O|1997-03-29|1997-05-10|1997-04-22|COLLECT COD|MAIL|sleep along the furiou|
+1543|68|7|7|3|2904.18|0.00|0.02|N|O|1997-03-22|1997-04-06|1997-03-30|NONE|AIR|quickly. final accounts haggle slyl|
+1568|90|1|1|36|35643.24|0.02|0.03|N|O|1997-05-31|1997-04-22|1997-06-21|TAKE BACK RETURN|RAIL|platelets-- furiously sly excu|
+1568|9|2|2|46|41814.00|0.04|0.00|N|O|1997-04-06|1997-04-08|1997-04-23|TAKE BACK RETURN|MAIL|g the blithely even acco|
+1569|75|3|1|5|4875.35|0.07|0.00|N|O|1998-04-16|1998-06-21|1998-04-18|COLLECT COD|REG AIR| packages. ironic, even excuses a|
+1569|39|10|2|16|15024.48|0.01|0.08|N|O|1998-04-26|1998-06-16|1998-05-26|COLLECT COD|MAIL|deposits. blithely final asymptotes ac|
+1569|49|10|3|43|40808.72|0.10|0.03|N|O|1998-06-05|1998-05-31|1998-06-28|DELIVER IN PERSON|FOB| instructions.|
+1569|70|1|4|30|29102.10|0.02|0.03|N|O|1998-07-19|1998-06-04|1998-08-10|NONE|SHIP|packages. excuses lose evenly carefully reg|
+1570|183|4|1|25|27079.50|0.00|0.06|N|O|1998-05-03|1998-06-02|1998-06-02|DELIVER IN PERSON|REG AIR|its. slyly regular sentiments|
+1570|86|7|2|7|6902.56|0.05|0.05|N|O|1998-07-10|1998-06-01|1998-07-23|TAKE BACK RETURN|MAIL|requests boost quickly re|
+1571|52|3|1|47|44746.35|0.00|0.05|R|F|1992-12-07|1993-02-24|1993-01-01|TAKE BACK RETURN|REG AIR|ng to the fluffily unusual |
+1571|183|4|2|6|6499.08|0.03|0.00|A|F|1993-01-08|1993-02-13|1993-02-07|COLLECT COD|SHIP| special, ironic depo|
+1571|59|7|3|18|17262.90|0.05|0.08|A|F|1993-01-09|1993-01-12|1993-01-31|COLLECT COD|AIR| pending grouches |
+1571|101|4|4|48|48052.80|0.05|0.05|A|F|1992-12-28|1993-01-04|1993-01-04|DELIVER IN PERSON|RAIL|slyly pending p|
+1571|42|5|5|10|9420.40|0.03|0.06|R|F|1992-12-12|1993-02-13|1992-12-29|DELIVER IN PERSON|AIR|lets. carefully regular ideas wake|
+1571|34|10|6|24|22416.72|0.05|0.07|A|F|1993-03-22|1993-01-31|1993-04-09|NONE|TRUCK|warthogs wake carefully acro|
+1572|24|5|1|41|37884.82|0.02|0.00|N|O|1996-05-16|1996-04-09|1996-05-28|TAKE BACK RETURN|REG AIR|. pinto beans alongside|
+1572|93|7|2|10|9930.90|0.04|0.06|N|O|1996-05-17|1996-03-26|1996-05-19|NONE|AIR| accounts affix slyly. |
+1573|186|7|1|5|5430.90|0.05|0.01|A|F|1993-04-24|1993-03-13|1993-05-17|TAKE BACK RETURN|MAIL|ymptotes could u|
+1573|31|2|2|17|15827.51|0.00|0.06|R|F|1993-02-24|1993-02-16|1993-03-08|TAKE BACK RETURN|TRUCK|carefully regular deposits. |
+1573|83|4|3|16|15729.28|0.04|0.03|A|F|1993-03-15|1993-03-16|1993-03-31|COLLECT COD|AIR|ely. furiously final requests wake slyl|
+1573|194|7|4|11|12036.09|0.09|0.01|R|F|1993-03-23|1993-03-24|1993-04-12|TAKE BACK RETURN|RAIL|nently pending|
+1573|137|8|5|7|7259.91|0.00|0.01|R|F|1993-01-30|1993-03-14|1993-02-27|DELIVER IN PERSON|SHIP|eodolites sleep slyly. slyly f|
+1573|154|6|6|30|31624.50|0.03|0.01|A|F|1992-12-29|1993-03-06|1993-01-02|DELIVER IN PERSON|TRUCK|. blithely even theodolites boos|
+1574|48|7|1|41|38869.64|0.06|0.02|N|O|1997-03-08|1997-02-09|1997-04-01|COLLECT COD|AIR|s. slyly regular depen|
+1574|191|5|2|50|54559.50|0.00|0.05|N|O|1996-12-14|1997-02-14|1996-12-16|TAKE BACK RETURN|FOB|le regular, regular foxes. blithely e|
+1574|55|3|3|25|23876.25|0.06|0.02|N|O|1997-01-16|1997-02-14|1997-02-12|DELIVER IN PERSON|TRUCK|ly silent accounts.|
+1574|191|4|4|6|6547.14|0.03|0.05|N|O|1997-02-24|1997-02-03|1997-03-01|NONE|AIR|e silent, final packages. speci|
+1574|109|4|5|6|6054.60|0.05|0.05|N|O|1997-02-09|1997-03-02|1997-02-14|COLLECT COD|MAIL|nic, final ideas snooze. |
+1574|5|2|6|42|38010.00|0.07|0.01|N|O|1996-12-19|1997-01-13|1996-12-28|NONE|FOB|o beans according t|
+1574|136|7|7|14|14505.82|0.04|0.01|N|O|1996-12-30|1997-01-19|1997-01-20|NONE|AIR|ily bold a|
+1575|29|10|1|42|39018.84|0.05|0.08|N|O|1995-10-21|1995-11-25|1995-10-24|DELIVER IN PERSON|RAIL|ly pending pinto beans.|
+1575|36|7|2|39|36505.17|0.00|0.06|N|O|1995-10-30|1995-10-15|1995-11-10|COLLECT COD|TRUCK| ironic requests snooze ironic, regular acc|
+1575|2|5|3|12|10824.00|0.01|0.05|N|O|1995-12-27|1995-11-11|1996-01-23|TAKE BACK RETURN|AIR| bold accounts. furi|
+1575|111|1|4|39|39433.29|0.07|0.00|N|O|1995-09-23|1995-11-05|1995-09-25|TAKE BACK RETURN|TRUCK| after the unusual asym|
+1575|83|4|5|10|9830.80|0.09|0.00|N|O|1996-01-10|1995-11-20|1996-01-13|DELIVER IN PERSON|RAIL|k excuses. pinto beans wake a|
+1575|178|6|6|14|15094.38|0.08|0.06|N|O|1995-10-31|1995-12-06|1995-11-30|NONE|AIR|beans breach among the furiously specia|
+1575|117|1|7|48|48821.28|0.08|0.04|N|O|1995-11-19|1995-10-25|1995-12-07|DELIVER IN PERSON|TRUCK|cies. regu|
+1600|172|10|1|20|21443.40|0.02|0.01|R|F|1993-06-16|1993-04-23|1993-07-02|COLLECT COD|FOB|pths sleep blithely about the|
+1600|44|3|2|48|45313.92|0.07|0.02|R|F|1993-04-17|1993-04-14|1993-05-03|DELIVER IN PERSON|FOB|furiously silent foxes could wake. car|
+1600|39|10|3|8|7512.24|0.04|0.07|R|F|1993-03-07|1993-04-22|1993-03-26|TAKE BACK RETURN|FOB|cajole furiously fluf|
+1600|69|8|4|25|24226.50|0.00|0.06|A|F|1993-05-25|1993-04-07|1993-06-05|TAKE BACK RETURN|REG AIR|press packages. ironic excuses bo|
+1600|147|8|5|30|31414.20|0.03|0.08|R|F|1993-06-03|1993-05-03|1993-06-07|DELIVER IN PERSON|RAIL|al escapades alongside of the depo|
+1601|167|8|1|6|6402.96|0.00|0.00|A|F|1994-10-19|1994-09-28|1994-10-23|COLLECT COD|SHIP| bold sheaves. furiously per|
+1601|175|3|2|50|53758.50|0.03|0.02|R|F|1994-12-24|1994-10-23|1995-01-11|COLLECT COD|FOB|ideas doubt|
+1601|90|1|3|14|13861.26|0.04|0.08|R|F|1994-09-17|1994-11-22|1994-10-03|DELIVER IN PERSON|RAIL|he special, fin|
+1602|183|4|1|4|4332.72|0.08|0.06|R|F|1993-10-31|1993-09-05|1993-11-21|NONE|RAIL|y. even excuses|
+1603|39|5|1|1|939.03|0.08|0.00|R|F|1993-08-17|1993-09-04|1993-08-22|TAKE BACK RETURN|REG AIR|d accounts. special warthogs use fur|
+1603|66|5|2|29|28015.74|0.06|0.08|A|F|1993-09-28|1993-09-20|1993-10-28|NONE|SHIP|ses wake furiously. theodolite|
+1604|42|3|1|15|14130.60|0.09|0.08|R|F|1993-09-22|1993-09-03|1993-09-29|TAKE BACK RETURN|MAIL| instructions haggle|
+1604|141|4|2|37|38522.18|0.06|0.06|A|F|1993-08-22|1993-09-21|1993-09-10|COLLECT COD|SHIP|requests. blithely ironic somas s|
+1604|114|8|3|19|19268.09|0.09|0.07|A|F|1993-10-15|1993-10-04|1993-11-09|COLLECT COD|RAIL| ideas. bol|
+1604|175|4|4|15|16127.55|0.03|0.00|R|F|1993-09-10|1993-08-31|1993-09-30|TAKE BACK RETURN|RAIL|ending realms along the special, p|
+1604|21|4|5|23|21183.46|0.08|0.05|A|F|1993-10-11|1993-08-30|1993-10-18|DELIVER IN PERSON|RAIL|en requests. blithely fin|
+1605|142|1|1|47|48980.58|0.00|0.01|N|O|1998-04-29|1998-06-12|1998-05-20|DELIVER IN PERSON|AIR|. carefully r|
+1605|180|8|2|18|19443.24|0.10|0.00|N|O|1998-05-13|1998-06-17|1998-06-03|COLLECT COD|REG AIR|ly regular foxes wake carefully. bol|
+1605|59|10|3|39|37402.95|0.02|0.03|N|O|1998-07-12|1998-06-05|1998-08-09|DELIVER IN PERSON|MAIL|nal dependencies-- quickly final frets acc|
+1605|183|4|4|25|27079.50|0.06|0.02|N|O|1998-05-26|1998-06-14|1998-06-05|COLLECT COD|AIR|ole carefully car|
+1606|115|6|1|21|21317.31|0.04|0.00|N|O|1997-06-02|1997-07-02|1997-06-27|DELIVER IN PERSON|RAIL| pending theodolites prom|
+1606|174|3|2|35|37595.95|0.00|0.02|N|O|1997-06-20|1997-06-19|1997-06-22|COLLECT COD|TRUCK|carefully sil|
+1606|100|4|3|23|23002.30|0.00|0.06|N|O|1997-04-19|1997-06-26|1997-04-30|NONE|MAIL|ously final requests. slowly ironic ex|
+1606|97|9|4|20|19941.80|0.02|0.04|N|O|1997-05-01|1997-05-26|1997-05-28|TAKE BACK RETURN|TRUCK|fily carefu|
+1606|71|10|5|14|13594.98|0.10|0.01|N|O|1997-05-19|1997-07-05|1997-06-10|COLLECT COD|FOB|structions haggle f|
+1607|190|1|1|2|2180.38|0.02|0.00|N|O|1996-01-11|1996-02-15|1996-01-19|DELIVER IN PERSON|MAIL|packages haggle. regular requests boost s|
+1607|119|3|2|37|37707.07|0.05|0.02|N|O|1996-02-27|1996-02-18|1996-03-16|NONE|AIR|alongside |
+1607|123|4|3|39|39901.68|0.00|0.00|N|O|1996-02-01|1996-02-12|1996-02-16|NONE|FOB|uches cajole. accounts ar|
+1607|76|6|4|34|33186.38|0.05|0.06|N|O|1996-01-06|1996-02-24|1996-01-10|DELIVER IN PERSON|SHIP| quickly above the |
+1607|178|8|5|48|51752.16|0.00|0.05|N|O|1996-02-22|1996-02-13|1996-03-09|TAKE BACK RETURN|MAIL|ular forges. deposits a|
+1632|191|5|1|47|51285.93|0.08|0.00|N|O|1997-01-25|1997-02-09|1997-02-19|TAKE BACK RETURN|RAIL|g to the closely special no|
+1632|148|7|2|14|14673.96|0.08|0.05|N|O|1997-01-15|1997-02-25|1997-01-28|NONE|RAIL|oxes. deposits nag slyly along the slyly |
+1632|177|6|3|47|50626.99|0.03|0.04|N|O|1997-01-29|1997-03-03|1997-02-21|NONE|MAIL|sts. blithely regular |
+1632|57|9|4|33|31582.65|0.09|0.02|N|O|1997-04-01|1997-02-24|1997-04-29|TAKE BACK RETURN|REG AIR|ructions! slyly|
+1632|142|1|5|43|44812.02|0.10|0.03|N|O|1997-02-24|1997-02-19|1997-03-25|DELIVER IN PERSON|FOB|ts. blithe, bold ideas cajo|
+1633|178|7|1|35|37735.95|0.01|0.02|N|O|1996-01-09|1995-12-02|1996-01-21|COLLECT COD|REG AIR|ly against the dolph|
+1633|5|6|2|15|13575.00|0.00|0.05|N|O|1995-12-13|1995-11-13|1996-01-04|TAKE BACK RETURN|FOB|ges wake fluffil|
+1634|48|9|1|21|19908.84|0.00|0.00|N|O|1996-10-04|1996-10-22|1996-11-01|NONE|MAIL|counts alo|
+1634|172|3|2|44|47175.48|0.05|0.01|N|O|1996-09-17|1996-11-09|1996-10-03|COLLECT COD|SHIP|requests affix slyly. quickly even pack|
+1634|19|10|3|21|19299.21|0.06|0.07|N|O|1996-11-16|1996-10-21|1996-11-27|NONE|TRUCK|y along the excuses.|
+1634|68|3|4|17|16457.02|0.08|0.07|N|O|1996-10-29|1996-10-15|1996-11-02|TAKE BACK RETURN|SHIP|cial, bold platelets alongside of the f|
+1634|76|7|5|2|1952.14|0.07|0.04|N|O|1996-11-22|1996-10-28|1996-12-17|NONE|SHIP|ly. carefully regular asymptotes wake|
+1634|170|9|6|11|11771.87|0.01|0.08|N|O|1996-10-04|1996-12-06|1996-10-14|DELIVER IN PERSON|SHIP|final requests |
+1634|13|7|7|35|31955.35|0.06|0.02|N|O|1996-11-25|1996-11-25|1996-12-12|TAKE BACK RETURN|RAIL|cies. regular, special de|
+1635|71|1|1|3|2913.21|0.06|0.08|N|O|1997-03-13|1997-03-25|1997-03-27|COLLECT COD|FOB| quickly ironic r|
+1635|90|1|2|8|7920.72|0.04|0.05|N|O|1997-04-30|1997-04-21|1997-05-09|DELIVER IN PERSON|AIR|ravely carefully express |
+1635|114|5|3|20|20282.20|0.07|0.01|N|O|1997-05-19|1997-04-01|1997-06-17|TAKE BACK RETURN|FOB|oost according to the carefully even accou|
+1635|77|5|4|40|39082.80|0.01|0.04|N|O|1997-02-25|1997-03-20|1997-03-12|TAKE BACK RETURN|RAIL|uriously up the ironic deposits. slyly i|
+1636|85|6|1|2|1970.16|0.09|0.03|N|O|1997-09-26|1997-08-22|1997-10-05|NONE|TRUCK|nal foxes cajole above the blithely reg|
+1636|169|10|2|45|48112.20|0.03|0.01|N|O|1997-07-14|1997-08-08|1997-07-27|COLLECT COD|RAIL|ely express reque|
+1636|108|1|3|24|24194.40|0.07|0.08|N|O|1997-10-07|1997-08-12|1997-11-04|TAKE BACK RETURN|MAIL|e carefully unusual ideas are f|
+1636|153|1|4|43|45285.45|0.06|0.00|N|O|1997-08-23|1997-08-10|1997-09-17|NONE|REG AIR|blithely special r|
+1636|19|6|5|22|20218.22|0.05|0.02|N|O|1997-07-22|1997-08-18|1997-08-03|COLLECT COD|AIR|ular, regu|
+1636|63|2|6|34|32744.04|0.10|0.01|N|O|1997-08-11|1997-09-09|1997-08-23|NONE|TRUCK|ular depos|
+1636|114|1|7|7|7098.77|0.04|0.00|N|O|1997-07-28|1997-09-10|1997-07-31|NONE|MAIL|ronic instructions. final|
+1637|86|7|1|49|48317.92|0.02|0.03|N|F|1995-06-08|1995-04-19|1995-07-01|COLLECT COD|REG AIR|. blithely i|
+1637|73|2|2|1|973.07|0.10|0.02|A|F|1995-02-14|1995-03-26|1995-03-09|TAKE BACK RETURN|AIR|ly final pinto beans. furiously|
+1637|22|1|3|10|9220.20|0.02|0.05|R|F|1995-02-21|1995-03-17|1995-03-11|NONE|AIR|uriously? blithely even sauternes wake. |
+1637|93|5|4|42|41709.78|0.06|0.01|A|F|1995-03-18|1995-04-24|1995-03-31|COLLECT COD|SHIP|blithely a|
+1637|5|8|5|25|22625.00|0.05|0.00|R|F|1995-06-07|1995-03-26|1995-06-08|COLLECT COD|RAIL| haggle carefully silent accou|
+1637|109|4|6|38|38345.80|0.02|0.08|R|F|1995-03-20|1995-05-05|1995-04-14|DELIVER IN PERSON|SHIP|even, pending foxes nod regular|
+1637|52|10|7|21|19993.05|0.07|0.08|A|F|1995-04-30|1995-04-30|1995-05-05|COLLECT COD|SHIP|ly ironic theodolites use b|
+1638|6|7|1|46|41676.00|0.03|0.02|N|O|1997-10-16|1997-10-28|1997-11-09|COLLECT COD|MAIL|otes haggle before the slyly bold instructi|
+1638|149|10|2|30|31474.20|0.00|0.04|N|O|1997-12-05|1997-09-17|1997-12-06|NONE|REG AIR|s cajole boldly bold requests. closely |
+1638|31|7|3|5|4655.15|0.08|0.07|N|O|1997-10-15|1997-11-01|1997-11-08|DELIVER IN PERSON|FOB|xcuses sleep furiou|
+1638|56|8|4|19|18164.95|0.00|0.08|N|O|1997-10-15|1997-10-27|1997-11-03|DELIVER IN PERSON|MAIL| quickly expres|
+1638|143|6|5|25|26078.50|0.05|0.03|N|O|1997-10-06|1997-09-30|1997-11-02|DELIVER IN PERSON|REG AIR|gle final, ironic pinto beans. |
+1638|155|10|6|46|48536.90|0.07|0.08|N|O|1997-08-20|1997-10-10|1997-09-09|COLLECT COD|AIR|ckages are carefully even instru|
+1639|187|8|1|24|26092.32|0.07|0.00|N|O|1995-08-24|1995-10-06|1995-08-31|COLLECT COD|REG AIR| the regular packages. courts dou|
+1639|43|6|2|38|35835.52|0.01|0.04|N|O|1995-08-23|1995-11-09|1995-08-29|TAKE BACK RETURN|FOB|y regular packages. b|
+1639|171|10|3|41|43917.97|0.04|0.02|N|O|1995-12-19|1995-11-11|1996-01-12|DELIVER IN PERSON|FOB|structions w|
+1664|118|5|1|48|48869.28|0.04|0.02|N|O|1996-06-21|1996-05-01|1996-07-19|TAKE BACK RETURN|RAIL| use. ironic deposits integrate. slyly unu|
+1664|173|2|2|30|32195.10|0.06|0.05|N|O|1996-04-04|1996-05-04|1996-05-03|COLLECT COD|FOB|ess multip|
+1664|151|2|3|10|10511.50|0.00|0.06|N|O|1996-04-10|1996-05-13|1996-05-07|TAKE BACK RETURN|RAIL|instructions up the acc|
+1664|155|3|4|35|36930.25|0.00|0.04|N|O|1996-03-06|1996-05-16|1996-03-09|DELIVER IN PERSON|REG AIR|y regular ide|
+1664|57|8|5|9|8613.45|0.07|0.04|N|O|1996-04-15|1996-05-14|1996-05-11|DELIVER IN PERSON|TRUCK|ges. fluffil|
+1664|141|8|6|40|41645.60|0.09|0.07|N|O|1996-04-02|1996-04-22|1996-04-17|COLLECT COD|REG AIR|se blithely unusual pains. carefully|
+1665|47|6|1|4|3788.16|0.02|0.03|A|F|1994-09-01|1994-06-07|1994-09-12|DELIVER IN PERSON|TRUCK|ely final requests. requests|
+1665|78|6|2|1|978.07|0.03|0.05|R|F|1994-05-22|1994-07-06|1994-05-24|TAKE BACK RETURN|TRUCK|sly final p|
+1666|185|6|1|30|32555.40|0.04|0.03|N|O|1995-10-28|1995-11-30|1995-11-18|TAKE BACK RETURN|AIR| breach evenly final accounts. r|
+1666|64|1|2|20|19281.20|0.01|0.00|N|O|1996-01-27|1995-12-12|1996-01-31|NONE|REG AIR|uietly regular foxes wake quick|
+1666|134|10|3|31|32058.03|0.05|0.07|N|O|1996-02-11|1996-01-11|1996-02-28|COLLECT COD|RAIL|ding to the express, bold accounts. fu|
+1666|169|8|4|41|43835.56|0.06|0.08|N|O|1995-11-29|1996-01-04|1995-12-24|NONE|TRUCK|ly regular excuses; regular ac|
+1667|21|4|1|6|5526.12|0.04|0.02|N|O|1997-12-07|1997-11-16|1998-01-02|COLLECT COD|FOB|riously busy requests. blithely final a|
+1667|22|1|2|29|26738.58|0.06|0.07|N|O|1997-10-15|1997-11-09|1997-11-11|TAKE BACK RETURN|MAIL|l accounts. furiously final courts h|
+1667|95|8|3|48|47764.32|0.05|0.01|N|O|1998-01-27|1998-01-06|1998-02-09|TAKE BACK RETURN|SHIP|tes sleep furiously. carefully eve|
+1667|59|1|4|24|23017.20|0.04|0.01|N|O|1997-10-14|1997-12-01|1997-11-09|TAKE BACK RETURN|MAIL|hrash final requests. care|
+1667|195|9|5|2|2190.38|0.07|0.00|N|O|1997-12-17|1997-11-22|1998-01-16|NONE|SHIP|pecial requests hag|
+1667|48|7|6|6|5688.24|0.01|0.03|N|O|1998-01-21|1997-12-19|1998-01-28|NONE|TRUCK| nag quickly above th|
+1667|40|6|7|19|17860.76|0.09|0.03|N|O|1998-01-23|1997-11-24|1998-01-26|DELIVER IN PERSON|SHIP|around the pinto beans. express, special|
+1668|132|8|1|8|8257.04|0.06|0.01|N|O|1997-07-23|1997-10-09|1997-08-06|DELIVER IN PERSON|FOB|arefully regular tithes! slyl|
+1668|1|8|2|25|22525.00|0.01|0.06|N|O|1997-08-08|1997-09-28|1997-09-01|NONE|TRUCK|y ironic requests. bold, final ideas a|
+1668|75|5|3|42|40952.94|0.08|0.01|N|O|1997-08-09|1997-09-08|1997-08-31|NONE|FOB|ole carefully excuses. final|
+1668|191|5|4|9|9820.71|0.05|0.03|N|O|1997-10-17|1997-09-05|1997-11-01|COLLECT COD|RAIL|wake furiously even instructions. sil|
+1668|128|9|5|25|25703.00|0.01|0.02|N|O|1997-10-08|1997-09-20|1997-10-11|DELIVER IN PERSON|REG AIR|even platelets across the silent |
+1668|10|3|6|38|34580.38|0.07|0.01|N|O|1997-08-26|1997-09-17|1997-09-05|DELIVER IN PERSON|TRUCK|ep slyly across the furi|
+1669|79|10|1|24|23497.68|0.04|0.08|N|O|1997-09-04|1997-07-30|1997-09-20|DELIVER IN PERSON|RAIL| regular, final deposits use quick|
+1670|32|3|1|41|38213.23|0.07|0.01|N|O|1997-07-19|1997-08-20|1997-07-23|DELIVER IN PERSON|TRUCK|thely according to the sly|
+1670|122|3|2|10|10221.20|0.07|0.03|N|O|1997-09-14|1997-08-16|1997-09-23|NONE|SHIP|fily special ideas |
+1670|186|7|3|41|44533.38|0.07|0.07|N|O|1997-07-19|1997-08-05|1997-07-26|COLLECT COD|SHIP|al gifts. speci|
+1671|149|2|1|21|22031.94|0.02|0.07|N|O|1996-07-28|1996-09-28|1996-08-08|TAKE BACK RETURN|AIR|s accounts slee|
+1671|96|10|2|4|3984.36|0.05|0.00|N|O|1996-08-30|1996-09-19|1996-09-23|DELIVER IN PERSON|TRUCK|lyly regular ac|
+1671|124|3|3|11|11265.32|0.06|0.08|N|O|1996-09-16|1996-10-21|1996-09-18|NONE|SHIP|tes sleep blithely|
+1671|178|7|4|5|5390.85|0.00|0.00|N|O|1996-11-14|1996-10-20|1996-11-25|TAKE BACK RETURN|FOB|luffily regular deposits|
+1671|127|8|5|12|12325.44|0.07|0.04|N|O|1996-11-17|1996-09-02|1996-12-17|COLLECT COD|RAIL|special, ironic|
+1671|197|9|6|46|50470.74|0.08|0.05|N|O|1996-09-13|1996-10-14|1996-09-28|TAKE BACK RETURN|REG AIR|. slyly bold instructions boost. furiousl|
+1696|16|3|1|8|7328.08|0.04|0.02|N|O|1998-04-28|1998-02-07|1998-05-10|NONE|TRUCK|the blithely|
+1696|139|5|2|13|13508.69|0.08|0.06|N|O|1998-03-01|1998-03-25|1998-03-24|TAKE BACK RETURN|TRUCK|tructions play slyly q|
+1696|2|5|3|19|17138.00|0.08|0.05|N|O|1998-05-03|1998-03-13|1998-05-28|TAKE BACK RETURN|REG AIR|its maintain alongside of the f|
+1696|193|4|4|21|22956.99|0.05|0.00|N|O|1998-05-04|1998-02-18|1998-05-07|NONE|MAIL|y players sleep along the final, pending |
+1696|94|7|5|43|42745.87|0.03|0.06|N|O|1998-02-14|1998-03-29|1998-02-20|COLLECT COD|FOB|arefully regular dep|
+1697|75|5|1|6|5850.42|0.05|0.00|N|O|1997-01-28|1996-11-27|1997-01-31|NONE|FOB|accounts breach slyly even de|
+1697|104|7|2|24|24098.40|0.00|0.08|N|O|1996-12-29|1996-12-19|1997-01-10|NONE|SHIP|ts cajole carefully above the carefully|
+1697|124|9|3|27|27651.24|0.06|0.00|N|O|1997-01-20|1996-12-02|1997-02-05|COLLECT COD|MAIL|ly regular packages across the silent, b|
+1697|94|5|4|49|48710.41|0.08|0.04|N|O|1996-12-07|1997-01-02|1996-12-31|COLLECT COD|TRUCK|lar foxes. fluffily furious ideas doubt qu|
+1697|35|1|5|19|17765.57|0.03|0.07|N|O|1997-01-08|1996-11-12|1997-01-11|DELIVER IN PERSON|FOB|ons? special, special accounts after|
+1698|97|8|1|44|43871.96|0.05|0.05|N|O|1997-05-16|1997-07-05|1997-05-27|NONE|RAIL|ts wake slyly after t|
+1698|93|5|2|6|5958.54|0.08|0.00|N|O|1997-08-21|1997-06-08|1997-09-03|DELIVER IN PERSON|RAIL| pending packages affix ne|
+1698|21|6|3|22|20262.44|0.03|0.04|N|O|1997-08-07|1997-05-28|1997-08-24|DELIVER IN PERSON|TRUCK|oward the furiously iro|
+1698|112|6|4|19|19230.09|0.00|0.07|N|O|1997-07-04|1997-06-21|1997-08-01|NONE|RAIL| fluffily e|
+1698|53|4|5|37|35262.85|0.00|0.03|N|O|1997-05-16|1997-05-29|1997-05-27|NONE|AIR|ly regular ideas. deposit|
+1698|166|7|6|15|15992.40|0.10|0.01|N|O|1997-07-20|1997-06-07|1997-07-21|TAKE BACK RETURN|RAIL|final ideas. even, ironic |
+1699|38|9|1|50|46901.50|0.00|0.06|A|F|1994-03-26|1994-03-23|1994-04-20|NONE|FOB|to the final requests are carefully silent |
+1699|135|6|2|17|17597.21|0.07|0.02|R|F|1994-01-12|1994-03-12|1994-02-08|NONE|AIR|haggle blithely slyly|
+1700|140|1|1|38|39525.32|0.04|0.04|N|O|1996-10-03|1996-07-27|1996-10-22|NONE|RAIL|ular dependencies engage slyly |
+1700|156|7|2|49|51751.35|0.04|0.00|N|O|1996-09-26|1996-07-28|1996-10-16|NONE|TRUCK|kly even dependencies haggle fluffi|
+1701|150|9|1|47|49357.05|0.08|0.05|R|F|1992-05-25|1992-06-29|1992-06-15|NONE|RAIL|slyly final requests cajole requests. f|
+1701|54|5|2|2|1908.10|0.01|0.04|R|F|1992-06-24|1992-07-12|1992-06-29|COLLECT COD|SHIP|ween the pending, final accounts. |
+1701|35|1|3|26|24310.78|0.10|0.06|R|F|1992-06-04|1992-07-11|1992-07-04|DELIVER IN PERSON|FOB| accounts. blithely pending pinto be|
+1702|67|2|1|19|18374.14|0.02|0.01|N|F|1995-06-02|1995-06-30|1995-06-29|NONE|REG AIR|ies haggle blith|
+1702|30|5|2|38|35341.14|0.00|0.00|N|O|1995-09-01|1995-06-10|1995-09-10|DELIVER IN PERSON|REG AIR|as believe blithely. bo|
+1702|195|6|3|46|50378.74|0.00|0.08|N|O|1995-07-14|1995-06-30|1995-07-20|NONE|FOB|y even foxes. carefully final dependencies |
+1702|93|4|4|28|27806.52|0.07|0.05|R|F|1995-06-10|1995-07-26|1995-06-16|TAKE BACK RETURN|AIR|nts haggle along the packa|
+1702|89|10|5|34|33628.72|0.01|0.06|N|O|1995-07-04|1995-06-08|1995-07-28|DELIVER IN PERSON|AIR|y careful packages; dogged acco|
+1702|42|9|6|28|26377.12|0.10|0.00|N|O|1995-08-14|1995-07-31|1995-09-08|COLLECT COD|RAIL|ackages sleep. furiously even excuses snooz|
+1703|166|5|1|36|38381.76|0.09|0.01|R|F|1993-04-22|1993-03-05|1993-04-24|DELIVER IN PERSON|SHIP|riously express |
+1703|137|8|2|35|36299.55|0.01|0.08|R|F|1993-04-14|1993-03-31|1993-04-27|NONE|RAIL|he carefully|
+1703|124|5|3|48|49157.76|0.06|0.02|R|F|1993-02-07|1993-04-20|1993-02-24|TAKE BACK RETURN|AIR|ggle slyly furiously regular theodol|
+1728|126|5|1|1|1026.12|0.07|0.04|N|O|1996-09-16|1996-08-19|1996-09-18|COLLECT COD|FOB|lly. carefully ex|
+1728|105|8|2|23|23117.30|0.05|0.02|N|O|1996-09-08|1996-07-24|1996-09-20|NONE|FOB|ns. pending, final ac|
+1728|165|10|3|44|46867.04|0.08|0.07|N|O|1996-07-31|1996-06-22|1996-08-06|COLLECT COD|FOB|ide of the slyly blithe|
+1728|27|8|4|34|31518.68|0.08|0.05|N|O|1996-08-28|1996-07-20|1996-09-12|DELIVER IN PERSON|MAIL|special req|
+1728|199|2|5|31|34074.89|0.09|0.02|N|O|1996-07-26|1996-06-28|1996-08-14|NONE|REG AIR|kly sly theodolites.|
+1729|157|8|1|12|12685.80|0.08|0.04|A|F|1992-08-11|1992-07-24|1992-08-16|COLLECT COD|RAIL|y pending packages detect. carefully re|
+1730|166|5|1|41|43712.56|0.01|0.03|N|O|1998-08-11|1998-08-29|1998-09-02|TAKE BACK RETURN|TRUCK| instructions. unusual, even Tiresi|
+1730|162|3|2|15|15932.40|0.07|0.04|N|O|1998-09-07|1998-09-12|1998-09-30|TAKE BACK RETURN|AIR|pinto beans cajole. bravely bold|
+1730|162|1|3|9|9559.44|0.10|0.00|N|O|1998-09-18|1998-09-15|1998-09-21|DELIVER IN PERSON|FOB|gular dependencies wake. blithely final e|
+1730|10|7|4|40|36400.40|0.02|0.03|N|O|1998-10-02|1998-10-06|1998-10-03|NONE|SHIP|ven dinos slee|
+1730|141|4|5|43|44769.02|0.04|0.06|N|O|1998-10-26|1998-10-22|1998-11-02|DELIVER IN PERSON|TRUCK|ng deposits cajo|
+1731|184|5|1|36|39030.48|0.10|0.00|N|O|1996-04-18|1996-04-03|1996-04-29|TAKE BACK RETURN|MAIL|ngside of the even instruct|
+1731|139|10|2|7|7273.91|0.04|0.07|N|O|1996-04-11|1996-02-13|1996-04-30|DELIVER IN PERSON|REG AIR|fily quick asymptotes|
+1731|51|9|3|50|47552.50|0.05|0.04|N|O|1996-01-14|1996-03-13|1996-01-29|COLLECT COD|RAIL|ly slyly speci|
+1731|196|10|4|23|25212.37|0.10|0.04|N|O|1996-04-22|1996-02-25|1996-05-16|TAKE BACK RETURN|RAIL|rays? bold, express pac|
+1731|53|4|5|37|35262.85|0.10|0.05|N|O|1996-04-30|1996-03-17|1996-05-27|TAKE BACK RETURN|RAIL| beans use furiously slyly b|
+1731|124|7|6|41|41988.92|0.03|0.08|N|O|1996-04-05|1996-02-28|1996-05-01|TAKE BACK RETURN|RAIL|haggle across the blithely ironi|
+1732|5|6|1|50|45250.00|0.02|0.01|R|F|1993-12-05|1994-01-23|1993-12-20|TAKE BACK RETURN|FOB|fily final asymptotes according |
+1732|99|10|2|36|35967.24|0.01|0.03|A|F|1994-03-15|1994-02-09|1994-04-02|DELIVER IN PERSON|TRUCK|ve the accounts. slowly ironic multip|
+1732|161|8|3|41|43507.56|0.00|0.04|R|F|1994-02-20|1994-01-07|1994-02-27|TAKE BACK RETURN|AIR|quests sublate against the silent |
+1732|152|3|4|9|9469.35|0.04|0.04|A|F|1994-02-25|1994-01-29|1994-03-16|TAKE BACK RETURN|FOB|ular platelets. deposits wak|
+1732|169|8|5|25|26729.00|0.02|0.05|A|F|1994-02-15|1994-01-07|1994-02-21|COLLECT COD|REG AIR|nag slyly. even, special de|
+1732|73|1|6|16|15569.12|0.01|0.05|R|F|1994-01-07|1994-01-02|1994-01-25|COLLECT COD|SHIP|ix carefully at the furiously regular pac|
+1733|111|5|1|41|41455.51|0.08|0.01|N|O|1996-06-13|1996-07-08|1996-07-07|TAKE BACK RETURN|AIR|ess notornis. fur|
+1733|24|7|2|16|14784.32|0.00|0.04|N|O|1996-08-28|1996-07-25|1996-09-27|COLLECT COD|MAIL|slyly express deposits sleep abo|
+1733|120|10|3|29|29583.48|0.10|0.06|N|O|1996-07-16|1996-08-08|1996-07-28|NONE|TRUCK|ns detect among the special accounts. qu|
+1733|136|7|4|38|39372.94|0.01|0.03|N|O|1996-08-26|1996-07-23|1996-08-28|NONE|FOB| deposits |
+1733|34|5|5|22|20548.66|0.06|0.07|N|O|1996-07-16|1996-07-24|1996-07-30|COLLECT COD|AIR|gainst the final deposits. carefully final |
+1733|66|7|6|9|8694.54|0.06|0.08|N|O|1996-05-25|1996-07-23|1996-06-10|COLLECT COD|TRUCK|ven foxes was according to t|
+1733|146|9|7|13|13599.82|0.02|0.03|N|O|1996-08-03|1996-08-02|1996-08-18|NONE|MAIL|olites sleep furious|
+1734|155|3|1|38|40095.70|0.03|0.03|R|F|1994-08-09|1994-09-07|1994-08-12|COLLECT COD|FOB|ts doubt b|
+1734|118|2|2|4|4072.44|0.06|0.03|A|F|1994-08-20|1994-07-17|1994-08-25|DELIVER IN PERSON|AIR|final warhorses.|
+1735|156|7|1|43|45414.45|0.02|0.06|A|F|1993-01-14|1993-03-25|1993-02-02|DELIVER IN PERSON|FOB|iously after the |
+1735|139|5|2|49|50917.37|0.03|0.04|A|F|1992-12-31|1993-02-03|1993-01-25|TAKE BACK RETURN|TRUCK|y express accounts above the exp|
+1760|96|9|1|38|37851.42|0.09|0.03|N|O|1996-06-15|1996-06-29|1996-07-11|NONE|MAIL|tions. blithely regular orbits against the |
+1760|8|9|2|3|2724.00|0.00|0.06|N|O|1996-07-18|1996-07-01|1996-08-01|NONE|RAIL|lyly bold dolphins haggle carefully. sl|
+1760|137|8|3|44|45633.72|0.05|0.01|N|O|1996-06-11|1996-06-16|1996-07-02|COLLECT COD|REG AIR|instructions poach slyly ironic theodolites|
+1761|52|4|1|33|31417.65|0.09|0.03|R|F|1994-01-03|1994-01-23|1994-01-31|NONE|FOB|s. excuses a|
+1761|52|3|2|37|35225.85|0.02|0.07|R|F|1994-02-17|1994-03-08|1994-03-16|NONE|RAIL| integrate. quickly unusual|
+1761|49|6|3|37|35114.48|0.06|0.04|R|F|1994-01-02|1994-03-12|1994-01-25|DELIVER IN PERSON|TRUCK|regular packages wake after|
+1761|73|1|4|49|47680.43|0.06|0.07|R|F|1994-01-08|1994-03-03|1994-02-05|TAKE BACK RETURN|FOB|y even packages promise|
+1761|157|5|5|37|39114.55|0.03|0.04|R|F|1994-04-24|1994-03-14|1994-04-29|TAKE BACK RETURN|MAIL|express requests print blithely around the|
+1761|24|7|6|12|11088.24|0.01|0.05|A|F|1994-04-16|1994-03-08|1994-04-21|DELIVER IN PERSON|AIR| sleep furiously. deposits are acco|
+1761|1|6|7|13|11713.00|0.03|0.08|R|F|1994-03-06|1994-03-18|1994-03-22|DELIVER IN PERSON|TRUCK|ons boost fu|
+1762|26|5|1|15|13890.30|0.04|0.08|A|F|1994-12-18|1994-10-29|1995-01-17|TAKE BACK RETURN|REG AIR|old packages thrash. care|
+1762|50|3|2|39|37051.95|0.10|0.02|A|F|1994-09-12|1994-11-09|1994-10-08|DELIVER IN PERSON|MAIL| ironic platelets sleep along t|
+1762|32|8|3|7|6524.21|0.05|0.01|R|F|1994-09-03|1994-10-02|1994-09-10|NONE|REG AIR|uickly express packages wake slyly-- regul|
+1762|145|2|4|24|25083.36|0.03|0.03|A|F|1994-11-30|1994-11-02|1994-12-20|NONE|REG AIR|accounts solve alongside of the fluffily |
+1762|8|9|5|49|44492.00|0.08|0.05|A|F|1994-10-20|1994-11-02|1994-11-10|TAKE BACK RETURN|SHIP| packages sleep fluffily pen|
+1762|94|7|6|35|34793.15|0.05|0.05|A|F|1994-11-25|1994-10-21|1994-11-28|COLLECT COD|AIR|ind quickly. accounts ca|
+1762|73|3|7|47|45734.29|0.03|0.01|A|F|1994-11-02|1994-10-07|1994-11-08|NONE|SHIP| blithely brave|
+1763|12|9|1|22|20064.22|0.09|0.06|N|O|1997-01-17|1997-01-15|1997-02-03|TAKE BACK RETURN|SHIP|ld. fluffily final ideas boos|
+1763|157|5|2|43|45457.45|0.04|0.04|N|O|1996-11-04|1996-12-09|1996-11-28|DELIVER IN PERSON|FOB|r deposits integrate blithely pending, quic|
+1763|25|10|3|16|14800.32|0.06|0.02|N|O|1996-12-12|1996-12-04|1996-12-25|DELIVER IN PERSON|RAIL|ously pending asymptotes a|
+1763|61|6|4|44|42286.64|0.04|0.05|N|O|1996-12-04|1997-01-06|1996-12-25|DELIVER IN PERSON|REG AIR| instructions need to integrate deposits. |
+1763|147|4|5|13|13612.82|0.03|0.05|N|O|1996-11-23|1997-01-24|1996-12-05|TAKE BACK RETURN|SHIP|s sleep carefully. fluffily unusua|
+1763|143|4|6|3|3129.42|0.05|0.03|N|O|1996-12-10|1996-12-06|1997-01-04|TAKE BACK RETURN|FOB|ut the slyly pending deposi|
+1763|184|5|7|2|2168.36|0.05|0.07|N|O|1997-02-27|1996-12-04|1997-03-27|COLLECT COD|FOB|even pinto beans snooze fluffi|
+1764|121|2|1|20|20422.40|0.09|0.02|A|F|1992-06-09|1992-05-22|1992-07-06|COLLECT COD|MAIL|y quickly regular packages. car|
+1764|67|4|2|3|2901.18|0.07|0.07|R|F|1992-05-13|1992-06-07|1992-05-26|COLLECT COD|RAIL|es wake slowly. |
+1764|78|6|3|27|26407.89|0.07|0.04|A|F|1992-05-06|1992-05-11|1992-05-23|COLLECT COD|TRUCK|ly final foxes wake blithely even requests|
+1765|161|2|1|36|38201.76|0.08|0.04|N|O|1996-03-02|1996-02-17|1996-03-14|DELIVER IN PERSON|SHIP|he blithely pending accou|
+1766|87|8|1|32|31586.56|0.08|0.01|N|O|1997-01-08|1996-11-11|1997-01-31|TAKE BACK RETURN|AIR|ess accounts. stealthily ironic accou|
+1766|34|10|2|12|11208.36|0.05|0.01|N|O|1996-10-28|1996-12-18|1996-11-15|DELIVER IN PERSON|AIR|heodolites above the final, regular acc|
+1766|111|1|3|1|1011.11|0.10|0.02|N|O|1997-01-21|1997-01-07|1997-02-19|NONE|TRUCK|ly blithely pending accounts. reg|
+1767|25|4|1|32|29600.64|0.08|0.04|A|F|1995-05-22|1995-05-14|1995-05-23|COLLECT COD|SHIP|to the bravely ironic requests i|
+1767|42|1|2|1|942.04|0.09|0.05|N|O|1995-06-23|1995-05-25|1995-07-03|TAKE BACK RETURN|RAIL|ing to the slyly fin|
+1767|174|5|3|24|25780.08|0.06|0.03|R|F|1995-03-16|1995-04-29|1995-04-11|DELIVER IN PERSON|RAIL|luffy theodolites need to detect furi|
+1767|23|8|4|50|46151.00|0.01|0.02|R|F|1995-05-29|1995-04-14|1995-06-15|NONE|REG AIR|y unusual foxe|
+1767|52|10|5|40|38082.00|0.06|0.00|R|F|1995-04-16|1995-05-06|1995-04-21|TAKE BACK RETURN|AIR|ep. accounts nag blithely fu|
+1792|88|9|1|9|8892.72|0.09|0.04|R|F|1994-02-28|1993-12-11|1994-03-12|TAKE BACK RETURN|AIR|final packages s|
+1792|9|6|2|5|4545.00|0.04|0.02|R|F|1994-02-13|1994-01-03|1994-02-28|DELIVER IN PERSON|TRUCK|ely regular accounts are slyly. pending, bo|
+1792|9|2|3|8|7272.00|0.01|0.04|A|F|1994-02-21|1994-01-26|1994-02-27|DELIVER IN PERSON|RAIL|nts. fluffily special instructions integr|
+1792|191|3|4|45|49103.55|0.00|0.01|A|F|1994-02-27|1993-12-24|1994-03-07|DELIVER IN PERSON|MAIL|ests are. ironic, regular asy|
+1792|199|2|5|35|38471.65|0.06|0.05|R|F|1994-01-31|1994-01-20|1994-02-17|NONE|FOB|e against the quic|
+1793|48|5|1|29|27493.16|0.01|0.06|R|F|1992-10-24|1992-09-20|1992-11-23|NONE|MAIL|ar excuses. |
+1793|126|9|2|4|4104.48|0.07|0.05|A|F|1992-07-28|1992-08-26|1992-08-21|COLLECT COD|RAIL|nic foxes along the even|
+1793|131|7|3|6|6186.78|0.01|0.05|R|F|1992-09-21|1992-09-05|1992-10-01|DELIVER IN PERSON|REG AIR|uctions; depo|
+1793|118|8|4|4|4072.44|0.00|0.08|R|F|1992-09-27|1992-09-21|1992-10-07|DELIVER IN PERSON|AIR|equests nod ac|
+1793|25|6|5|42|38850.84|0.03|0.03|A|F|1992-10-13|1992-10-02|1992-11-06|NONE|RAIL|uctions sleep carefully special, fl|
+1794|168|9|1|36|38453.76|0.09|0.08|N|O|1997-11-07|1997-11-01|1997-11-18|TAKE BACK RETURN|FOB|ely fluffily ironi|
+1794|95|8|2|3|2985.27|0.02|0.03|N|O|1997-11-15|1997-12-16|1997-11-20|DELIVER IN PERSON|FOB| sentiments according to the q|
+1794|117|8|3|23|23393.53|0.08|0.04|N|O|1997-10-13|1997-11-30|1997-10-28|TAKE BACK RETURN|AIR|usly unusual theodolites doze about |
+1794|85|6|4|34|33492.72|0.06|0.08|N|O|1997-09-29|1997-11-13|1997-10-07|TAKE BACK RETURN|SHIP|rs above the accoun|
+1794|117|4|5|47|47804.17|0.10|0.06|N|O|1998-01-15|1997-11-30|1998-02-14|DELIVER IN PERSON|TRUCK| haggle slyly. furiously express orbit|
+1794|91|3|6|37|36670.33|0.01|0.01|N|O|1998-01-12|1997-12-21|1998-01-17|DELIVER IN PERSON|MAIL|ackages. pinto|
+1795|137|8|1|44|45633.72|0.08|0.08|A|F|1994-04-28|1994-05-24|1994-05-27|NONE|AIR|ites sleep carefully slyly p|
+1795|114|5|2|34|34479.74|0.08|0.00|A|F|1994-04-24|1994-06-01|1994-05-08|DELIVER IN PERSON|SHIP|closely regular instructions wake. |
+1795|168|3|3|25|26704.00|0.07|0.01|A|F|1994-05-18|1994-05-22|1994-05-20|TAKE BACK RETURN|RAIL|he always express accounts ca|
+1795|125|8|4|32|32803.84|0.03|0.06|R|F|1994-05-10|1994-04-21|1994-05-17|DELIVER IN PERSON|SHIP| asymptotes across the bold,|
+1795|163|8|5|11|11694.76|0.08|0.02|R|F|1994-06-19|1994-04-24|1994-07-02|TAKE BACK RETURN|TRUCK|slyly. special pa|
+1796|10|1|1|28|25480.28|0.08|0.04|A|F|1992-12-01|1993-01-01|1992-12-24|DELIVER IN PERSON|FOB|y quickly ironic accounts.|
+1796|185|6|2|8|8681.44|0.00|0.08|R|F|1993-01-07|1993-01-04|1993-01-10|NONE|SHIP|slyly bold accounts are furiously agains|
+1797|31|7|1|17|15827.51|0.01|0.02|N|O|1996-08-06|1996-07-11|1996-08-29|NONE|TRUCK| cajole carefully. unusual Tiresias e|
+1797|145|2|2|16|16722.24|0.01|0.00|N|O|1996-06-03|1996-07-21|1996-06-07|NONE|FOB|o beans wake regular accounts. blit|
+1797|12|9|3|21|19152.21|0.02|0.01|N|O|1996-08-05|1996-08-05|1996-08-06|DELIVER IN PERSON|AIR|ns. regular, regular deposit|
+1798|109|10|1|43|43391.30|0.01|0.08|N|O|1997-08-27|1997-10-23|1997-09-09|DELIVER IN PERSON|MAIL|ld packages sleep furiously. depend|
+1799|52|10|1|8|7616.40|0.04|0.08|R|F|1994-06-14|1994-05-27|1994-06-27|TAKE BACK RETURN|MAIL|ealms upon the special, ironic waters|
+1799|27|10|2|42|38934.84|0.02|0.02|R|F|1994-04-05|1994-04-28|1994-04-09|DELIVER IN PERSON|FOB|es pending |
+1824|120|10|1|45|45905.40|0.03|0.02|R|F|1994-08-21|1994-06-21|1994-09-19|NONE|RAIL|ent Tiresias. quickly express |
+1824|69|4|2|40|38762.40|0.10|0.03|A|F|1994-05-08|1994-07-24|1994-06-06|NONE|FOB|es mold furiously final instructions. s|
+1825|156|1|1|43|45414.45|0.05|0.05|A|F|1994-02-18|1994-02-19|1994-03-02|TAKE BACK RETURN|RAIL| accounts breach fluffily spe|
+1825|148|5|2|39|40877.46|0.00|0.00|R|F|1994-04-01|1994-01-12|1994-04-21|DELIVER IN PERSON|REG AIR|ual, bold ideas haggle above the quickly ir|
+1825|17|4|3|7|6419.07|0.04|0.03|A|F|1994-01-02|1994-01-30|1994-01-30|TAKE BACK RETURN|REG AIR|fully ironic requests. requests cajole ex|
+1825|121|10|4|23|23485.76|0.05|0.01|R|F|1994-01-08|1994-02-08|1994-01-19|NONE|MAIL| wake express, even r|
+1825|178|9|5|33|35579.61|0.04|0.04|A|F|1993-12-07|1994-03-01|1993-12-16|TAKE BACK RETURN|RAIL|about the ne|
+1826|27|10|1|4|3708.08|0.06|0.00|R|F|1992-07-05|1992-06-12|1992-08-04|DELIVER IN PERSON|MAIL|alongside of the quickly unusual re|
+1826|68|3|2|9|8712.54|0.07|0.07|R|F|1992-07-12|1992-07-11|1992-07-15|DELIVER IN PERSON|TRUCK| blithely special|
+1826|176|4|3|14|15066.38|0.05|0.01|A|F|1992-04-28|1992-05-31|1992-05-25|COLLECT COD|TRUCK|uriously bold pinto beans are carefully ag|
+1826|180|9|4|6|6481.08|0.05|0.04|R|F|1992-06-30|1992-05-17|1992-07-30|DELIVER IN PERSON|RAIL|kages. blithely silent|
+1826|135|1|5|46|47615.98|0.05|0.06|R|F|1992-05-02|1992-06-25|1992-05-26|TAKE BACK RETURN|FOB|ously? quickly pe|
+1826|108|3|6|43|43348.30|0.02|0.03|A|F|1992-07-28|1992-06-14|1992-08-03|NONE|MAIL|ss tithes use even ideas. fluffily final t|
+1827|90|1|1|47|46534.23|0.00|0.01|N|O|1996-08-01|1996-08-07|1996-08-23|TAKE BACK RETURN|RAIL|. pending courts about the even e|
+1827|154|9|2|48|50599.20|0.03|0.05|N|O|1996-08-28|1996-09-15|1996-09-01|COLLECT COD|RAIL|oxes. special, final asymptote|
+1827|200|1|3|37|40707.40|0.01|0.07|N|O|1996-07-20|1996-08-18|1996-08-08|DELIVER IN PERSON|REG AIR|ously ironic theodolites serve quickly af|
+1827|127|10|4|4|4108.48|0.04|0.04|N|O|1996-07-22|1996-09-10|1996-08-11|DELIVER IN PERSON|RAIL|special requests. blithely|
+1827|80|10|5|24|23521.92|0.00|0.08|N|O|1996-08-07|1996-09-01|1996-09-04|DELIVER IN PERSON|SHIP|al gifts! re|
+1827|21|2|6|7|6447.14|0.10|0.02|N|O|1996-08-28|1996-08-07|1996-08-31|DELIVER IN PERSON|AIR|egular foxes|
+1827|6|7|7|38|34428.00|0.05|0.01|N|O|1996-10-17|1996-08-29|1996-11-07|TAKE BACK RETURN|SHIP| blithely. express, bo|
+1828|100|4|1|33|33003.30|0.05|0.04|R|F|1994-06-27|1994-06-10|1994-07-24|COLLECT COD|FOB|s boost carefully. pending d|
+1828|13|3|2|40|36520.40|0.08|0.07|R|F|1994-05-05|1994-07-02|1994-05-19|COLLECT COD|REG AIR|s use above the quietly fin|
+1828|196|7|3|11|12058.09|0.07|0.08|R|F|1994-07-21|1994-05-28|1994-08-13|DELIVER IN PERSON|FOB| wake blithely |
+1828|8|3|4|45|40860.00|0.02|0.05|R|F|1994-05-15|1994-05-29|1994-05-28|COLLECT COD|RAIL| accounts run slyly |
+1828|79|7|5|14|13706.98|0.01|0.08|A|F|1994-05-20|1994-06-02|1994-05-25|TAKE BACK RETURN|SHIP|. final packages along the carefully bold|
+1829|150|7|1|12|12601.80|0.05|0.06|A|F|1994-08-23|1994-07-13|1994-09-04|DELIVER IN PERSON|FOB|ges wake furiously express pinto|
+1829|5|6|2|11|9955.00|0.04|0.05|A|F|1994-05-18|1994-06-13|1994-06-07|COLLECT COD|MAIL|ding orbits|
+1829|104|9|3|49|49200.90|0.09|0.08|A|F|1994-08-26|1994-08-01|1994-09-16|NONE|TRUCK|ound the quickly |
+1829|153|4|4|14|14744.10|0.03|0.06|A|F|1994-08-15|1994-06-08|1994-08-30|TAKE BACK RETURN|AIR|regular deposits alongside of the flu|
+1829|166|5|5|6|6396.96|0.02|0.07|A|F|1994-08-09|1994-08-05|1994-09-05|DELIVER IN PERSON|MAIL|s haggle! slyl|
+1829|115|5|6|36|36543.96|0.09|0.04|R|F|1994-06-10|1994-06-23|1994-06-22|NONE|FOB|ackages-- express requests sleep; pen|
+1830|120|4|1|38|38764.56|0.00|0.07|R|F|1995-04-20|1995-05-22|1995-04-24|TAKE BACK RETURN|TRUCK|ely even a|
+1830|25|10|2|9|8325.18|0.05|0.07|R|F|1995-03-09|1995-05-24|1995-03-14|NONE|SHIP|st furiously among |
+1830|82|3|3|36|35354.88|0.07|0.07|R|F|1995-04-21|1995-04-14|1995-05-10|DELIVER IN PERSON|SHIP| slowly unusual orbits. carefull|
+1831|136|2|1|9|9325.17|0.02|0.03|A|F|1993-12-17|1994-01-27|1993-12-26|NONE|TRUCK|mptotes. furiously regular dolphins al|
+1831|48|9|2|9|8532.36|0.07|0.06|R|F|1994-03-22|1994-01-07|1994-04-06|COLLECT COD|MAIL|ent deposits. regular saute|
+1831|115|5|3|17|17256.87|0.02|0.08|R|F|1994-01-18|1994-02-12|1994-01-30|TAKE BACK RETURN|MAIL|s boost ironic foxe|
+1831|95|8|4|23|22887.07|0.06|0.02|R|F|1993-12-21|1994-02-08|1994-01-04|NONE|SHIP|ests. express pinto beans abou|
+1856|55|10|1|10|9550.50|0.05|0.07|R|F|1992-05-11|1992-05-20|1992-06-02|TAKE BACK RETURN|FOB|he furiously even theodolites. account|
+1856|97|10|2|47|46863.23|0.07|0.07|R|F|1992-03-22|1992-06-09|1992-04-17|DELIVER IN PERSON|FOB|ingly blithe theodolites. slyly pending |
+1856|117|7|3|20|20342.20|0.04|0.06|R|F|1992-05-04|1992-05-06|1992-05-11|DELIVER IN PERSON|MAIL|ost carefully. slyly bold accounts|
+1856|150|1|4|22|23103.30|0.08|0.02|A|F|1992-05-02|1992-05-26|1992-05-20|TAKE BACK RETURN|REG AIR|platelets detect slyly regular packages. ca|
+1856|190|1|5|14|15262.66|0.01|0.01|A|F|1992-04-14|1992-05-02|1992-05-11|COLLECT COD|SHIP|ans are even requests. deposits caj|
+1856|23|6|6|36|33228.72|0.03|0.05|A|F|1992-06-19|1992-05-12|1992-06-28|TAKE BACK RETURN|TRUCK|ly even foxes kindle blithely even realm|
+1856|130|3|7|42|43265.46|0.04|0.00|R|F|1992-05-23|1992-06-06|1992-06-19|COLLECT COD|RAIL|usly final deposits|
+1857|174|5|1|15|16112.55|0.10|0.03|R|F|1993-04-05|1993-02-28|1993-04-13|COLLECT COD|RAIL|egular, regular inst|
+1857|167|6|2|40|42686.40|0.10|0.00|R|F|1993-02-15|1993-03-08|1993-02-21|NONE|AIR|slyly close d|
+1857|119|3|3|8|8152.88|0.01|0.07|R|F|1993-01-27|1993-04-04|1993-02-20|TAKE BACK RETURN|AIR|slyly about the fluffily silent req|
+1857|100|3|4|41|41004.10|0.07|0.07|A|F|1993-04-16|1993-02-16|1993-04-18|NONE|REG AIR| the slyly|
+1858|14|8|1|33|30162.33|0.01|0.02|N|O|1997-12-28|1998-02-03|1998-01-13|NONE|RAIL|tect along the slyly final|
+1859|75|6|1|18|17551.26|0.10|0.00|N|O|1997-08-08|1997-06-30|1997-08-26|TAKE BACK RETURN|SHIP|e carefully a|
+1859|188|9|2|36|39174.48|0.02|0.01|N|O|1997-05-05|1997-07-08|1997-05-25|TAKE BACK RETURN|REG AIR|regular requests. carefully unusual theo|
+1859|158|10|3|5|5290.75|0.06|0.03|N|O|1997-06-20|1997-05-20|1997-07-19|TAKE BACK RETURN|AIR|across the p|
+1859|191|2|4|21|22914.99|0.00|0.03|N|O|1997-08-06|1997-05-29|1997-08-26|TAKE BACK RETURN|REG AIR|lar packages wake quickly exp|
+1859|46|3|5|11|10406.44|0.06|0.06|N|O|1997-07-15|1997-06-05|1997-07-29|TAKE BACK RETURN|SHIP|ffily ironic pac|
+1859|105|8|6|12|12061.20|0.08|0.03|N|O|1997-05-22|1997-06-08|1997-06-07|COLLECT COD|TRUCK|es. unusual, silent request|
+1860|113|4|1|9|9117.99|0.04|0.04|N|O|1996-08-03|1996-05-31|1996-08-04|DELIVER IN PERSON|TRUCK|c realms print carefully car|
+1861|68|5|1|7|6776.42|0.08|0.05|A|F|1994-01-14|1994-04-03|1994-01-16|COLLECT COD|RAIL|s foxes. slyly|
+1861|27|8|2|31|28737.62|0.10|0.05|R|F|1994-01-29|1994-03-07|1994-02-15|TAKE BACK RETURN|RAIL|arefully unusual|
+1861|24|9|3|23|21252.46|0.00|0.08|A|F|1994-04-09|1994-03-04|1994-04-11|DELIVER IN PERSON|MAIL|in packages sleep silent dolphins; sly|
+1861|116|6|4|38|38612.18|0.10|0.05|R|F|1994-02-26|1994-02-05|1994-03-01|NONE|RAIL|pending deposits cajole quic|
+1861|16|3|5|2|1832.02|0.03|0.08|R|F|1994-04-26|1994-03-15|1994-05-15|TAKE BACK RETURN|MAIL|e final, regular requests. carefully |
+1862|30|5|1|41|38131.23|0.10|0.00|N|O|1998-06-05|1998-05-17|1998-07-04|COLLECT COD|FOB| carefully along|
+1862|166|7|2|37|39447.92|0.06|0.02|N|O|1998-04-15|1998-05-15|1998-05-14|TAKE BACK RETURN|MAIL|l deposits. carefully even dep|
+1862|104|1|3|26|26106.60|0.02|0.01|N|O|1998-03-25|1998-05-17|1998-04-17|TAKE BACK RETURN|TRUCK|g carefully: thinly ironic deposits af|
+1863|63|2|1|48|46226.88|0.09|0.04|A|F|1993-10-10|1993-12-09|1993-10-19|NONE|FOB|ans hinder furiou|
+1863|157|2|2|48|50743.20|0.04|0.08|A|F|1993-11-08|1993-11-05|1993-12-08|COLLECT COD|AIR|onic theodolites alongside of the pending a|
+1888|98|10|1|27|26948.43|0.03|0.06|R|F|1994-02-13|1994-01-16|1994-02-25|NONE|REG AIR|. carefully special dolphins sle|
+1888|74|5|2|38|37014.66|0.03|0.03|R|F|1993-11-29|1994-01-16|1993-12-08|TAKE BACK RETURN|TRUCK|dazzle carefull|
+1888|80|1|3|49|48023.92|0.07|0.05|A|F|1994-02-27|1994-01-14|1994-03-28|DELIVER IN PERSON|FOB|lar accounts haggle carefu|
+1888|19|10|4|9|8271.09|0.01|0.04|A|F|1994-02-09|1994-01-22|1994-02-19|NONE|AIR| packages are blithely. carefu|
+1888|160|1|5|4|4240.64|0.03|0.06|R|F|1993-12-28|1993-12-19|1994-01-11|COLLECT COD|FOB|lphins. ironically special theodolit|
+1888|53|8|6|48|45746.40|0.08|0.08|R|F|1994-02-28|1993-12-16|1994-03-15|COLLECT COD|TRUCK|ar ideas cajole. regular p|
+1888|167|6|7|50|53358.00|0.04|0.07|R|F|1993-12-22|1994-01-10|1994-01-06|DELIVER IN PERSON|FOB|ependencies affix blithely regular warhors|
+1889|152|4|1|41|43138.15|0.10|0.02|N|O|1997-06-15|1997-05-10|1997-07-08|NONE|AIR|s! furiously pending r|
+1889|172|3|2|13|13938.21|0.05|0.00|N|O|1997-06-12|1997-04-28|1997-06-23|NONE|REG AIR|to the regular accounts. carefully express|
+1889|138|9|3|36|37372.68|0.05|0.07|N|O|1997-05-19|1997-06-14|1997-05-23|NONE|SHIP|l pinto beans kindle |
+1889|168|5|4|5|5340.80|0.02|0.07|N|O|1997-06-26|1997-06-09|1997-07-21|COLLECT COD|AIR|ording to the blithely silent r|
+1890|141|8|1|26|27069.64|0.03|0.07|N|O|1997-04-02|1997-03-13|1997-04-22|DELIVER IN PERSON|FOB|ngage. slyly ironic |
+1890|100|1|2|43|43004.30|0.07|0.03|N|O|1996-12-30|1997-01-31|1997-01-19|DELIVER IN PERSON|FOB|p ironic, express accounts. fu|
+1890|59|1|3|24|23017.20|0.06|0.04|N|O|1997-02-09|1997-02-10|1997-02-12|COLLECT COD|MAIL|is wake carefully above the even id|
+1890|68|9|4|43|41626.58|0.09|0.04|N|O|1997-04-08|1997-02-19|1997-04-30|TAKE BACK RETURN|FOB|lyly. instructions across the furiously|
+1890|122|3|5|45|45995.40|0.08|0.05|N|O|1997-04-15|1997-03-16|1997-04-19|COLLECT COD|FOB|he carefully regular sauternes. ironic fret|
+1890|181|2|6|16|17298.88|0.08|0.02|N|O|1997-02-13|1997-02-18|1997-03-12|TAKE BACK RETURN|TRUCK|ged pinto beans. regular, regular id|
+1890|121|4|7|10|10211.20|0.01|0.04|N|O|1996-12-24|1997-02-19|1997-01-01|DELIVER IN PERSON|AIR|. even, unusual inst|
+1891|77|8|1|45|43968.15|0.07|0.04|A|F|1994-12-20|1995-01-16|1995-01-05|NONE|RAIL|ests along|
+1891|184|5|2|18|19515.24|0.06|0.00|A|F|1995-01-24|1995-01-29|1995-02-14|NONE|RAIL| foxes above the carefu|
+1891|198|9|3|15|16472.85|0.03|0.00|R|F|1995-03-11|1995-03-05|1995-03-18|TAKE BACK RETURN|MAIL| accounts are furiou|
+1892|113|7|1|48|48629.28|0.02|0.01|A|F|1994-06-16|1994-06-16|1994-06-28|NONE|RAIL|tornis detect regul|
+1892|43|2|2|35|33006.40|0.04|0.08|R|F|1994-04-05|1994-05-09|1994-05-03|NONE|MAIL|hes nod furiously around the instruc|
+1892|134|5|3|37|38262.81|0.10|0.03|R|F|1994-04-11|1994-06-04|1994-04-24|TAKE BACK RETURN|SHIP|nts. slyly regular asymptot|
+1892|197|9|4|14|15360.66|0.06|0.07|R|F|1994-04-08|1994-06-12|1994-04-27|DELIVER IN PERSON|FOB|furiously about the furiously|
+1893|99|1|1|43|42960.87|0.10|0.00|N|O|1998-01-25|1998-01-06|1998-02-14|COLLECT COD|SHIP|he carefully regular |
+1893|148|9|2|49|51358.86|0.03|0.05|N|O|1998-01-19|1998-01-28|1998-02-02|TAKE BACK RETURN|FOB|y final foxes bo|
+1893|45|6|3|3|2835.12|0.03|0.02|N|O|1998-02-10|1998-01-18|1998-02-25|DELIVER IN PERSON|MAIL|gular, even ideas. fluffily bol|
+1893|101|6|4|18|18019.80|0.07|0.06|N|O|1998-01-24|1998-01-12|1998-02-13|TAKE BACK RETURN|RAIL|g packages. fluffily final reques|
+1893|53|4|5|6|5718.30|0.10|0.02|N|O|1998-01-23|1997-12-22|1998-02-09|DELIVER IN PERSON|TRUCK|ar accounts use. daringly ironic packag|
+1894|169|10|1|40|42766.40|0.03|0.07|R|F|1992-06-07|1992-05-11|1992-07-01|DELIVER IN PERSON|FOB|ily furiously bold packages. flu|
+1895|161|6|1|43|45629.88|0.09|0.07|R|F|1994-07-26|1994-07-19|1994-08-11|NONE|AIR| carefully eve|
+1920|96|7|1|24|23906.16|0.04|0.05|N|O|1998-09-27|1998-08-23|1998-10-15|DELIVER IN PERSON|AIR|thely. bold, pend|
+1920|51|6|2|31|29482.55|0.05|0.06|N|O|1998-08-01|1998-08-30|1998-08-17|COLLECT COD|SHIP|lly. ideas wa|
+1920|18|2|3|6|5508.06|0.01|0.05|N|O|1998-10-01|1998-08-20|1998-10-24|COLLECT COD|SHIP|l ideas boost slyly pl|
+1920|84|5|4|50|49204.00|0.09|0.06|N|O|1998-10-03|1998-08-04|1998-10-29|DELIVER IN PERSON|MAIL|e blithely unusual foxes. brave packages|
+1920|34|10|5|14|13076.42|0.08|0.05|N|O|1998-10-22|1998-08-10|1998-10-27|DELIVER IN PERSON|AIR|ickly ironic d|
+1921|21|10|1|9|8289.18|0.08|0.00|R|F|1994-02-01|1994-03-20|1994-03-01|DELIVER IN PERSON|FOB|to beans. even excuses integrate specia|
+1921|140|6|2|21|21842.94|0.02|0.06|R|F|1994-02-08|1994-03-28|1994-02-15|COLLECT COD|FOB|ckly regula|
+1921|71|2|3|27|26218.89|0.00|0.04|A|F|1994-04-26|1994-04-07|1994-04-30|TAKE BACK RETURN|FOB|ing pinto beans above the pend|
+1922|10|5|1|13|11830.13|0.05|0.03|N|O|1996-10-24|1996-09-21|1996-11-15|NONE|SHIP|quests. furiously|
+1923|37|8|1|9|8433.27|0.01|0.08|N|O|1997-08-29|1997-09-13|1997-09-07|NONE|FOB|lites. ironic instructions integrate bravel|
+1923|178|8|2|23|24797.91|0.07|0.05|N|O|1997-09-08|1997-08-11|1997-09-14|TAKE BACK RETURN|MAIL|aggle carefully. furiously permanent|
+1923|180|1|3|11|11881.98|0.03|0.03|N|O|1997-07-12|1997-09-04|1997-08-01|TAKE BACK RETURN|REG AIR|ages wake slyly about the furiously regular|
+1923|193|5|4|49|53566.31|0.06|0.05|N|O|1997-07-21|1997-08-08|1997-07-26|NONE|AIR|de of the carefully expre|
+1923|184|5|5|25|27104.50|0.10|0.08|N|O|1997-08-18|1997-08-20|1997-09-12|DELIVER IN PERSON|TRUCK|the ideas: slyly pendin|
+1923|37|3|6|50|46851.50|0.03|0.03|N|O|1997-11-04|1997-08-08|1997-11-25|NONE|TRUCK|uickly along the bold courts. bold the|
+1924|73|1|1|7|6811.49|0.06|0.07|N|O|1997-01-01|1996-12-02|1997-01-08|COLLECT COD|SHIP|osits. even accounts nag furious|
+1924|18|8|2|47|43146.47|0.02|0.06|N|O|1996-11-24|1996-10-18|1996-12-13|COLLECT COD|REG AIR|silent requests cajole blithely final pack|
+1924|57|8|3|40|38282.00|0.04|0.08|N|O|1996-10-31|1996-11-30|1996-11-21|NONE|REG AIR|ains sleep carefully|
+1924|34|5|4|31|28954.93|0.03|0.03|N|O|1996-09-20|1996-10-19|1996-10-19|DELIVER IN PERSON|SHIP| the slyly regular foxes. ruthle|
+1924|36|7|5|17|15912.51|0.04|0.05|N|O|1996-12-31|1996-11-12|1997-01-25|COLLECT COD|TRUCK|e carefully theodolites. ironically ironic |
+1924|76|4|6|15|14641.05|0.02|0.04|N|O|1997-01-04|1996-11-13|1997-01-27|NONE|SHIP|he package|
+1924|40|1|7|21|19740.84|0.09|0.03|N|O|1996-09-21|1996-11-12|1996-10-02|TAKE BACK RETURN|AIR| blithely reg|
+1925|184|5|1|50|54209.00|0.01|0.02|R|F|1992-04-12|1992-04-23|1992-05-08|TAKE BACK RETURN|TRUCK|usual pinto|
+1925|135|1|2|35|36229.55|0.06|0.06|R|F|1992-05-11|1992-04-10|1992-05-14|TAKE BACK RETURN|AIR|counts. carefully ironic packages boost ab|
+1925|116|10|3|40|40644.40|0.08|0.08|A|F|1992-05-17|1992-05-20|1992-06-08|TAKE BACK RETURN|AIR|e carefully regul|
+1925|30|5|4|17|15810.51|0.06|0.02|R|F|1992-05-18|1992-04-06|1992-06-16|TAKE BACK RETURN|MAIL|instructions sleep. pinto bea|
+1926|51|9|1|24|22825.20|0.06|0.05|N|O|1996-05-04|1996-03-14|1996-06-01|DELIVER IN PERSON|RAIL|e theodolites.|
+1926|106|9|2|29|29176.90|0.09|0.08|N|O|1996-02-26|1996-03-14|1996-03-14|TAKE BACK RETURN|TRUCK|es. dependencies according to the fl|
+1926|178|6|3|10|10781.70|0.02|0.03|N|O|1996-05-23|1996-03-02|1996-06-04|NONE|AIR|usly bold accounts. express accounts|
+1926|68|9|4|13|12584.78|0.04|0.02|N|O|1996-04-26|1996-04-13|1996-05-08|DELIVER IN PERSON|MAIL|eans wake bli|
+1926|40|1|5|29|27261.16|0.06|0.00|N|O|1996-02-29|1996-03-13|1996-03-24|DELIVER IN PERSON|MAIL|hily unusual packages are fluffily am|
+1927|68|5|1|3|2904.18|0.00|0.05|N|O|1995-10-06|1995-12-08|1995-11-05|COLLECT COD|FOB|ccounts affi|
+1927|73|2|2|15|14596.05|0.08|0.08|N|O|1995-12-25|1995-12-26|1995-12-31|COLLECT COD|RAIL| carefully regular requests sleep car|
+1927|65|10|3|6|5790.36|0.05|0.05|N|O|1995-11-29|1995-11-20|1995-12-08|TAKE BACK RETURN|TRUCK|furiously even wat|
+1952|53|8|1|7|6671.35|0.04|0.05|A|F|1994-05-06|1994-06-11|1994-05-12|NONE|RAIL|about the express, even requ|
+1952|142|5|2|6|6252.84|0.06|0.05|A|F|1994-05-09|1994-05-21|1994-05-26|DELIVER IN PERSON|AIR|packages haggle. |
+1953|128|1|1|25|25703.00|0.07|0.06|A|F|1994-01-07|1994-01-28|1994-01-29|TAKE BACK RETURN|RAIL|ular, regular i|
+1953|14|5|2|35|31990.35|0.06|0.06|R|F|1994-02-03|1994-02-25|1994-02-14|DELIVER IN PERSON|FOB|among the fur|
+1954|152|7|1|31|32616.65|0.06|0.06|N|O|1997-08-18|1997-07-07|1997-09-03|DELIVER IN PERSON|RAIL|against the packages. bold, ironic e|
+1954|182|3|2|1|1082.18|0.03|0.01|N|O|1997-09-16|1997-07-08|1997-10-07|COLLECT COD|MAIL|te. furiously final deposits hag|
+1954|199|2|3|11|12091.09|0.07|0.07|N|O|1997-08-07|1997-07-23|1997-08-25|DELIVER IN PERSON|TRUCK|y carefully ironi|
+1954|159|4|4|12|12709.80|0.02|0.08|N|O|1997-07-19|1997-07-04|1997-08-06|COLLECT COD|AIR|ongside of the slyly unusual requests. reg|
+1954|170|7|5|29|31034.93|0.08|0.08|N|O|1997-08-25|1997-07-15|1997-09-02|DELIVER IN PERSON|RAIL|use thinly furiously regular asy|
+1954|177|8|6|13|14003.21|0.00|0.07|N|O|1997-06-15|1997-08-22|1997-06-20|TAKE BACK RETURN|MAIL|y ironic instructions cajole|
+1954|194|5|7|49|53615.31|0.05|0.06|N|O|1997-06-04|1997-08-29|1997-06-14|COLLECT COD|TRUCK|eans. final pinto beans sleep furiousl|
+1955|137|3|1|32|33188.16|0.02|0.02|A|F|1992-07-05|1992-06-29|1992-08-03|TAKE BACK RETURN|TRUCK|g to the carefully sile|
+1955|18|8|2|2|1836.02|0.03|0.01|R|F|1992-07-06|1992-07-06|1992-08-01|COLLECT COD|TRUCK|ickly aroun|
+1955|158|6|3|41|43384.15|0.08|0.06|A|F|1992-08-01|1992-06-04|1992-08-07|COLLECT COD|AIR| carefully against the furiously reg|
+1955|9|4|4|16|14544.00|0.03|0.07|A|F|1992-04-30|1992-06-23|1992-05-23|TAKE BACK RETURN|FOB|odolites eat s|
+1955|159|10|5|11|11650.65|0.09|0.01|A|F|1992-06-03|1992-07-04|1992-06-07|NONE|REG AIR|ously quickly pendi|
+1956|177|8|1|8|8617.36|0.02|0.04|A|F|1992-12-25|1992-11-24|1993-01-12|TAKE BACK RETURN|AIR|efully about the ironic, ironic de|
+1956|103|6|2|16|16049.60|0.00|0.05|R|F|1992-11-11|1992-11-11|1992-11-30|NONE|FOB|es cajole blithely. pen|
+1956|139|5|3|39|40526.07|0.08|0.02|A|F|1992-09-24|1992-11-26|1992-10-15|DELIVER IN PERSON|REG AIR|r theodolites sleep above the b|
+1956|29|10|4|11|10219.22|0.10|0.00|A|F|1992-12-19|1992-10-29|1993-01-07|TAKE BACK RETURN|AIR| the braids slee|
+1956|155|10|5|16|16882.40|0.08|0.02|R|F|1992-09-28|1992-10-21|1992-09-30|TAKE BACK RETURN|FOB| wake after the |
+1957|79|9|1|50|48953.50|0.09|0.05|N|O|1998-08-08|1998-09-28|1998-08-27|COLLECT COD|FOB|gainst the re|
+1957|119|3|2|31|31592.41|0.10|0.08|N|O|1998-08-13|1998-08-31|1998-08-16|NONE|REG AIR|express packages maintain fluffi|
+1958|73|2|1|9|8757.63|0.01|0.05|N|O|1995-12-08|1995-12-17|1995-12-18|DELIVER IN PERSON|REG AIR|ickly. slyly bold |
+1958|176|7|2|29|31208.93|0.05|0.06|N|O|1996-01-19|1995-12-05|1996-02-14|COLLECT COD|SHIP|d pinto beans|
+1958|102|3|3|4|4008.40|0.04|0.02|N|O|1995-10-24|1995-12-09|1995-10-28|DELIVER IN PERSON|AIR|he slyly even dependencies |
+1958|83|4|4|38|37357.04|0.09|0.07|N|O|1995-10-09|1995-11-26|1995-11-05|COLLECT COD|TRUCK|yly. slyly regular courts use silentl|
+1958|101|8|5|31|31034.10|0.08|0.01|N|O|1995-10-31|1995-11-12|1995-11-07|TAKE BACK RETURN|TRUCK|r deposits c|
+1958|17|4|6|44|40348.44|0.08|0.04|N|O|1995-12-17|1995-11-30|1996-01-15|TAKE BACK RETURN|RAIL|c theodolites after the unusual deposit|
+1958|39|5|7|29|27231.87|0.02|0.05|N|O|1995-10-14|1995-11-06|1995-11-01|NONE|REG AIR|final requests nag according to the |
+1959|169|10|1|46|49181.36|0.04|0.00|N|O|1997-05-05|1997-03-03|1997-05-24|TAKE BACK RETURN|AIR| furiously ex|
+1959|120|7|2|15|15301.80|0.08|0.07|N|O|1997-01-20|1997-02-18|1997-02-08|DELIVER IN PERSON|MAIL| quickly sp|
+1984|53|5|1|45|42887.25|0.03|0.04|N|O|1998-04-09|1998-06-11|1998-05-01|COLLECT COD|AIR|p. quickly final ideas sle|
+1984|70|7|2|35|33952.45|0.01|0.07|N|O|1998-05-18|1998-05-04|1998-06-01|COLLECT COD|RAIL|tes. quickly pending packages haggle boldl|
+1985|28|1|1|33|30624.66|0.10|0.03|R|F|1994-12-04|1994-11-01|1994-12-05|DELIVER IN PERSON|FOB|s are express packages. pendin|
+1985|21|6|2|50|46051.00|0.04|0.02|R|F|1994-09-30|1994-10-18|1994-10-12|COLLECT COD|AIR|ate carefully. carefully|
+1985|134|10|3|20|20682.60|0.07|0.03|R|F|1994-10-29|1994-11-12|1994-11-27|NONE|TRUCK|regular requests. furiously express|
+1985|199|10|4|30|32975.70|0.05|0.07|R|F|1994-09-06|1994-10-10|1994-09-26|NONE|RAIL|uickly. instr|
+1985|124|9|5|42|43013.04|0.05|0.05|R|F|1994-10-25|1994-11-03|1994-11-19|DELIVER IN PERSON|SHIP| patterns? final requests after the sp|
+1985|20|7|6|2|1840.04|0.02|0.00|A|F|1994-11-25|1994-10-09|1994-12-25|TAKE BACK RETURN|FOB| silent inst|
+1986|92|3|1|12|11905.08|0.06|0.05|A|F|1994-08-17|1994-06-28|1994-09-02|COLLECT COD|RAIL|sleep furiously fluffily final|
+1986|105|8|2|10|10051.00|0.10|0.03|R|F|1994-05-14|1994-06-21|1994-06-02|COLLECT COD|REG AIR|yly into the carefully even |
+1986|63|2|3|14|13482.84|0.04|0.02|R|F|1994-07-14|1994-06-19|1994-08-08|NONE|SHIP|the packages. pending, unusual|
+1987|16|6|1|7|6412.07|0.03|0.03|A|F|1994-07-30|1994-07-06|1994-08-29|NONE|REG AIR| regular a|
+1988|72|1|1|36|34994.52|0.09|0.04|N|O|1996-01-21|1995-11-24|1996-01-27|NONE|RAIL|gular theodolites. |
+1988|199|3|2|19|20884.61|0.08|0.08|N|O|1996-02-03|1995-12-10|1996-02-14|COLLECT COD|FOB|lly about the slyly thin instructions. f|
+1988|54|6|3|8|7632.40|0.06|0.01|N|O|1995-10-20|1995-11-11|1995-11-18|DELIVER IN PERSON|AIR|le quickly ac|
+1988|36|2|4|27|25272.81|0.08|0.00|N|O|1996-01-27|1995-12-24|1996-02-24|TAKE BACK RETURN|TRUCK|uests. regular requests are according to t|
+1988|79|8|5|26|25455.82|0.08|0.04|N|O|1996-01-25|1995-12-15|1996-01-26|COLLECT COD|SHIP| ironic dolphins haggl|
+1988|86|7|6|9|8874.72|0.08|0.03|N|O|1995-12-26|1996-01-02|1996-01-25|DELIVER IN PERSON|MAIL|lar platelets. slyly ironic packa|
+1989|10|7|1|47|42770.47|0.10|0.02|R|F|1994-06-21|1994-05-27|1994-06-22|TAKE BACK RETURN|REG AIR|final deposits s|
+1990|101|2|1|46|46050.60|0.01|0.07|R|F|1994-12-29|1995-03-14|1995-01-13|NONE|TRUCK|ar sentiments.|
+1991|110|3|1|39|39394.29|0.06|0.02|A|F|1993-01-01|1992-11-29|1993-01-10|TAKE BACK RETURN|TRUCK|ckages? carefully bold depos|
+1991|53|1|2|49|46699.45|0.08|0.06|R|F|1992-10-19|1992-11-29|1992-10-25|NONE|SHIP|nd the ideas affi|
+1991|174|5|3|6|6445.02|0.02|0.01|A|F|1992-11-02|1992-10-08|1992-11-14|TAKE BACK RETURN|REG AIR|hes nag slyly|
+1991|138|9|4|6|6228.78|0.10|0.06|A|F|1992-11-21|1992-11-03|1992-11-27|NONE|RAIL|uickly blithely final de|
+1991|60|8|5|49|47042.94|0.06|0.00|R|F|1992-09-10|1992-11-30|1992-10-07|NONE|AIR|quests cajole blithely|
+2016|147|4|1|2|2094.28|0.02|0.07|N|O|1996-10-12|1996-11-09|1996-10-31|DELIVER IN PERSON|TRUCK|carefully according to the |
+2016|63|8|2|15|14445.90|0.04|0.05|N|O|1996-09-24|1996-10-05|1996-10-21|TAKE BACK RETURN|MAIL|uests haggle carefully furiously regul|
+2016|122|7|3|8|8176.96|0.09|0.02|N|O|1996-09-19|1996-10-21|1996-10-13|TAKE BACK RETURN|SHIP|mptotes haggle ideas. packages wake flu|
+2017|103|4|1|49|49151.90|0.10|0.06|N|O|1998-05-26|1998-07-01|1998-06-06|COLLECT COD|TRUCK| after the unusual instructions. sly|
+2017|71|2|2|14|13594.98|0.07|0.04|N|O|1998-06-28|1998-06-15|1998-07-11|NONE|TRUCK|ily final w|
+2017|84|5|3|11|10824.88|0.05|0.02|N|O|1998-05-22|1998-07-13|1998-05-26|TAKE BACK RETURN|TRUCK|gside of the slyly dogged dolp|
+2018|195|6|1|2|2190.38|0.02|0.07|N|O|1995-06-25|1995-06-20|1995-07-04|NONE|TRUCK|ly ironic accounts against the slyly sly|
+2018|129|10|2|23|23669.76|0.05|0.01|R|F|1995-05-05|1995-05-12|1995-05-22|TAKE BACK RETURN|RAIL|ingly even theodolites s|
+2019|4|9|1|31|28024.00|0.07|0.03|R|F|1992-11-18|1992-12-26|1992-11-24|DELIVER IN PERSON|FOB|l ideas across the slowl|
+2019|52|7|2|18|17136.90|0.04|0.03|R|F|1993-01-24|1992-12-22|1993-02-02|NONE|MAIL|are carefully furiously regular requ|
+2020|34|10|1|50|46701.50|0.06|0.01|R|F|1993-07-12|1993-08-28|1993-08-02|COLLECT COD|TRUCK|ts against the pending ideas serve along|
+2020|176|4|2|40|43046.80|0.09|0.00|A|F|1993-10-17|1993-09-14|1993-10-29|TAKE BACK RETURN|RAIL|ently across the|
+2020|14|4|3|30|27420.30|0.07|0.04|A|F|1993-09-08|1993-08-11|1993-09-29|TAKE BACK RETURN|AIR|ly about the blithely ironic foxes. bold|
+2020|61|8|4|27|25948.62|0.05|0.06|A|F|1993-07-14|1993-09-02|1993-08-03|NONE|FOB|e of the bold foxes haggle |
+2021|85|6|1|7|6895.56|0.08|0.04|N|O|1995-10-17|1995-09-29|1995-10-20|NONE|MAIL| accounts boost blithely. blithely reg|
+2021|166|3|2|19|20257.04|0.04|0.05|N|O|1995-08-14|1995-09-05|1995-08-23|NONE|RAIL| above the slyly fl|
+2022|169|8|1|38|40628.08|0.00|0.08|R|F|1992-07-05|1992-04-20|1992-07-13|TAKE BACK RETURN|REG AIR| against the express accounts wake ca|
+2022|55|3|2|38|36291.90|0.05|0.04|R|F|1992-06-17|1992-05-15|1992-06-28|COLLECT COD|SHIP|instructions dazzle carefull|
+2022|49|10|3|48|45553.92|0.10|0.02|A|F|1992-06-14|1992-06-04|1992-07-12|DELIVER IN PERSON|SHIP|counts. slyly enticing accounts are during |
+2022|182|3|4|16|17314.88|0.05|0.03|R|F|1992-06-23|1992-05-22|1992-07-07|NONE|TRUCK|ages wake slyly care|
+2022|100|1|5|36|36003.60|0.05|0.02|R|F|1992-03-24|1992-05-07|1992-04-13|NONE|MAIL|ly after the foxes. regular, final inst|
+2022|129|2|6|20|20582.40|0.08|0.08|A|F|1992-03-31|1992-04-17|1992-04-02|NONE|SHIP|r deposits kindle |
+2022|78|9|7|13|12714.91|0.06|0.08|R|F|1992-04-04|1992-05-30|1992-04-21|NONE|FOB| orbits haggle fluffily fl|
+2023|127|10|1|9|9244.08|0.05|0.04|R|F|1992-06-04|1992-06-30|1992-06-10|NONE|AIR|ly regular pinto beans poa|
+2023|38|4|2|2|1876.06|0.01|0.00|R|F|1992-08-27|1992-07-16|1992-08-29|DELIVER IN PERSON|RAIL|ing packages. fluffily silen|
+2023|19|6|3|25|22975.25|0.10|0.03|A|F|1992-07-19|1992-07-07|1992-08-15|NONE|REG AIR| wake furiously among the slyly final|
+2023|185|6|4|9|9766.62|0.02|0.00|A|F|1992-07-23|1992-07-04|1992-08-20|TAKE BACK RETURN|AIR|nts maintain blithely alongside of the|
+2023|20|10|5|22|20240.44|0.04|0.06|A|F|1992-06-15|1992-07-13|1992-06-21|TAKE BACK RETURN|SHIP|ronic attainments. |
+2023|43|2|6|29|27348.16|0.02|0.06|A|F|1992-08-29|1992-07-28|1992-09-18|COLLECT COD|RAIL|usual instructions. bli|
+2023|134|10|7|50|51706.50|0.00|0.03|R|F|1992-06-20|1992-07-04|1992-06-23|DELIVER IN PERSON|FOB|its! carefully ex|
+2048|35|1|1|7|6545.21|0.06|0.01|R|F|1993-12-07|1994-01-31|1994-01-05|TAKE BACK RETURN|REG AIR|lent platelets boost deposits. carefully sp|
+2048|8|5|2|5|4540.00|0.04|0.04|A|F|1994-01-18|1994-02-01|1994-01-29|TAKE BACK RETURN|TRUCK|affix carefully against |
+2048|101|2|3|12|12013.20|0.01|0.05|R|F|1994-01-28|1994-01-19|1994-02-08|NONE|AIR| even theodoli|
+2048|97|1|4|11|10967.99|0.10|0.03|R|F|1993-12-20|1994-01-19|1994-01-04|TAKE BACK RETURN|MAIL|totes. idly ironic packages nag|
+2049|189|10|1|25|27229.50|0.08|0.00|N|O|1996-03-31|1996-02-29|1996-04-15|DELIVER IN PERSON|MAIL| excuses above the |
+2049|35|1|2|31|28985.93|0.10|0.05|N|O|1995-12-25|1996-02-25|1995-12-29|TAKE BACK RETURN|MAIL| packages are slyly alongside|
+2049|67|6|3|18|17407.08|0.05|0.05|N|O|1996-01-09|1996-01-22|1996-01-25|TAKE BACK RETURN|AIR| sleep fluffily. dependencies use never|
+2049|6|7|4|39|35334.00|0.02|0.05|N|O|1996-01-17|1996-01-21|1996-02-03|TAKE BACK RETURN|MAIL|the even pinto beans |
+2049|126|1|5|30|30783.60|0.04|0.06|N|O|1995-12-16|1996-02-04|1995-12-22|NONE|TRUCK|ial accounts are among the furiously perma|
+2049|84|5|6|17|16729.36|0.07|0.00|N|O|1996-02-04|1996-03-01|1996-02-24|NONE|FOB|al, regular foxes. pending, |
+2050|73|2|1|47|45734.29|0.05|0.03|A|F|1994-08-25|1994-07-18|1994-09-15|DELIVER IN PERSON|TRUCK|tside the blithely pending packages eat f|
+2050|152|3|2|48|50503.20|0.05|0.01|A|F|1994-09-30|1994-08-23|1994-10-29|COLLECT COD|AIR| final packages. pinto|
+2050|113|4|3|41|41537.51|0.10|0.04|A|F|1994-06-08|1994-08-27|1994-06-23|NONE|AIR| final theodolites. depende|
+2050|32|8|4|11|10252.33|0.02|0.01|A|F|1994-07-27|1994-08-18|1994-08-02|DELIVER IN PERSON|REG AIR|ns. bold, final ideas cajole among the fi|
+2050|168|9|5|16|17090.56|0.07|0.01|R|F|1994-08-17|1994-07-28|1994-09-05|DELIVER IN PERSON|REG AIR|al accounts. closely even |
+2050|49|2|6|29|27522.16|0.00|0.05|A|F|1994-09-23|1994-08-01|1994-10-23|TAKE BACK RETURN|MAIL|oxes alongsid|
+2050|48|5|7|25|23701.00|0.10|0.00|R|F|1994-08-18|1994-07-04|1994-09-04|TAKE BACK RETURN|RAIL|y according to |
+2051|25|6|1|43|39775.86|0.08|0.04|N|O|1996-04-22|1996-06-16|1996-04-28|COLLECT COD|RAIL|ounts sleep fluffily even requ|
+2051|130|1|2|48|49446.24|0.01|0.02|N|O|1996-05-04|1996-06-14|1996-05-19|NONE|TRUCK|unts. pending platelets believe about|
+2052|68|7|1|50|48403.00|0.09|0.08|R|F|1992-06-22|1992-06-03|1992-07-19|DELIVER IN PERSON|AIR|wake after the decoy|
+2052|135|1|2|35|36229.55|0.09|0.05|A|F|1992-05-29|1992-05-24|1992-06-11|NONE|TRUCK|ts according t|
+2052|43|2|3|16|15088.64|0.01|0.08|A|F|1992-06-30|1992-07-09|1992-07-12|NONE|SHIP|y final deposits cajole according |
+2052|96|7|4|47|46816.23|0.08|0.01|A|F|1992-06-18|1992-05-16|1992-07-02|TAKE BACK RETURN|REG AIR|final requests. stealt|
+2053|101|4|1|20|20022.00|0.09|0.00|A|F|1995-04-25|1995-04-12|1995-05-13|NONE|TRUCK|ly ironic foxes haggle slyly speci|
+2053|33|4|2|34|31723.02|0.07|0.00|A|F|1995-03-15|1995-03-20|1995-04-09|TAKE BACK RETURN|TRUCK|ions. unusual dependencies|
+2053|65|2|3|46|44392.76|0.01|0.03|R|F|1995-04-01|1995-04-02|1995-04-18|NONE|RAIL|tions. furiously even requests hagg|
+2053|121|6|4|31|31654.72|0.06|0.08|R|F|1995-03-23|1995-03-13|1995-04-16|DELIVER IN PERSON|SHIP|ts. fluffily final mul|
+2054|113|4|1|11|11144.21|0.03|0.05|R|F|1992-08-13|1992-08-26|1992-08-22|NONE|AIR|ular accou|
+2054|120|7|2|31|31623.72|0.05|0.08|A|F|1992-08-18|1992-09-04|1992-08-24|NONE|FOB|se bold, regular accounts. unusual depos|
+2054|121|2|3|32|32675.84|0.06|0.00|A|F|1992-06-23|1992-07-08|1992-07-22|NONE|FOB| packages thrash. carefully final|
+2054|174|3|4|14|15038.38|0.10|0.05|R|F|1992-06-25|1992-09-05|1992-07-14|DELIVER IN PERSON|SHIP|uickly final|
+2054|6|1|5|40|36240.00|0.08|0.06|R|F|1992-06-23|1992-08-09|1992-07-04|TAKE BACK RETURN|RAIL|n pinto beans. ironic courts are iro|
+2054|134|10|6|17|17580.21|0.08|0.01|A|F|1992-06-09|1992-08-28|1992-06-16|NONE|AIR|ges nag acc|
+2054|11|1|7|4|3644.04|0.00|0.08|R|F|1992-08-12|1992-08-31|1992-08-15|DELIVER IN PERSON|AIR|lyly careful requests wake fl|
+2055|45|6|1|15|14175.60|0.04|0.06|A|F|1993-09-15|1993-10-06|1993-10-07|NONE|REG AIR|furiously bold |
+2055|9|10|2|15|13635.00|0.06|0.05|R|F|1993-10-30|1993-11-21|1993-11-22|COLLECT COD|RAIL|gular foxes. b|
+2055|135|1|3|12|12421.56|0.00|0.02|A|F|1993-10-26|1993-11-23|1993-11-22|COLLECT COD|TRUCK|al pains. acco|
+2055|134|10|4|16|16546.08|0.02|0.02|A|F|1993-11-16|1993-11-12|1993-11-28|NONE|TRUCK|arefully daringly regular accounts.|
+2080|7|4|1|5|4535.00|0.08|0.05|R|F|1993-08-26|1993-08-07|1993-09-02|DELIVER IN PERSON|TRUCK|refully unusual theo|
+2080|197|9|2|39|42790.41|0.07|0.04|A|F|1993-08-22|1993-09-09|1993-08-23|COLLECT COD|FOB|ic deposits haggle slyly carefully eve|
+2081|89|10|1|26|25716.08|0.03|0.08|N|O|1997-10-21|1997-10-03|1997-11-10|NONE|FOB|among the slyly express accounts. silen|
+2081|149|2|2|13|13638.82|0.07|0.05|N|O|1997-08-23|1997-08-22|1997-09-09|TAKE BACK RETURN|MAIL|fter the even deposi|
+2081|13|10|3|32|29216.32|0.09|0.07|N|O|1997-09-05|1997-09-26|1997-10-03|TAKE BACK RETURN|SHIP|e. final, regular dependencies sleep slyly!|
+2081|85|6|4|23|22656.84|0.03|0.08|N|O|1997-07-06|1997-09-11|1997-07-21|TAKE BACK RETURN|MAIL|ual requests wake blithely above the|
+2081|113|7|5|19|19249.09|0.02|0.06|N|O|1997-10-01|1997-08-12|1997-10-18|COLLECT COD|SHIP|s affix sometimes express requests. quickly|
+2081|142|9|6|31|32306.34|0.03|0.06|N|O|1997-09-19|1997-09-13|1997-09-27|NONE|AIR| silent, spe|
+2082|75|3|1|36|35102.52|0.00|0.00|R|F|1995-01-20|1995-03-18|1995-01-31|COLLECT COD|MAIL|haggle furiously silent pinto beans|
+2082|105|10|2|12|12061.20|0.08|0.05|A|F|1995-01-27|1995-02-11|1995-02-07|NONE|FOB| ironic instructions. carefull|
+2083|24|3|1|37|34188.74|0.07|0.00|R|F|1993-09-07|1993-09-30|1993-09-18|TAKE BACK RETURN|MAIL|ng the special foxes wake packages. f|
+2084|182|3|1|42|45451.56|0.03|0.05|A|F|1993-03-29|1993-05-05|1993-04-22|COLLECT COD|REG AIR|y fluffily even foxes. |
+2084|180|10|2|23|24844.14|0.09|0.08|A|F|1993-06-05|1993-05-26|1993-06-06|DELIVER IN PERSON|AIR|es against |
+2084|136|2|3|37|38336.81|0.07|0.05|A|F|1993-07-16|1993-04-20|1993-08-06|NONE|AIR|y careful courts.|
+2084|94|8|4|9|8946.81|0.02|0.02|A|F|1993-03-18|1993-06-08|1993-03-30|NONE|TRUCK|heaves boost slyly after the pla|
+2084|27|10|5|28|25956.56|0.07|0.02|R|F|1993-05-04|1993-05-14|1993-05-31|COLLECT COD|TRUCK|cajole quickly carefu|
+2084|115|9|6|15|15226.65|0.09|0.04|A|F|1993-06-23|1993-04-25|1993-07-23|COLLECT COD|SHIP|tithes. bravely pendi|
+2084|194|6|7|34|37202.46|0.09|0.02|R|F|1993-06-20|1993-05-28|1993-06-25|DELIVER IN PERSON|RAIL| carefully ironic requests. fluffil|
+2085|41|8|1|45|42346.80|0.00|0.07|R|F|1994-02-27|1994-01-11|1994-03-29|TAKE BACK RETURN|MAIL|. carefully e|
+2086|60|1|1|22|21121.32|0.03|0.07|R|F|1994-12-04|1994-12-16|1994-12-20|DELIVER IN PERSON|RAIL|idly busy acc|
+2086|141|10|2|32|33316.48|0.04|0.06|A|F|1994-11-15|1995-01-05|1994-12-09|TAKE BACK RETURN|TRUCK|e carefully along th|
+2086|105|6|3|44|44224.40|0.02|0.01|A|F|1994-12-04|1994-11-30|1994-12-21|DELIVER IN PERSON|FOB|latelets s|
+2086|84|5|4|27|26570.16|0.02|0.00|A|F|1994-11-04|1995-01-14|1994-11-25|COLLECT COD|REG AIR|theodolites haggle blithely blithe p|
+2086|156|1|5|33|34852.95|0.04|0.00|A|F|1995-02-06|1994-11-25|1995-02-15|NONE|SHIP| slyly regular foxes. un|
+2086|200|3|6|20|22004.00|0.01|0.03|R|F|1994-11-30|1994-12-28|1994-12-07|COLLECT COD|FOB|lithely ironic acc|
+2086|156|8|7|7|7393.05|0.04|0.05|R|F|1994-12-27|1994-12-10|1995-01-05|COLLECT COD|RAIL| beans haggle car|
+2087|127|8|1|1|1027.12|0.05|0.04|N|O|1998-03-27|1998-03-24|1998-04-18|DELIVER IN PERSON|REG AIR|the quickly idle acco|
+2087|168|3|2|46|49135.36|0.10|0.03|N|O|1998-02-24|1998-04-02|1998-03-04|DELIVER IN PERSON|AIR|ter the dolphins.|
+2087|62|3|3|1|962.06|0.02|0.05|N|O|1998-05-27|1998-04-11|1998-06-12|COLLECT COD|REG AIR|hely final acc|
+2087|59|1|4|6|5754.30|0.03|0.08|N|O|1998-04-23|1998-03-27|1998-05-18|DELIVER IN PERSON|REG AIR|dazzle after the slyly si|
+2112|71|2|1|18|17479.26|0.02|0.05|N|O|1997-05-02|1997-03-16|1997-05-25|TAKE BACK RETURN|TRUCK|lphins solve ideas. even, special reque|
+2113|123|8|1|40|40924.80|0.04|0.06|N|O|1998-01-16|1997-12-11|1998-02-06|TAKE BACK RETURN|TRUCK|bout the quickly ironic t|
+2113|112|2|2|24|24290.64|0.03|0.02|N|O|1998-02-19|1998-01-08|1998-03-16|COLLECT COD|MAIL|kly regular accounts hinder about the|
+2114|168|9|1|50|53408.00|0.05|0.05|A|F|1995-02-05|1995-03-18|1995-02-13|COLLECT COD|RAIL|pecial pinto bean|
+2114|186|7|2|26|28240.68|0.02|0.02|A|F|1995-04-30|1995-04-16|1995-05-28|NONE|SHIP|ar asymptotes sleep |
+2114|162|1|3|25|26554.00|0.07|0.01|A|F|1995-02-15|1995-03-13|1995-02-22|COLLECT COD|AIR|unts. regular, express accounts wake. b|
+2115|196|8|1|27|29597.13|0.06|0.03|N|O|1998-09-01|1998-07-29|1998-09-04|NONE|AIR|de of the carefully bold accounts |
+2115|184|5|2|43|46619.74|0.06|0.02|N|O|1998-07-14|1998-07-25|1998-07-24|COLLECT COD|FOB| carefully pending requests alongs|
+2115|51|3|3|3|2853.15|0.03|0.04|N|O|1998-07-23|1998-07-30|1998-08-14|DELIVER IN PERSON|FOB|quickly ironic dolphin|
+2115|49|10|4|47|44604.88|0.06|0.07|N|O|1998-08-29|1998-07-30|1998-09-05|TAKE BACK RETURN|REG AIR|regular accounts integrate brav|
+2115|199|3|5|13|14289.47|0.04|0.00|N|O|1998-08-07|1998-08-06|1998-08-13|DELIVER IN PERSON|REG AIR|into beans. even accounts abou|
+2116|131|2|1|2|2062.26|0.00|0.02|R|F|1994-10-16|1994-11-24|1994-11-09|DELIVER IN PERSON|TRUCK|r theodolites use blithely about the ir|
+2116|140|1|2|47|48886.58|0.10|0.06|R|F|1994-09-01|1994-11-18|1994-09-25|COLLECT COD|MAIL|iously ironic dependencies around the iro|
+2116|184|5|3|11|11925.98|0.03|0.05|R|F|1994-09-15|1994-10-21|1994-09-21|NONE|FOB| pinto beans. final, final sauternes play |
+2117|165|2|1|36|38345.76|0.10|0.01|N|O|1997-08-06|1997-07-15|1997-08-07|DELIVER IN PERSON|SHIP|ronic accounts wake|
+2117|61|6|2|19|18260.14|0.04|0.00|N|O|1997-07-30|1997-06-18|1997-08-13|DELIVER IN PERSON|REG AIR|s between the slyly regula|
+2117|58|3|3|43|41196.15|0.04|0.03|N|O|1997-06-27|1997-06-12|1997-07-22|DELIVER IN PERSON|SHIP| foxes sleep furiously |
+2117|91|4|4|24|23786.16|0.00|0.07|N|O|1997-06-15|1997-05-27|1997-06-18|COLLECT COD|SHIP|thely slyly pending platelets. ironic, |
+2117|147|8|5|3|3141.42|0.02|0.05|N|O|1997-05-05|1997-07-20|1997-05-26|TAKE BACK RETURN|TRUCK|tes cajole|
+2117|1|4|6|27|24327.00|0.09|0.08|N|O|1997-06-30|1997-06-27|1997-07-11|TAKE BACK RETURN|REG AIR| the carefully ironic ideas|
+2118|160|1|1|24|25443.84|0.10|0.03|N|O|1997-01-06|1996-12-14|1997-01-14|TAKE BACK RETURN|RAIL|about the slyly bold depende|
+2118|184|5|2|4|4336.72|0.08|0.01|N|O|1996-10-25|1996-11-10|1996-11-22|COLLECT COD|AIR|theodolites affix according |
+2118|145|4|3|11|11496.54|0.05|0.04|N|O|1996-12-23|1996-12-20|1997-01-01|COLLECT COD|RAIL|y ironic accounts sleep upon the packages. |
+2119|102|7|1|36|36075.60|0.04|0.00|N|O|1996-11-10|1996-10-25|1996-12-03|TAKE BACK RETURN|RAIL|ly bold foxes. ironic accoun|
+2144|92|6|1|33|32738.97|0.00|0.07|R|F|1994-04-04|1994-06-20|1994-04-23|NONE|AIR| ironic excuses haggle final dependencies. |
+2144|51|9|2|46|43748.30|0.03|0.08|R|F|1994-04-08|1994-04-29|1994-05-07|COLLECT COD|SHIP| foxes haggle blithel|
+2144|4|9|3|29|26216.00|0.00|0.07|R|F|1994-05-03|1994-05-16|1994-06-01|DELIVER IN PERSON|FOB|ns wake carefully carefully ironic|
+2144|158|9|4|10|10581.50|0.00|0.04|R|F|1994-06-16|1994-05-03|1994-07-05|COLLECT COD|AIR| furiously unusual ideas. carefull|
+2145|78|8|1|13|12714.91|0.04|0.05|A|F|1992-11-12|1992-12-13|1992-12-07|TAKE BACK RETURN|MAIL|alongside of the slyly final|
+2145|154|6|2|6|6324.90|0.05|0.01|A|F|1992-10-10|1992-11-29|1992-10-14|NONE|AIR|s. fluffily express accounts sleep. slyl|
+2146|57|5|1|42|40196.10|0.10|0.01|A|F|1992-09-21|1992-11-02|1992-09-23|NONE|AIR|ns according to the doggedly |
+2146|157|5|2|6|6342.90|0.07|0.05|A|F|1993-01-03|1992-10-24|1993-01-24|DELIVER IN PERSON|RAIL|ing to the requests. dependencies boost |
+2146|25|8|3|14|12950.28|0.03|0.01|R|F|1992-09-16|1992-10-16|1992-09-20|COLLECT COD|SHIP|ecial, express a|
+2146|26|9|4|31|28706.62|0.02|0.00|A|F|1993-01-04|1992-10-24|1993-01-15|DELIVER IN PERSON|TRUCK|lly even deposit|
+2146|169|4|5|28|29936.48|0.02|0.05|R|F|1993-01-03|1992-10-17|1993-01-08|COLLECT COD|MAIL|r accounts sleep furio|
+2146|71|9|6|32|31074.24|0.07|0.03|R|F|1993-01-10|1992-10-19|1993-02-05|COLLECT COD|TRUCK|y regular foxes wake among the final|
+2146|25|6|7|39|36075.78|0.07|0.06|R|F|1993-01-05|1992-11-06|1993-01-14|DELIVER IN PERSON|TRUCK|uickly regular excuses detect. regular c|
+2147|29|8|1|50|46451.00|0.04|0.06|R|F|1992-11-18|1992-11-30|1992-11-30|NONE|RAIL|al accounts. even, even foxes wake|
+2147|101|2|2|4|4004.40|0.01|0.04|A|F|1992-09-27|1992-11-15|1992-10-22|NONE|AIR|mong the blithely special|
+2147|44|7|3|34|32097.36|0.10|0.04|R|F|1992-11-29|1992-11-08|1992-12-22|TAKE BACK RETURN|REG AIR|egular deposits hang car|
+2147|11|8|4|11|10021.11|0.06|0.07|A|F|1992-09-27|1992-11-16|1992-10-16|NONE|AIR| the fluffily|
+2148|116|6|1|21|21338.31|0.09|0.01|R|F|1995-05-28|1995-05-26|1995-06-15|NONE|FOB|deposits ag|
+2149|19|9|1|12|11028.12|0.05|0.07|R|F|1993-06-01|1993-05-06|1993-06-11|TAKE BACK RETURN|TRUCK|riously bl|
+2149|99|10|2|10|9990.90|0.06|0.01|R|F|1993-06-09|1993-04-17|1993-06-16|DELIVER IN PERSON|TRUCK|eposits sleep above|
+2149|49|2|3|47|44604.88|0.00|0.04|R|F|1993-06-27|1993-05-12|1993-07-11|COLLECT COD|AIR|hely final depo|
+2149|129|8|4|18|18524.16|0.06|0.00|A|F|1993-04-05|1993-05-11|1993-04-23|DELIVER IN PERSON|REG AIR|uriously final pac|
+2149|60|5|5|22|21121.32|0.06|0.04|R|F|1993-05-24|1993-04-23|1993-06-20|TAKE BACK RETURN|SHIP|ptotes sleep along the blithely ir|
+2150|78|7|1|26|25429.82|0.00|0.03|A|F|1994-06-21|1994-08-05|1994-06-23|NONE|TRUCK|. always unusual packages|
+2150|18|8|2|29|26622.29|0.04|0.03|A|F|1994-09-02|1994-08-04|1994-10-02|TAKE BACK RETURN|RAIL|y ironic theodolites. foxes ca|
+2150|107|2|3|29|29205.90|0.04|0.08|R|F|1994-06-10|1994-07-31|1994-06-26|COLLECT COD|RAIL|arefully final att|
+2150|54|6|4|39|37207.95|0.05|0.02|R|F|1994-07-31|1994-08-17|1994-08-11|TAKE BACK RETURN|TRUCK|ess accounts nag. unusual asymptotes haggl|
+2150|183|4|5|35|37911.30|0.01|0.01|A|F|1994-09-27|1994-08-17|1994-10-13|COLLECT COD|RAIL|refully pending dependen|
+2150|7|10|6|12|10884.00|0.09|0.03|A|F|1994-08-27|1994-08-22|1994-09-18|COLLECT COD|AIR|press platelets haggle until the slyly fi|
+2151|167|2|1|23|24544.68|0.06|0.02|N|O|1996-11-20|1996-12-17|1996-11-30|DELIVER IN PERSON|AIR| silent dependencies about the slyl|
+2151|15|9|2|29|26535.29|0.00|0.02|N|O|1997-03-04|1996-12-27|1997-03-21|TAKE BACK RETURN|SHIP| bold packages acro|
+2151|165|2|3|49|52192.84|0.07|0.01|N|O|1997-01-20|1997-02-09|1997-02-18|NONE|FOB| packages. f|
+2151|18|5|4|28|25704.28|0.10|0.08|N|O|1996-12-11|1996-12-26|1996-12-12|DELIVER IN PERSON|AIR|y special packages. carefully ironic instru|
+2176|191|4|1|38|41465.22|0.02|0.08|R|F|1992-11-29|1993-01-14|1992-12-22|DELIVER IN PERSON|REG AIR|lithely ironic pinto beans. furious|
+2176|95|8|2|14|13931.26|0.00|0.06|A|F|1992-11-17|1993-01-07|1992-12-03|DELIVER IN PERSON|SHIP|ely ironic platelets |
+2176|160|1|3|25|26504.00|0.02|0.02|R|F|1993-02-23|1993-01-05|1993-03-07|COLLECT COD|RAIL| ruthless deposits according to the ent|
+2176|143|6|4|2|2086.28|0.05|0.06|A|F|1993-02-26|1993-01-08|1993-03-23|DELIVER IN PERSON|AIR|s pinto beans|
+2177|129|10|1|45|46310.40|0.02|0.01|N|O|1997-02-11|1997-02-27|1997-02-17|NONE|SHIP|. theodolites haggle carefu|
+2177|139|5|2|27|28056.51|0.04|0.08|N|O|1997-01-29|1997-03-20|1997-02-04|DELIVER IN PERSON|SHIP|even, regula|
+2177|81|2|3|23|22564.84|0.07|0.05|N|O|1997-01-28|1997-03-02|1997-02-13|DELIVER IN PERSON|AIR|he silent foxes. iro|
+2177|55|3|4|34|32471.70|0.05|0.07|N|O|1997-02-03|1997-04-10|1997-02-21|COLLECT COD|REG AIR|tes are doggedly quickly|
+2177|57|9|5|46|44024.30|0.09|0.05|N|O|1997-05-10|1997-02-23|1997-05-28|COLLECT COD|RAIL|ending asymptotes.|
+2177|122|7|6|11|11243.32|0.02|0.04|N|O|1997-03-20|1997-03-07|1997-04-09|DELIVER IN PERSON|MAIL|gainst the ca|
+2178|157|2|1|15|15857.25|0.10|0.01|N|O|1997-03-27|1997-03-10|1997-04-18|NONE|REG AIR|l accounts. quickly expr|
+2178|16|10|2|27|24732.27|0.01|0.02|N|O|1997-02-26|1997-02-19|1997-03-25|NONE|MAIL| across the ironic reques|
+2178|5|2|3|40|36200.00|0.00|0.03|N|O|1997-03-17|1997-02-09|1997-04-15|COLLECT COD|RAIL|foxes are slowly regularly specia|
+2178|78|6|4|3|2934.21|0.07|0.07|N|O|1997-04-07|1997-01-23|1997-04-18|COLLECT COD|MAIL| permanentl|
+2179|130|9|1|22|22662.86|0.05|0.08|N|O|1996-11-16|1996-11-03|1996-11-25|DELIVER IN PERSON|FOB|lphins cajole acr|
+2179|139|5|2|20|20782.60|0.03|0.01|N|O|1996-09-30|1996-11-10|1996-10-30|NONE|REG AIR|ncies. fin|
+2179|104|9|3|5|5020.50|0.03|0.02|N|O|1996-11-09|1996-10-08|1996-11-11|DELIVER IN PERSON|REG AIR|ts haggle blithely. ironic, careful theodol|
+2179|6|3|4|24|21744.00|0.04|0.04|N|O|1996-10-26|1996-11-05|1996-11-16|COLLECT COD|RAIL| cajole carefully. |
+2179|108|5|5|7|7056.70|0.00|0.02|N|O|1996-10-24|1996-11-14|1996-11-21|TAKE BACK RETURN|RAIL|gular dependencies. ironic packages haggle|
+2180|16|3|1|31|28396.31|0.06|0.04|N|O|1996-10-20|1996-11-21|1996-11-06|COLLECT COD|REG AIR|n requests are furiously at the quickly|
+2180|193|7|2|39|42634.41|0.01|0.00|N|O|1997-01-03|1996-10-29|1997-01-25|NONE|RAIL|ep furiously furiously final request|
+2180|197|9|3|24|26332.56|0.03|0.00|N|O|1997-01-03|1996-10-24|1997-01-19|NONE|SHIP|uriously f|
+2180|111|5|4|47|47522.17|0.07|0.02|N|O|1996-09-23|1996-12-08|1996-10-12|NONE|FOB|pending, regular ideas. iron|
+2180|143|2|5|23|23992.22|0.02|0.06|N|O|1996-11-08|1996-10-25|1996-11-28|NONE|TRUCK|ggle alongside of the fluffily speci|
+2180|55|6|6|48|45842.40|0.09|0.03|N|O|1996-12-30|1996-11-22|1997-01-16|DELIVER IN PERSON|RAIL|nic instructions haggle careful|
+2181|178|9|1|4|4312.68|0.05|0.04|N|O|1995-09-25|1995-11-12|1995-09-28|COLLECT COD|FOB|tes. slyly silent packages use along th|
+2181|88|9|2|46|45451.68|0.00|0.02|N|O|1995-11-28|1995-10-17|1995-12-26|COLLECT COD|AIR|osits. final packages sleep|
+2181|91|2|3|15|14866.35|0.08|0.05|N|O|1995-10-05|1995-10-27|1995-11-03|DELIVER IN PERSON|FOB|e above the fluffily regul|
+2181|55|10|4|28|26741.40|0.04|0.05|N|O|1995-12-21|1995-10-23|1996-01-04|TAKE BACK RETURN|AIR|s excuses sleep car|
+2181|96|7|5|9|8964.81|0.06|0.07|N|O|1996-01-05|1995-12-05|1996-01-08|COLLECT COD|TRUCK|ward the quietly even requests. ir|
+2182|132|8|1|27|27867.51|0.02|0.07|R|F|1994-05-10|1994-07-04|1994-06-04|DELIVER IN PERSON|SHIP|en platele|
+2182|190|1|2|3|3270.57|0.05|0.03|R|F|1994-04-20|1994-07-04|1994-04-24|TAKE BACK RETURN|SHIP|y bold theodolites wi|
+2182|94|6|3|34|33799.06|0.02|0.03|R|F|1994-05-28|1994-06-02|1994-06-10|COLLECT COD|MAIL| slow tithes. ironi|
+2182|7|4|4|12|10884.00|0.04|0.07|A|F|1994-05-08|1994-06-02|1994-05-09|COLLECT COD|REG AIR|ments are fu|
+2182|179|9|5|37|39929.29|0.06|0.02|A|F|1994-04-08|1994-06-29|1994-04-18|TAKE BACK RETURN|TRUCK|ges. blithely ironic|
+2183|71|1|1|29|28161.03|0.05|0.01|N|O|1996-07-21|1996-08-24|1996-08-15|TAKE BACK RETURN|RAIL|ly unusual deposits sleep carefully|
+2183|52|3|2|25|23801.25|0.06|0.02|N|O|1996-07-06|1996-08-21|1996-08-05|NONE|RAIL|he quickly f|
+2208|58|3|1|48|45986.40|0.08|0.07|A|F|1995-05-13|1995-06-30|1995-05-20|COLLECT COD|MAIL|sits. idly permanent request|
+2208|97|1|2|11|10967.99|0.08|0.01|A|F|1995-05-06|1995-07-19|1995-05-22|COLLECT COD|TRUCK|ding waters lose. furiously regu|
+2208|74|4|3|41|39936.87|0.08|0.02|N|O|1995-08-18|1995-06-19|1995-09-05|COLLECT COD|RAIL|nd the furious, express dependencies.|
+2208|43|2|4|50|47152.00|0.07|0.07|N|F|1995-06-11|1995-05-31|1995-06-29|TAKE BACK RETURN|FOB|al foxes will hav|
+2208|30|5|5|43|39991.29|0.03|0.06|A|F|1995-05-10|1995-06-02|1995-06-09|TAKE BACK RETURN|MAIL|es. accounts cajole. fi|
+2208|167|2|6|18|19208.88|0.02|0.08|R|F|1995-06-06|1995-06-10|1995-06-11|TAKE BACK RETURN|TRUCK|packages are quickly bold de|
+2208|7|2|7|45|40815.00|0.00|0.08|A|F|1995-05-05|1995-06-10|1995-05-11|NONE|SHIP|e fluffily regular theodolites caj|
+2209|23|2|1|40|36920.80|0.05|0.01|R|F|1992-11-01|1992-09-25|1992-11-08|DELIVER IN PERSON|SHIP|ully special sheaves serve|
+2209|103|4|2|10|10031.00|0.00|0.02|R|F|1992-09-02|1992-09-24|1992-09-21|DELIVER IN PERSON|AIR|players. carefully reg|
+2209|64|1|3|11|10604.66|0.01|0.01|A|F|1992-07-12|1992-08-24|1992-08-10|DELIVER IN PERSON|REG AIR|express, regular pinto be|
+2209|181|2|4|39|42166.02|0.08|0.07|R|F|1992-11-04|1992-09-02|1992-11-11|TAKE BACK RETURN|MAIL|ly around the final packages. deposits ca|
+2209|124|7|5|24|24578.88|0.08|0.06|R|F|1992-08-09|1992-08-18|1992-08-25|COLLECT COD|AIR| along the bol|
+2209|178|7|6|7|7547.19|0.09|0.07|A|F|1992-08-18|1992-09-09|1992-09-12|DELIVER IN PERSON|AIR| quickly regular pack|
+2210|78|7|1|36|35210.52|0.10|0.00|A|F|1992-03-04|1992-03-24|1992-03-21|DELIVER IN PERSON|AIR| requests wake enticingly final|
+2211|48|1|1|25|23701.00|0.04|0.01|A|F|1994-10-09|1994-08-04|1994-11-03|TAKE BACK RETURN|RAIL|deas. carefully special theodolites along|
+2211|140|6|2|40|41605.60|0.09|0.06|A|F|1994-09-30|1994-09-10|1994-10-26|NONE|MAIL|posits among the express dolphins|
+2211|160|2|3|25|26504.00|0.00|0.07|A|F|1994-08-13|1994-08-17|1994-08-16|NONE|AIR|ly regular, express|
+2211|85|6|4|23|22656.84|0.03|0.02|R|F|1994-10-05|1994-09-13|1994-10-17|DELIVER IN PERSON|AIR|ependencies |
+2211|135|1|5|3|3105.39|0.02|0.04|A|F|1994-08-28|1994-09-10|1994-09-06|TAKE BACK RETURN|SHIP|pendencies after the regular f|
+2211|187|8|6|18|19569.24|0.05|0.08|A|F|1994-08-31|1994-09-07|1994-09-22|NONE|TRUCK|c grouches. slyly express pinto |
+2211|79|9|7|3|2937.21|0.06|0.05|R|F|1994-09-21|1994-08-10|1994-10-19|TAKE BACK RETURN|RAIL|y slyly final|
+2212|71|10|1|18|17479.26|0.07|0.06|R|F|1994-06-22|1994-06-18|1994-06-25|TAKE BACK RETURN|FOB| cajole. final, pending ideas should are bl|
+2213|118|8|1|20|20362.20|0.01|0.00|A|F|1993-01-21|1993-04-14|1993-01-29|COLLECT COD|REG AIR|iously express accounts; |
+2213|60|1|2|4|3840.24|0.09|0.05|R|F|1993-04-15|1993-04-15|1993-05-05|COLLECT COD|SHIP| affix carefully furiously |
+2213|70|5|3|1|970.07|0.05|0.05|A|F|1993-04-25|1993-04-06|1993-04-28|TAKE BACK RETURN|AIR|s along the ironic reques|
+2213|174|3|4|39|41892.63|0.09|0.05|A|F|1993-05-12|1993-04-07|1993-05-23|TAKE BACK RETURN|SHIP|the blithely |
+2213|38|9|5|43|40335.29|0.04|0.03|A|F|1993-04-18|1993-03-11|1993-05-11|TAKE BACK RETURN|RAIL|r packages are along the carefully bol|
+2213|48|5|6|41|38869.64|0.01|0.00|R|F|1993-01-31|1993-03-31|1993-02-28|COLLECT COD|FOB| carefully pend|
+2213|64|9|7|3|2892.18|0.02|0.04|A|F|1993-03-09|1993-03-17|1993-04-07|TAKE BACK RETURN|AIR|o wake. ironic platel|
+2214|76|5|1|27|26353.89|0.04|0.04|N|O|1998-05-31|1998-06-07|1998-06-19|DELIVER IN PERSON|REG AIR|x fluffily along the even packages-- |
+2214|194|5|2|50|54709.50|0.00|0.02|N|O|1998-07-06|1998-06-16|1998-07-16|TAKE BACK RETURN|MAIL|accounts. blith|
+2214|113|7|3|42|42550.62|0.04|0.08|N|O|1998-05-26|1998-07-13|1998-06-22|COLLECT COD|FOB|ons. deposi|
+2214|196|9|4|22|24116.18|0.01|0.01|N|O|1998-05-30|1998-07-02|1998-06-09|DELIVER IN PERSON|RAIL|t the blithely|
+2215|73|1|1|33|32111.31|0.00|0.00|N|O|1996-07-19|1996-08-10|1996-07-30|COLLECT COD|RAIL|dolites cajole b|
+2215|33|9|2|30|27990.90|0.01|0.00|N|O|1996-08-15|1996-09-10|1996-08-25|NONE|FOB|ckages caj|
+2215|57|5|3|30|28711.50|0.07|0.03|N|O|1996-09-09|1996-07-20|1996-09-28|COLLECT COD|TRUCK|against the carefu|
+2215|146|3|4|20|20922.80|0.02|0.02|N|O|1996-09-09|1996-08-10|1996-09-19|NONE|MAIL| unusual deposits haggle carefully. ide|
+2240|164|3|1|6|6384.96|0.01|0.00|A|F|1992-06-23|1992-05-17|1992-07-20|COLLECT COD|AIR|ymptotes boost. furiously bold p|
+2240|28|1|2|37|34336.74|0.03|0.07|R|F|1992-03-16|1992-05-31|1992-04-05|COLLECT COD|FOB| quickly after the packages? blithely si|
+2240|53|5|3|39|37168.95|0.08|0.06|A|F|1992-05-22|1992-05-10|1992-06-08|NONE|FOB|y orbits. final depos|
+2240|86|7|4|10|9860.80|0.09|0.00|A|F|1992-05-25|1992-04-14|1992-06-23|DELIVER IN PERSON|REG AIR|are across the ironic packages.|
+2240|161|10|5|29|30773.64|0.02|0.06|A|F|1992-03-29|1992-05-08|1992-04-09|COLLECT COD|MAIL|lyly even ideas w|
+2240|81|2|6|32|31394.56|0.06|0.06|R|F|1992-04-11|1992-04-18|1992-04-22|NONE|MAIL|ss thinly deposits. blithely bold package|
+2240|78|7|7|24|23473.68|0.04|0.05|R|F|1992-05-13|1992-04-09|1992-05-14|DELIVER IN PERSON|FOB|ng the silent accounts. slyly ironic t|
+2241|5|6|1|25|22625.00|0.00|0.08|R|F|1993-08-11|1993-07-23|1993-09-01|DELIVER IN PERSON|MAIL| final deposits use fluffily. even f|
+2241|195|8|2|38|41617.22|0.04|0.06|A|F|1993-08-04|1993-07-31|1993-08-06|TAKE BACK RETURN|TRUCK| silent, unusual d|
+2241|97|10|3|48|47860.32|0.08|0.04|A|F|1993-05-14|1993-07-30|1993-05-26|TAKE BACK RETURN|RAIL|ss accounts engage furiously. slyly even re|
+2241|167|4|4|19|20276.04|0.10|0.00|A|F|1993-06-01|1993-08-05|1993-06-07|TAKE BACK RETURN|TRUCK| are furiously quickl|
+2241|82|3|5|2|1964.16|0.04|0.03|A|F|1993-08-16|1993-08-02|1993-08-24|NONE|REG AIR|, express deposits. pear|
+2241|116|3|6|22|22354.42|0.02|0.08|R|F|1993-08-13|1993-06-15|1993-08-16|DELIVER IN PERSON|TRUCK|, ironic depen|
+2241|142|3|7|9|9379.26|0.09|0.03|A|F|1993-05-14|1993-07-12|1993-05-29|NONE|AIR|lyly final |
+2242|123|4|1|15|15346.80|0.09|0.08|N|O|1997-08-04|1997-09-21|1997-08-11|COLLECT COD|FOB|its. carefully express packages cajole. bli|
+2243|127|8|1|10|10271.20|0.04|0.06|N|O|1995-07-26|1995-07-18|1995-08-03|NONE|RAIL|express, daring foxes affix fur|
+2244|51|6|1|3|2853.15|0.02|0.02|A|F|1993-04-30|1993-03-15|1993-05-19|TAKE BACK RETURN|FOB| beans for the regular platel|
+2244|193|6|2|16|17491.04|0.01|0.06|R|F|1993-02-12|1993-03-09|1993-02-28|COLLECT COD|FOB|rate around the reques|
+2245|76|7|1|44|42947.08|0.03|0.03|A|F|1993-06-12|1993-06-10|1993-06-16|NONE|TRUCK|refully even sheaves|
+2245|74|3|2|28|27273.96|0.05|0.03|R|F|1993-08-19|1993-07-27|1993-09-04|COLLECT COD|TRUCK|e requests sleep furiou|
+2245|86|7|3|33|32540.64|0.03|0.01|R|F|1993-06-26|1993-06-11|1993-07-17|TAKE BACK RETURN|AIR|ing to the carefully ruthless accounts|
+2245|189|10|4|14|15248.52|0.02|0.04|R|F|1993-05-06|1993-07-21|1993-05-19|DELIVER IN PERSON|RAIL|nts. always unusual dep|
+2245|80|8|5|33|32342.64|0.03|0.07|R|F|1993-06-16|1993-06-05|1993-07-07|NONE|MAIL| across the express reques|
+2246|53|4|1|22|20967.10|0.02|0.01|N|O|1996-07-25|1996-08-03|1996-08-24|DELIVER IN PERSON|SHIP|ructions wake carefully fina|
+2246|104|5|2|43|43176.30|0.07|0.06|N|O|1996-08-25|1996-08-23|1996-09-19|DELIVER IN PERSON|AIR|ainst the ironic theodolites haggle fi|
+2246|18|8|3|11|10098.11|0.10|0.00|N|O|1996-06-21|1996-07-24|1996-07-18|TAKE BACK RETURN|TRUCK|quests alongside o|
+2246|163|8|4|13|13821.08|0.08|0.05|N|O|1996-09-15|1996-07-21|1996-10-08|DELIVER IN PERSON|AIR|equests. fluffily special epitaphs use|
+2247|172|2|1|12|12866.04|0.02|0.07|A|F|1992-09-06|1992-09-18|1992-09-26|NONE|MAIL|final accounts. requests across the furiou|
+2272|90|1|1|18|17821.62|0.04|0.00|R|F|1993-08-01|1993-07-06|1993-08-25|NONE|MAIL|ons along the blithely e|
+2272|34|10|2|40|37361.20|0.07|0.00|A|F|1993-04-25|1993-07-12|1993-05-15|DELIVER IN PERSON|FOB|lithely ir|
+2272|56|4|3|36|34417.80|0.03|0.02|A|F|1993-05-25|1993-05-23|1993-06-09|TAKE BACK RETURN|RAIL|about the ironic packages; quickly iron|
+2272|138|4|4|30|31143.90|0.09|0.07|A|F|1993-07-27|1993-05-15|1993-08-13|NONE|RAIL|quests at the foxes haggle evenly pack|
+2272|76|4|5|12|11712.84|0.03|0.03|A|F|1993-04-19|1993-05-14|1993-04-23|NONE|RAIL| accounts cajole. quickly b|
+2273|184|5|1|34|36862.12|0.02|0.03|N|O|1997-01-08|1997-02-02|1997-01-23|COLLECT COD|MAIL| furiously carefully bold de|
+2273|85|6|2|35|34477.80|0.00|0.05|N|O|1997-01-02|1997-01-19|1997-01-14|NONE|REG AIR|arefully f|
+2273|95|8|3|8|7960.72|0.00|0.04|N|O|1996-12-15|1997-02-27|1997-01-10|NONE|FOB|dependencies. slyly ir|
+2273|161|6|4|20|21223.20|0.06|0.04|N|O|1997-03-05|1997-02-25|1997-04-01|NONE|RAIL|cuses. quickly enticing requests wake |
+2273|162|7|5|18|19118.88|0.07|0.00|N|O|1996-12-16|1997-01-21|1997-01-03|COLLECT COD|TRUCK| beans. doggedly final packages wake|
+2273|155|7|6|16|16882.40|0.10|0.03|N|O|1997-01-10|1997-02-03|1997-02-01|TAKE BACK RETURN|RAIL|furiously above the ironic requests. |
+2273|20|1|7|7|6440.14|0.05|0.05|N|O|1997-02-19|1997-01-22|1997-02-21|TAKE BACK RETURN|TRUCK|ts. furiou|
+2274|12|6|1|18|16416.18|0.04|0.03|R|F|1993-09-06|1993-12-03|1993-09-22|COLLECT COD|SHIP|usly final re|
+2274|111|8|2|23|23255.53|0.04|0.03|R|F|1993-10-28|1993-11-03|1993-11-05|NONE|MAIL|kly special warhorse|
+2274|129|10|3|18|18524.16|0.03|0.06|R|F|1993-09-28|1993-11-22|1993-10-12|DELIVER IN PERSON|SHIP| express packages. even accounts hagg|
+2275|34|5|1|30|28020.90|0.08|0.05|R|F|1993-01-10|1992-11-21|1993-01-22|NONE|REG AIR|re slyly slyly special idea|
+2275|91|4|2|11|10901.99|0.08|0.03|A|F|1993-01-16|1992-12-10|1993-01-25|COLLECT COD|REG AIR|ost across the never express instruction|
+2276|119|9|1|5|5095.55|0.07|0.08|N|O|1996-05-09|1996-06-18|1996-05-13|DELIVER IN PERSON|FOB|ias instea|
+2276|135|1|2|13|13456.69|0.08|0.04|N|O|1996-07-24|1996-06-18|1996-08-16|COLLECT COD|RAIL|arefully ironic foxes cajole q|
+2276|171|2|3|27|28921.59|0.07|0.08|N|O|1996-07-30|1996-06-10|1996-07-31|DELIVER IN PERSON|RAIL|the carefully unusual accoun|
+2276|109|6|4|38|38345.80|0.06|0.03|N|O|1996-07-07|1996-06-28|1996-07-17|COLLECT COD|RAIL|ans. pinto beans boost c|
+2276|153|5|5|50|52657.50|0.03|0.05|N|O|1996-07-13|1996-06-25|1996-07-22|DELIVER IN PERSON|REG AIR| accounts dete|
+2276|6|9|6|4|3624.00|0.10|0.03|N|O|1996-07-05|1996-06-30|1996-08-04|COLLECT COD|FOB|s. deposits |
+2277|137|8|1|38|39410.94|0.03|0.07|R|F|1995-04-23|1995-03-25|1995-05-20|TAKE BACK RETURN|TRUCK|fully bold|
+2277|8|1|2|2|1816.00|0.10|0.08|A|F|1995-02-01|1995-02-04|1995-03-02|TAKE BACK RETURN|AIR|endencies sleep idly pending p|
+2277|198|10|3|4|4392.76|0.05|0.06|R|F|1995-04-27|1995-03-16|1995-04-29|TAKE BACK RETURN|SHIP|. quickly unusual deposi|
+2277|159|4|4|31|32833.65|0.02|0.00|R|F|1995-03-07|1995-03-19|1995-03-26|TAKE BACK RETURN|MAIL|ic instructions detect ru|
+2278|45|2|1|36|34021.44|0.04|0.05|N|O|1998-06-04|1998-06-06|1998-06-30|NONE|TRUCK|y ironic pinto beans br|
+2278|45|2|2|50|47252.00|0.02|0.00|N|O|1998-08-09|1998-07-08|1998-09-05|DELIVER IN PERSON|RAIL|into beans. blit|
+2278|97|9|3|22|21935.98|0.03|0.00|N|O|1998-05-15|1998-07-14|1998-06-04|TAKE BACK RETURN|REG AIR|ep regular accounts. blithely even|
+2279|14|5|1|12|10968.12|0.07|0.08|A|F|1993-05-10|1993-03-25|1993-06-02|COLLECT COD|REG AIR|lets across the excuses nag quickl|
+2279|41|2|2|38|35759.52|0.08|0.07|R|F|1993-06-09|1993-04-06|1993-06-26|COLLECT COD|TRUCK|s above the furiously express dep|
+2279|4|7|3|3|2712.00|0.09|0.04|A|F|1993-05-31|1993-05-07|1993-06-05|COLLECT COD|REG AIR|ing foxes above the even accounts use slyly|
+2279|52|4|4|42|39986.10|0.02|0.00|R|F|1993-02-28|1993-04-25|1993-03-02|TAKE BACK RETURN|REG AIR| above the furiously ironic deposits. |
+2279|169|8|5|9|9622.44|0.05|0.04|R|F|1993-05-21|1993-03-29|1993-06-17|DELIVER IN PERSON|MAIL|ns cajole after the final platelets. s|
+2279|147|10|6|12|12565.68|0.02|0.00|R|F|1993-05-04|1993-04-26|1993-05-28|DELIVER IN PERSON|FOB|ccounts. slyl|
+2279|119|9|7|32|32611.52|0.05|0.05|A|F|1993-04-20|1993-05-22|1993-05-18|DELIVER IN PERSON|RAIL|re quickly. furiously ironic ide|
+2304|200|2|1|42|46208.40|0.00|0.01|A|F|1994-01-20|1994-03-04|1994-02-05|COLLECT COD|RAIL|quests are blithely alongside of|
+2304|19|9|2|48|44112.48|0.00|0.00|R|F|1994-02-12|1994-02-16|1994-03-10|COLLECT COD|REG AIR| deposits cajole blithely e|
+2304|48|9|3|3|2844.12|0.00|0.05|R|F|1994-03-19|1994-03-04|1994-03-20|DELIVER IN PERSON|AIR|l excuses after the ev|
+2305|174|4|1|3|3222.51|0.00|0.01|A|F|1993-03-24|1993-04-05|1993-03-29|NONE|AIR|kages haggle quickly across the blithely |
+2305|60|8|2|39|37442.34|0.07|0.00|R|F|1993-04-16|1993-04-17|1993-04-22|COLLECT COD|MAIL|ms after the foxes |
+2305|102|3|3|32|32067.20|0.03|0.06|A|F|1993-04-02|1993-03-18|1993-04-03|NONE|AIR| haggle caref|
+2305|112|3|4|17|17205.87|0.00|0.05|A|F|1993-02-21|1993-03-30|1993-03-19|TAKE BACK RETURN|MAIL| carefully alongside of |
+2305|155|7|5|26|27433.90|0.06|0.07|A|F|1993-05-14|1993-02-28|1993-06-04|NONE|SHIP|arefully final theodo|
+2305|51|3|6|7|6657.35|0.06|0.00|R|F|1993-05-15|1993-04-25|1993-06-09|DELIVER IN PERSON|RAIL|gular deposits boost about the foxe|
+2306|196|9|1|50|54809.50|0.09|0.01|N|O|1995-07-27|1995-09-26|1995-08-06|DELIVER IN PERSON|FOB|y quickly |
+2306|149|2|2|39|40916.46|0.04|0.00|N|O|1995-09-07|1995-09-13|1995-10-03|COLLECT COD|SHIP|f the slyly unusual accounts. furiousl|
+2306|178|6|3|35|37735.95|0.01|0.07|N|O|1995-08-18|1995-08-30|1995-08-20|TAKE BACK RETURN|RAIL|raids along the furiously unusual asympto|
+2306|119|3|4|21|21401.31|0.06|0.01|N|O|1995-10-07|1995-09-18|1995-10-17|COLLECT COD|MAIL| ironic pinto |
+2306|142|9|5|42|43769.88|0.04|0.07|N|O|1995-09-05|1995-08-25|1995-09-28|COLLECT COD|MAIL|furiously final acco|
+2306|124|5|6|29|29699.48|0.00|0.03|N|O|1995-11-01|1995-09-01|1995-11-22|DELIVER IN PERSON|REG AIR|uld have to mold. s|
+2306|176|4|7|19|20447.23|0.07|0.01|N|O|1995-11-17|1995-09-06|1995-11-30|DELIVER IN PERSON|AIR|tainments nag furiously carefull|
+2307|142|9|1|24|25011.36|0.10|0.05|R|F|1993-10-07|1993-08-05|1993-10-20|COLLECT COD|AIR|stealthily special packages nag a|
+2307|140|6|2|2|2080.28|0.01|0.00|A|F|1993-09-21|1993-08-22|1993-10-03|COLLECT COD|SHIP|ously. furiously furious requ|
+2307|34|10|3|7|6538.21|0.07|0.04|R|F|1993-08-03|1993-09-04|1993-08-28|DELIVER IN PERSON|AIR|ven instructions wake fluffily |
+2307|165|6|4|19|20238.04|0.08|0.06|R|F|1993-10-23|1993-09-09|1993-11-09|TAKE BACK RETURN|TRUCK|olites haggle furiously around the |
+2307|143|4|5|7|7301.98|0.01|0.06|R|F|1993-09-01|1993-08-08|1993-09-29|NONE|AIR| packages cajo|
+2308|118|9|1|24|24434.64|0.06|0.04|R|F|1993-02-23|1992-12-24|1993-03-10|NONE|MAIL|ts sleep. busy excuses along the s|
+2308|56|1|2|36|34417.80|0.05|0.06|A|F|1992-11-11|1992-11-27|1992-11-23|NONE|MAIL|ong the pending hockey players. blithe|
+2309|170|7|1|14|14982.38|0.10|0.03|N|O|1996-01-01|1995-10-22|1996-01-23|NONE|AIR|asymptotes. furiously pending acco|
+2309|169|8|2|1|1069.16|0.01|0.05|N|O|1995-12-08|1995-11-03|1995-12-31|COLLECT COD|RAIL|eposits alongside of the final re|
+2309|15|2|3|5|4575.05|0.01|0.00|N|O|1995-12-10|1995-10-29|1996-01-06|TAKE BACK RETURN|SHIP|s. requests wake blithely specia|
+2309|139|10|4|46|47799.98|0.08|0.04|N|O|1995-10-02|1995-10-30|1995-10-30|NONE|REG AIR|sly according to the carefully |
+2309|137|3|5|9|9334.17|0.00|0.07|N|O|1995-12-21|1995-10-10|1996-01-20|COLLECT COD|AIR|ding, unusual instructions. dep|
+2309|195|8|6|21|22998.99|0.09|0.00|N|O|1995-11-05|1995-11-07|1995-11-22|NONE|AIR|unts around the dolphins ar|
+2309|138|4|7|48|49830.24|0.03|0.05|N|O|1995-10-21|1995-11-21|1995-11-09|NONE|MAIL|ccounts. id|
+2310|58|6|1|36|34489.80|0.03|0.03|N|O|1996-10-09|1996-10-28|1996-10-29|TAKE BACK RETURN|RAIL|iously against the slyly special accounts|
+2310|171|2|2|6|6427.02|0.07|0.01|N|O|1996-11-08|1996-12-09|1996-12-07|COLLECT COD|REG AIR|e slyly about the quickly ironic theodo|
+2310|42|1|3|48|45217.92|0.08|0.02|N|O|1996-10-04|1996-11-20|1996-10-25|TAKE BACK RETURN|FOB|ep slyly alongside of the |
+2311|141|8|1|18|18740.52|0.01|0.01|N|F|1995-06-11|1995-06-18|1995-07-02|NONE|FOB| fluffily even patterns haggle blithely. re|
+2311|122|1|2|49|50083.88|0.09|0.02|R|F|1995-05-14|1995-07-11|1995-05-20|COLLECT COD|FOB|ideas sleep|
+2311|54|5|3|15|14310.75|0.08|0.04|N|O|1995-06-23|1995-06-06|1995-07-09|COLLECT COD|AIR|ve the blithely pending accounts. furio|
+2311|90|1|4|42|41583.78|0.01|0.06|R|F|1995-06-03|1995-06-27|1995-06-11|DELIVER IN PERSON|MAIL|gle furiously. bold |
+2311|47|10|5|1|947.04|0.05|0.02|A|F|1995-06-07|1995-06-20|1995-06-10|NONE|AIR|ptotes. furiously regular theodolite|
+2311|12|9|6|32|29184.32|0.01|0.03|N|O|1995-07-19|1995-06-26|1995-07-26|NONE|RAIL|sts along the slyly|
+2336|193|5|1|20|21863.80|0.01|0.03|N|O|1996-03-12|1996-02-25|1996-03-18|NONE|REG AIR|across the fi|
+2337|45|2|1|49|46306.96|0.06|0.05|N|O|1997-08-08|1997-08-15|1997-08-31|TAKE BACK RETURN|FOB| along the packages. furiously p|
+2338|52|7|1|30|28561.50|0.07|0.06|N|O|1997-12-10|1997-10-15|1997-12-11|TAKE BACK RETURN|REG AIR|ould have to nag quickly|
+2339|192|3|1|22|24028.18|0.03|0.03|A|F|1994-01-06|1994-03-06|1994-01-10|NONE|FOB| furiously above |
+2339|30|5|2|28|26040.84|0.00|0.00|R|F|1994-01-25|1994-01-22|1994-01-28|DELIVER IN PERSON|RAIL|e bold, even packag|
+2339|117|4|3|13|13222.43|0.06|0.08|R|F|1994-03-10|1994-02-18|1994-03-20|TAKE BACK RETURN|REG AIR|ges. blithely special depend|
+2340|138|4|1|9|9343.17|0.08|0.02|N|O|1996-05-01|1996-02-24|1996-05-16|COLLECT COD|RAIL|. carefully ironic|
+2340|193|5|2|21|22956.99|0.06|0.02|N|O|1996-01-17|1996-03-04|1996-01-29|DELIVER IN PERSON|SHIP| asymptotes. unusual theo|
+2341|47|10|1|12|11364.48|0.08|0.03|R|F|1993-06-06|1993-07-08|1993-06-17|DELIVER IN PERSON|FOB|. quickly final deposits sl|
+2341|71|10|2|37|35929.59|0.07|0.08|A|F|1993-09-23|1993-07-25|1993-10-14|DELIVER IN PERSON|RAIL|was blithel|
+2341|195|8|3|8|8761.52|0.03|0.07|R|F|1993-06-08|1993-07-09|1993-06-10|COLLECT COD|FOB|ns affix above the iron|
+2342|42|1|1|12|11304.48|0.00|0.08|N|O|1996-07-31|1996-07-26|1996-08-14|NONE|TRUCK|print blithely even deposits. carefull|
+2342|117|1|2|24|24410.64|0.10|0.06|N|O|1996-09-30|1996-07-22|1996-10-28|TAKE BACK RETURN|AIR|nstructions c|
+2342|170|1|3|50|53508.50|0.10|0.01|N|O|1996-08-28|1996-07-18|1996-09-22|COLLECT COD|RAIL|cial asymptotes pr|
+2342|36|7|4|1|936.03|0.04|0.06|N|O|1996-08-31|1996-08-09|1996-09-07|DELIVER IN PERSON|REG AIR|ffily. unusual pinto beans wake c|
+2342|27|2|5|22|20394.44|0.08|0.01|N|O|1996-08-10|1996-08-02|1996-08-31|DELIVER IN PERSON|AIR|s. ironic |
+2343|110|1|1|27|27272.97|0.00|0.00|N|O|1995-11-10|1995-11-17|1995-12-10|TAKE BACK RETURN|SHIP|old theodolites.|
+2343|66|1|2|35|33812.10|0.03|0.06|N|O|1995-10-24|1995-11-09|1995-10-26|COLLECT COD|TRUCK|ges haggle furiously carefully regular req|
+2343|179|7|3|21|22662.57|0.00|0.03|N|O|1995-09-07|1995-10-26|1995-10-07|TAKE BACK RETURN|RAIL|osits. unusual theodolites boost furio|
+2368|152|3|1|16|16834.40|0.04|0.03|R|F|1993-10-31|1993-10-22|1993-11-06|NONE|REG AIR|telets wake carefully iro|
+2368|14|5|2|32|29248.32|0.03|0.00|R|F|1993-09-23|1993-10-07|1993-09-27|COLLECT COD|TRUCK|gular courts use blithely around the|
+2368|149|6|3|39|40916.46|0.08|0.03|R|F|1993-09-03|1993-09-20|1993-09-28|COLLECT COD|RAIL|ng the doggedly ironic requests are blithe|
+2368|156|8|4|17|17954.55|0.10|0.08|A|F|1993-10-03|1993-09-27|1993-10-05|NONE|FOB|fily. slyly final ideas alongside o|
+2369|24|7|1|30|27720.60|0.05|0.04|N|O|1997-04-23|1997-02-12|1997-05-21|COLLECT COD|REG AIR|pecial deposits sleep. blithely unusual w|
+2369|169|10|2|47|50250.52|0.10|0.02|N|O|1997-01-02|1997-02-18|1997-01-13|COLLECT COD|RAIL| to the regular dep|
+2370|46|3|1|3|2838.12|0.03|0.07|R|F|1994-03-24|1994-03-26|1994-04-15|COLLECT COD|SHIP|ly regular Tiresia|
+2370|2|5|2|24|21648.00|0.00|0.05|A|F|1994-05-15|1994-04-09|1994-06-12|NONE|REG AIR|final depen|
+2370|61|2|3|32|30753.92|0.05|0.02|A|F|1994-04-24|1994-03-03|1994-05-15|DELIVER IN PERSON|MAIL|ies since the final deposits|
+2370|6|3|4|21|19026.00|0.04|0.01|R|F|1994-02-01|1994-02-19|1994-02-09|TAKE BACK RETURN|MAIL|ecial dependencies must have to |
+2371|159|4|1|37|39188.55|0.05|0.05|N|O|1998-02-11|1998-03-24|1998-02-27|DELIVER IN PERSON|TRUCK|s boost fluffil|
+2371|35|1|2|21|19635.63|0.00|0.05|N|O|1998-04-14|1998-02-14|1998-04-18|COLLECT COD|AIR|gle furiously regu|
+2371|101|4|3|11|11012.10|0.05|0.02|N|O|1998-02-25|1998-04-06|1998-03-23|TAKE BACK RETURN|TRUCK|requests. regular pinto beans wake. car|
+2371|43|6|4|33|31120.32|0.05|0.08|N|O|1998-03-30|1998-02-06|1998-04-05|DELIVER IN PERSON|AIR|deas are. express r|
+2371|165|2|5|22|23433.52|0.02|0.05|N|O|1998-03-26|1998-03-19|1998-04-16|DELIVER IN PERSON|REG AIR|y daring accounts. regular ins|
+2371|86|7|6|39|38457.12|0.05|0.03|N|O|1998-04-01|1998-03-13|1998-04-27|NONE|REG AIR|tructions. regular, stealthy packages wak|
+2371|36|2|7|32|29952.96|0.07|0.07|N|O|1998-02-15|1998-04-03|1998-02-23|NONE|REG AIR|the ruthless accounts. |
+2372|43|4|1|42|39607.68|0.08|0.02|N|O|1998-01-04|1998-01-02|1998-02-02|COLLECT COD|REG AIR|lar packages. regular|
+2372|3|10|2|17|15351.00|0.07|0.01|N|O|1997-12-17|1998-01-17|1997-12-25|NONE|RAIL|xcuses. slyly ironic theod|
+2372|164|1|3|12|12769.92|0.04|0.04|N|O|1998-03-21|1997-12-21|1998-04-12|DELIVER IN PERSON|SHIP|lyly according to|
+2372|122|1|4|4|4088.48|0.00|0.07|N|O|1997-12-14|1997-12-28|1997-12-16|TAKE BACK RETURN|REG AIR|e carefully blithely even epitaphs. r|
+2372|20|7|5|5|4600.10|0.02|0.04|N|O|1998-02-08|1998-01-18|1998-03-02|TAKE BACK RETURN|RAIL|ets against the |
+2372|189|10|6|11|11980.98|0.02|0.06|N|O|1998-02-14|1998-01-18|1998-03-10|TAKE BACK RETURN|FOB| silent, pending de|
+2372|57|8|7|19|18183.95|0.01|0.06|N|O|1997-12-26|1998-02-19|1998-01-02|COLLECT COD|SHIP| beans haggle sometimes|
+2373|191|5|1|17|18550.23|0.02|0.01|R|F|1994-03-29|1994-05-19|1994-04-20|COLLECT COD|AIR|auternes. blithely even pinto bea|
+2373|136|2|2|3|3108.39|0.08|0.08|R|F|1994-05-15|1994-06-10|1994-06-04|COLLECT COD|TRUCK|dependencies wake ironical|
+2373|141|8|3|29|30193.06|0.05|0.02|A|F|1994-06-01|1994-05-14|1994-06-17|NONE|TRUCK|yly silent ideas affix furiousl|
+2373|91|5|4|5|4955.45|0.10|0.01|R|F|1994-06-02|1994-05-03|1994-06-21|NONE|REG AIR|uffily blithely ironic requests|
+2374|118|2|1|41|41742.51|0.07|0.00|A|F|1994-01-27|1993-12-11|1994-02-12|TAKE BACK RETURN|RAIL|heodolites. requests|
+2374|160|2|2|24|25443.84|0.07|0.08|A|F|1994-02-02|1994-01-12|1994-02-04|DELIVER IN PERSON|TRUCK|. requests are above t|
+2374|61|8|3|2|1922.12|0.06|0.02|R|F|1993-12-30|1994-01-24|1994-01-02|COLLECT COD|FOB|, unusual ideas. deposits cajole quietl|
+2374|74|5|4|28|27273.96|0.04|0.08|R|F|1994-02-19|1993-12-16|1994-03-15|COLLECT COD|MAIL|ets cajole fu|
+2374|1|2|5|25|22525.00|0.08|0.00|A|F|1993-11-26|1993-12-15|1993-12-10|COLLECT COD|RAIL|refully pending d|
+2375|168|9|1|3|3204.48|0.02|0.08|N|O|1997-02-14|1996-12-25|1997-02-22|COLLECT COD|RAIL|slyly across the furiously e|
+2375|132|8|2|9|9289.17|0.09|0.02|N|O|1997-02-17|1996-12-27|1997-02-27|DELIVER IN PERSON|MAIL|ly against the packages. bold pinto bean|
+2375|47|4|3|26|24623.04|0.02|0.06|N|O|1997-03-18|1997-02-02|1997-03-29|TAKE BACK RETURN|TRUCK|rate across the|
+2375|5|8|4|5|4525.00|0.01|0.00|N|O|1997-01-31|1997-01-25|1997-02-22|COLLECT COD|REG AIR|final packages cajole according to the furi|
+2375|88|9|5|42|41499.36|0.01|0.08|N|O|1997-01-24|1997-02-15|1997-02-07|DELIVER IN PERSON|FOB|apades. idea|
+2375|126|7|6|20|20522.40|0.09|0.08|N|O|1996-12-01|1996-12-26|1996-12-19|TAKE BACK RETURN|SHIP|ckages! blithely enticing deposi|
+2400|103|6|1|48|48148.80|0.01|0.02|N|O|1998-10-07|1998-08-30|1998-11-03|DELIVER IN PERSON|MAIL|fore the car|
+2400|90|1|2|1|990.09|0.04|0.07|N|O|1998-08-18|1998-09-12|1998-09-11|NONE|MAIL|silent deposits serve furious|
+2400|53|5|3|23|21920.15|0.02|0.08|N|O|1998-08-05|1998-08-28|1998-08-30|NONE|SHIP|tions. fluffily ironic platelets cajole c|
+2400|17|7|4|23|21091.23|0.09|0.04|N|O|1998-10-04|1998-10-04|1998-10-31|NONE|RAIL|ages lose carefully around the regula|
+2401|182|3|1|39|42205.02|0.00|0.03|N|O|1997-09-29|1997-10-21|1997-10-17|DELIVER IN PERSON|FOB|ould affix |
+2401|3|8|2|49|44247.00|0.05|0.07|N|O|1997-09-02|1997-09-11|1997-09-13|TAKE BACK RETURN|AIR|lites cajole carefully |
+2402|86|7|1|43|42401.44|0.03|0.08|N|O|1996-09-17|1996-11-20|1996-09-22|DELIVER IN PERSON|RAIL|slyly slyly blithe sheaves|
+2402|152|4|2|24|25251.60|0.02|0.05|N|O|1996-11-21|1996-10-19|1996-11-29|DELIVER IN PERSON|SHIP|as; blithely ironic requ|
+2403|83|4|1|34|33424.72|0.04|0.07|N|O|1998-05-30|1998-06-19|1998-06-05|NONE|REG AIR| slyly bold re|
+2403|152|4|2|19|19990.85|0.08|0.07|N|O|1998-04-20|1998-07-02|1998-05-13|DELIVER IN PERSON|FOB|sits. ironic in|
+2403|193|4|3|27|29516.13|0.05|0.03|N|O|1998-07-27|1998-07-08|1998-08-03|NONE|SHIP|deposits sleep slyly special theodolit|
+2403|31|2|4|30|27930.90|0.05|0.06|N|O|1998-08-08|1998-06-17|1998-08-20|NONE|TRUCK|ackages sleep furiously pendin|
+2404|147|10|1|36|37697.04|0.07|0.00|N|O|1997-03-27|1997-05-16|1997-04-06|COLLECT COD|REG AIR|s nag furi|
+2404|36|2|2|1|936.03|0.02|0.04|N|O|1997-05-22|1997-06-06|1997-05-28|DELIVER IN PERSON|MAIL|from the final orbits? even pinto beans hag|
+2404|18|5|3|41|37638.41|0.02|0.06|N|O|1997-06-12|1997-05-03|1997-07-12|NONE|AIR| dolphins are|
+2404|57|8|4|19|18183.95|0.09|0.03|N|O|1997-05-07|1997-05-24|1997-05-24|TAKE BACK RETURN|SHIP|cuses. quickly even in|
+2404|4|9|5|18|16272.00|0.00|0.04|N|O|1997-06-25|1997-05-06|1997-07-02|NONE|RAIL|packages. even requests according to |
+2405|89|10|1|18|17803.44|0.09|0.07|N|O|1997-01-23|1997-03-10|1997-02-03|COLLECT COD|REG AIR|carefully ironic accounts. slyly |
+2405|27|10|2|30|27810.60|0.10|0.08|N|O|1997-03-24|1997-03-10|1997-04-14|TAKE BACK RETURN|AIR|y final deposits are slyly caref|
+2405|17|8|3|49|44933.49|0.00|0.06|N|O|1996-12-24|1997-03-23|1997-01-01|TAKE BACK RETURN|FOB|cial requests. ironic, regu|
+2405|177|7|4|23|24774.91|0.08|0.05|N|O|1996-12-28|1997-01-29|1997-01-07|NONE|AIR|t wake blithely blithely regular idea|
+2406|170|5|1|18|19263.06|0.07|0.05|N|O|1997-02-17|1996-12-25|1997-02-19|COLLECT COD|MAIL|azzle furiously careful|
+2406|41|8|2|40|37641.60|0.02|0.07|N|O|1997-01-09|1996-12-02|1997-01-16|NONE|SHIP|gular accounts caj|
+2406|50|1|3|16|15200.80|0.07|0.03|N|O|1996-10-31|1996-11-28|1996-11-08|TAKE BACK RETURN|SHIP| special accou|
+2406|146|9|4|34|35568.76|0.07|0.06|N|O|1996-12-01|1996-12-07|1996-12-16|NONE|AIR|hinly even accounts are slyly q|
+2406|187|8|5|25|27179.50|0.08|0.02|N|O|1996-12-03|1996-12-14|1996-12-26|COLLECT COD|MAIL|al, regular in|
+2406|59|4|6|22|21099.10|0.05|0.02|N|O|1996-11-22|1997-01-17|1996-12-15|NONE|TRUCK|hely even foxes unwind furiously aga|
+2406|60|2|7|30|28801.80|0.07|0.07|N|O|1997-01-17|1997-01-12|1997-01-22|TAKE BACK RETURN|TRUCK| final pinto beans han|
+2407|64|3|1|14|13496.84|0.04|0.02|N|O|1998-10-10|1998-08-25|1998-10-27|NONE|FOB|l dependencies s|
+2407|166|7|2|9|9595.44|0.07|0.05|N|O|1998-08-06|1998-08-11|1998-08-20|TAKE BACK RETURN|TRUCK|ts. special deposits are closely.|
+2407|131|2|3|39|40214.07|0.02|0.02|N|O|1998-08-20|1998-09-12|1998-08-22|DELIVER IN PERSON|MAIL|iously final deposits solv|
+2407|91|4|4|10|9910.90|0.01|0.07|N|O|1998-08-14|1998-09-10|1998-08-29|COLLECT COD|FOB| pending instructions. theodolites x-|
+2407|198|1|5|14|15374.66|0.04|0.05|N|O|1998-09-24|1998-08-18|1998-10-06|DELIVER IN PERSON|FOB|tructions wake stealt|
+2407|71|9|6|18|17479.26|0.04|0.01|N|O|1998-10-03|1998-08-30|1998-10-19|TAKE BACK RETURN|MAIL| wake carefully. fluffily |
+2407|161|8|7|7|7428.12|0.07|0.03|N|O|1998-09-11|1998-08-15|1998-09-30|TAKE BACK RETURN|MAIL|totes are carefully accordin|
+2432|50|3|1|30|28501.50|0.03|0.02|N|O|1996-09-05|1996-10-10|1996-10-05|TAKE BACK RETURN|TRUCK| requests wake alongside of|
+2432|162|3|2|8|8497.28|0.07|0.01|N|O|1996-10-16|1996-10-01|1996-11-13|COLLECT COD|RAIL|s about the bold, close deposit|
+2432|109|2|3|13|13118.30|0.07|0.06|N|O|1996-09-03|1996-10-10|1996-10-03|NONE|RAIL|arefully about the caref|
+2432|13|4|4|14|12782.14|0.00|0.06|N|O|1996-08-18|1996-09-04|1996-08-27|TAKE BACK RETURN|RAIL|riously regular packages. p|
+2433|87|8|1|39|38496.12|0.01|0.04|R|F|1994-11-20|1994-09-23|1994-12-10|DELIVER IN PERSON|SHIP|ly final asy|
+2433|134|5|2|20|20682.60|0.05|0.06|A|F|1994-12-09|1994-10-20|1994-12-15|COLLECT COD|REG AIR|lithely blithely final ide|
+2433|157|2|3|38|40171.70|0.08|0.03|A|F|1994-10-15|1994-10-23|1994-11-06|DELIVER IN PERSON|SHIP|. slyly regular requests sle|
+2433|121|6|4|43|43908.16|0.01|0.05|A|F|1994-10-16|1994-10-23|1994-11-08|DELIVER IN PERSON|RAIL|ular requests. slyly even pa|
+2433|108|1|5|3|3024.30|0.06|0.02|A|F|1994-11-08|1994-09-24|1994-11-17|COLLECT COD|AIR|usly pending depos|
+2434|95|6|1|1|995.09|0.01|0.06|N|O|1997-08-02|1997-05-28|1997-08-19|TAKE BACK RETURN|MAIL| furiously express packages. ironic, pend|
+2434|127|10|2|39|40057.68|0.09|0.05|N|O|1997-06-10|1997-06-08|1997-07-03|COLLECT COD|RAIL|r deposits sleep furiou|
+2434|130|3|3|28|28843.64|0.02|0.05|N|O|1997-06-28|1997-06-26|1997-07-15|COLLECT COD|RAIL|ven theodolites around the slyly|
+2434|168|9|4|49|52339.84|0.00|0.05|N|O|1997-08-08|1997-07-23|1997-08-27|DELIVER IN PERSON|FOB| after the requests haggle bold, fina|
+2435|39|10|1|8|7512.24|0.08|0.03|A|F|1993-06-08|1993-04-04|1993-06-29|COLLECT COD|SHIP|e fluffily quickly final accounts. care|
+2435|49|2|2|43|40808.72|0.03|0.08|A|F|1993-03-27|1993-05-20|1993-04-18|DELIVER IN PERSON|TRUCK|alongside of the s|
+2435|12|9|3|24|21888.24|0.07|0.08|R|F|1993-03-14|1993-05-20|1993-03-26|DELIVER IN PERSON|SHIP|s. carefully regular d|
+2435|156|4|4|22|23235.30|0.02|0.05|R|F|1993-05-23|1993-04-14|1993-06-04|NONE|SHIP|e final, final deposits. carefully regular|
+2435|72|2|5|3|2916.21|0.07|0.07|R|F|1993-06-01|1993-03-25|1993-06-27|DELIVER IN PERSON|FOB| final accounts ar|
+2435|46|9|6|17|16082.68|0.02|0.02|A|F|1993-06-05|1993-05-05|1993-06-14|NONE|TRUCK|cajole aft|
+2435|121|10|7|8|8168.96|0.07|0.02|R|F|1993-05-03|1993-04-02|1993-05-17|COLLECT COD|SHIP|ng the fluffily special foxes nag |
+2436|155|6|1|48|50647.20|0.04|0.02|N|O|1995-10-22|1995-10-22|1995-11-16|DELIVER IN PERSON|FOB|he furiously |
+2436|117|7|2|18|18307.98|0.05|0.03|N|O|1995-10-14|1995-11-21|1995-11-12|TAKE BACK RETURN|TRUCK|y ironic accounts. furiously even packa|
+2436|164|3|3|6|6384.96|0.06|0.08|N|O|1995-10-25|1995-11-30|1995-11-24|DELIVER IN PERSON|RAIL|odolites. ep|
+2437|94|6|1|46|45728.14|0.07|0.04|A|F|1993-08-12|1993-06-16|1993-08-29|NONE|RAIL|e of the bold, dogged requests|
+2437|190|1|2|26|28344.94|0.00|0.04|A|F|1993-06-25|1993-05-22|1993-07-07|DELIVER IN PERSON|REG AIR|lyly regular accounts.|
+2437|2|7|3|23|20746.00|0.01|0.00|A|F|1993-08-15|1993-06-28|1993-08-23|TAKE BACK RETURN|SHIP|s deposits. pendi|
+2437|116|10|4|12|12193.32|0.03|0.08|A|F|1993-04-27|1993-07-01|1993-05-18|TAKE BACK RETURN|FOB|thely regular deposits. ironic fray|
+2437|17|7|5|29|26593.29|0.02|0.06|A|F|1993-05-12|1993-06-10|1993-05-25|NONE|FOB|ress dolphins. furiously fin|
+2437|19|3|6|10|9190.10|0.10|0.06|A|F|1993-05-20|1993-06-23|1993-05-22|TAKE BACK RETURN|MAIL|unts. even, ironic pl|
+2438|165|2|1|45|47932.20|0.01|0.00|A|F|1993-10-27|1993-09-24|1993-11-02|COLLECT COD|REG AIR|en theodolites w|
+2438|13|4|2|31|28303.31|0.08|0.01|R|F|1993-10-16|1993-08-31|1993-11-10|COLLECT COD|REG AIR|t. slyly ironic sh|
+2438|68|7|3|10|9680.60|0.10|0.00|R|F|1993-08-18|1993-08-28|1993-09-08|NONE|SHIP|engage car|
+2438|161|8|4|27|28651.32|0.01|0.02|R|F|1993-07-27|1993-10-01|1993-08-06|TAKE BACK RETURN|FOB|inal accounts. slyly final reques|
+2438|166|3|5|28|29852.48|0.07|0.06|R|F|1993-11-05|1993-08-22|1993-11-22|TAKE BACK RETURN|TRUCK|ctions. bli|
+2438|149|6|6|23|24130.22|0.09|0.02|R|F|1993-10-06|1993-08-17|1993-10-16|DELIVER IN PERSON|MAIL|ely; blithely special pinto beans breach|
+2438|183|4|7|46|49826.28|0.02|0.05|R|F|1993-10-27|1993-08-30|1993-11-14|COLLECT COD|SHIP| ironic requests cajole f|
+2439|164|1|1|2|2128.32|0.09|0.03|N|O|1997-04-14|1997-06-11|1997-05-09|COLLECT COD|MAIL|courts boos|
+2439|144|5|2|5|5220.70|0.07|0.01|N|O|1997-04-23|1997-04-26|1997-04-28|DELIVER IN PERSON|FOB|ites. furiously|
+2439|195|7|3|33|36141.27|0.08|0.05|N|O|1997-06-01|1997-05-15|1997-06-07|TAKE BACK RETURN|FOB|asymptotes wake packages-- furiously|
+2464|49|8|1|10|9490.40|0.05|0.03|N|O|1998-02-04|1997-12-29|1998-02-16|TAKE BACK RETURN|RAIL|slyly final pinto bean|
+2464|101|6|2|20|20022.00|0.01|0.07|N|O|1997-12-26|1998-01-02|1998-01-24|DELIVER IN PERSON|FOB|sts. slyly close ideas shall h|
+2465|68|5|1|27|26137.62|0.05|0.02|N|O|1995-09-05|1995-09-07|1995-09-17|DELIVER IN PERSON|FOB|posits boost carefully unusual instructio|
+2465|51|3|2|34|32335.70|0.02|0.05|N|O|1995-10-02|1995-08-04|1995-10-09|COLLECT COD|RAIL|posits wake. regular package|
+2465|32|3|3|8|7456.24|0.10|0.00|N|O|1995-10-16|1995-08-26|1995-11-07|TAKE BACK RETURN|FOB|s across the express deposits wak|
+2465|148|7|4|45|47166.30|0.03|0.01|N|O|1995-09-27|1995-08-25|1995-10-06|NONE|TRUCK|y silent foxes. final pinto beans above |
+2465|47|4|5|50|47352.00|0.01|0.04|N|O|1995-09-01|1995-09-06|1995-09-18|TAKE BACK RETURN|TRUCK|the pending th|
+2465|124|5|6|20|20482.40|0.03|0.03|N|O|1995-08-16|1995-08-13|1995-09-02|COLLECT COD|FOB|uriously? furiously ironic excu|
+2466|186|7|1|16|17378.88|0.00|0.02|R|F|1994-04-20|1994-04-20|1994-05-09|COLLECT COD|FOB|to beans sl|
+2466|105|8|2|10|10051.00|0.00|0.00|A|F|1994-05-08|1994-04-06|1994-06-05|DELIVER IN PERSON|AIR|sly regular deposits. regular, regula|
+2466|14|1|3|29|26506.29|0.10|0.07|A|F|1994-06-11|1994-04-27|1994-07-10|DELIVER IN PERSON|FOB|ckages. bold requests nag carefully.|
+2466|11|8|4|29|26419.29|0.04|0.04|A|F|1994-04-01|1994-04-20|1994-04-23|DELIVER IN PERSON|MAIL|es boost fluffily ab|
+2466|79|10|5|30|29372.10|0.02|0.01|A|F|1994-04-11|1994-05-02|1994-05-02|DELIVER IN PERSON|REG AIR|. fluffily even pinto beans are idly. f|
+2466|173|2|6|19|20390.23|0.10|0.07|R|F|1994-06-12|1994-04-18|1994-07-12|NONE|MAIL|ccounts cajole a|
+2466|155|7|7|35|36930.25|0.10|0.00|A|F|1994-06-01|1994-05-27|1994-06-21|COLLECT COD|AIR| packages detect carefully: ironically sl|
+2467|133|9|1|7|7231.91|0.00|0.00|N|O|1995-07-28|1995-10-04|1995-08-27|NONE|REG AIR|gular packages cajole |
+2468|94|7|1|46|45728.14|0.00|0.04|N|O|1997-07-16|1997-08-09|1997-08-07|COLLECT COD|SHIP|unusual theodolites su|
+2468|21|10|2|43|39603.86|0.00|0.04|N|O|1997-08-17|1997-08-21|1997-08-30|DELIVER IN PERSON|FOB|uriously eve|
+2468|195|6|3|44|48188.36|0.00|0.03|N|O|1997-10-01|1997-08-02|1997-10-09|TAKE BACK RETURN|RAIL|egular, silent sheave|
+2468|82|3|4|5|4910.40|0.08|0.00|N|O|1997-06-28|1997-08-02|1997-07-22|NONE|MAIL| sleep fluffily acc|
+2468|159|7|5|18|19064.70|0.07|0.00|N|O|1997-07-25|1997-08-26|1997-08-14|DELIVER IN PERSON|REG AIR|cies. fluffily r|
+2469|166|1|1|11|11727.76|0.00|0.04|N|O|1997-02-09|1997-01-26|1997-02-16|NONE|TRUCK|ies wake carefully b|
+2469|114|1|2|16|16225.76|0.07|0.06|N|O|1997-02-19|1997-02-04|1997-03-18|NONE|MAIL|ing asymptotes |
+2469|11|5|3|48|43728.48|0.05|0.06|N|O|1997-01-11|1997-01-03|1997-01-15|TAKE BACK RETURN|AIR|riously even theodolites u|
+2469|88|9|4|35|34582.80|0.06|0.06|N|O|1997-02-04|1997-02-02|1997-02-17|DELIVER IN PERSON|RAIL|ld packages haggle regular frets. fluffily |
+2469|121|4|5|30|30633.60|0.09|0.01|N|O|1996-12-21|1997-01-29|1997-01-02|COLLECT COD|SHIP| accounts. regular theodolites affix fu|
+2469|104|5|6|49|49200.90|0.02|0.02|N|O|1997-03-03|1996-12-26|1997-03-13|NONE|AIR| requests are car|
+2469|127|10|7|8|8216.96|0.02|0.00|N|O|1997-03-15|1997-01-20|1997-04-13|NONE|TRUCK|s. regular|
+2470|110|5|1|12|12121.32|0.06|0.06|N|O|1997-07-12|1997-05-24|1997-07-17|TAKE BACK RETURN|FOB|l accounts. deposits nag daringly. express,|
+2470|100|4|2|50|50005.00|0.03|0.03|N|O|1997-06-02|1997-06-01|1997-06-09|COLLECT COD|AIR| packages |
+2470|64|3|3|10|9640.60|0.05|0.08|N|O|1997-06-20|1997-06-19|1997-06-24|TAKE BACK RETURN|FOB| ironic requests a|
+2470|162|3|4|30|31864.80|0.04|0.08|N|O|1997-08-04|1997-07-13|1997-08-14|DELIVER IN PERSON|AIR|s across the furiously fina|
+2471|84|5|1|37|36410.96|0.05|0.01|N|O|1998-05-28|1998-04-17|1998-06-08|COLLECT COD|TRUCK|ounts mold blithely carefully express depo|
+2496|141|8|1|38|39563.32|0.02|0.07|R|F|1994-03-26|1994-04-06|1994-04-23|COLLECT COD|RAIL| bold accounts. furi|
+2496|23|4|2|39|35997.78|0.03|0.00|R|F|1994-03-23|1994-02-18|1994-04-10|TAKE BACK RETURN|FOB|arefully special dependencies abo|
+2496|189|10|3|36|39210.48|0.09|0.04|R|F|1994-03-27|1994-03-15|1994-04-17|TAKE BACK RETURN|SHIP|ully ironic f|
+2496|24|9|4|30|27720.60|0.04|0.01|A|F|1994-01-27|1994-03-11|1994-01-31|DELIVER IN PERSON|RAIL|ake. ironic foxes cajole quickly. fu|
+2497|12|2|1|34|31008.34|0.02|0.03|R|F|1992-09-02|1992-10-19|1992-09-12|COLLECT COD|AIR|ronic accounts. p|
+2497|77|7|2|15|14656.05|0.09|0.02|A|F|1992-12-23|1992-11-20|1993-01-18|DELIVER IN PERSON|SHIP|sly against the|
+2497|34|5|3|28|26152.84|0.02|0.08|A|F|1992-12-02|1992-11-21|1992-12-04|DELIVER IN PERSON|REG AIR|ouches. special, regular requests|
+2497|144|5|4|48|50118.72|0.06|0.05|A|F|1992-09-29|1992-11-13|1992-10-19|TAKE BACK RETURN|AIR| even, regular requests across |
+2497|175|5|5|28|30104.76|0.04|0.05|A|F|1992-11-10|1992-09-30|1992-11-18|DELIVER IN PERSON|MAIL|hely bold ideas. unusual instructions ac|
+2497|71|2|6|19|18450.33|0.05|0.08|A|F|1992-11-10|1992-11-20|1992-12-05|TAKE BACK RETURN|TRUCK| instructions? carefully daring accounts|
+2498|143|2|1|48|50070.72|0.10|0.01|R|F|1993-11-25|1994-01-09|1993-12-24|DELIVER IN PERSON|RAIL|onic requests wake|
+2499|150|3|1|15|15752.25|0.04|0.06|N|O|1995-12-21|1995-12-06|1996-01-19|DELIVER IN PERSON|FOB| slyly across the slyly|
+2499|46|3|2|48|45409.92|0.09|0.03|N|O|1995-10-14|1995-12-12|1995-11-11|DELIVER IN PERSON|AIR|ronic ideas cajole quickly requests. caref|
+2499|133|9|3|31|32027.03|0.09|0.05|N|O|1995-12-09|1995-10-28|1996-01-05|COLLECT COD|AIR|to beans across the carefully ironic theodo|
+2499|159|7|4|39|41306.85|0.06|0.02|N|O|1995-10-26|1995-10-27|1995-11-07|TAKE BACK RETURN|SHIP|otes sublat|
+2499|130|9|5|6|6180.78|0.02|0.01|N|O|1995-11-19|1995-12-14|1995-12-08|NONE|SHIP|cording to the|
+2499|119|3|6|12|12229.32|0.04|0.05|N|O|1995-11-18|1995-12-13|1995-11-23|COLLECT COD|REG AIR|le furiously along the r|
+2500|192|3|1|40|43687.60|0.00|0.02|A|F|1992-09-02|1992-09-30|1992-09-06|DELIVER IN PERSON|SHIP|efully unusual dolphins s|
+2500|37|8|2|34|31859.02|0.06|0.02|R|F|1992-10-03|1992-11-11|1992-10-29|DELIVER IN PERSON|TRUCK| stealthy a|
+2500|80|10|3|41|40183.28|0.02|0.00|R|F|1992-09-02|1992-11-11|1992-09-06|DELIVER IN PERSON|RAIL|s could have to integrate after the |
+2500|69|8|4|17|16474.02|0.01|0.02|A|F|1992-09-30|1992-10-16|1992-10-05|DELIVER IN PERSON|REG AIR|encies-- ironic, even packages|
+2501|84|5|1|4|3936.32|0.10|0.06|N|O|1997-07-17|1997-07-27|1997-07-22|COLLECT COD|RAIL|quests. furiously final|
+2501|106|1|2|33|33201.30|0.01|0.04|N|O|1997-07-14|1997-08-09|1997-07-26|NONE|MAIL|leep furiously packages. even sauternes |
+2501|72|2|3|20|19441.40|0.10|0.06|N|O|1997-09-23|1997-07-01|1997-10-03|DELIVER IN PERSON|RAIL|equests. furiou|
+2501|58|10|4|26|24909.30|0.09|0.01|N|O|1997-07-15|1997-08-15|1997-07-28|DELIVER IN PERSON|SHIP|c accounts. express, iron|
+2502|163|4|1|33|35084.28|0.10|0.06|R|F|1993-08-12|1993-07-22|1993-09-04|COLLECT COD|REG AIR|have to print|
+2503|123|2|1|33|33762.96|0.06|0.01|R|F|1993-07-06|1993-08-14|1993-08-02|NONE|SHIP|nal courts integrate according to the|
+2503|65|10|2|28|27021.68|0.06|0.01|R|F|1993-08-08|1993-08-31|1993-08-10|NONE|SHIP|s wake quickly slyly |
+2503|46|7|3|50|47302.00|0.09|0.01|A|F|1993-09-22|1993-08-17|1993-09-29|DELIVER IN PERSON|TRUCK|s around the slyly |
+2503|91|5|4|27|26759.43|0.09|0.00|A|F|1993-07-12|1993-07-24|1993-07-22|DELIVER IN PERSON|TRUCK|lly even p|
+2503|48|5|5|3|2844.12|0.04|0.02|A|F|1993-07-10|1993-09-17|1993-07-19|TAKE BACK RETURN|TRUCK|s cajole. slyly close courts nod f|
+2503|128|7|6|39|40096.68|0.05|0.05|R|F|1993-10-11|1993-09-09|1993-10-16|NONE|MAIL|d carefully fluffily|
+2503|19|6|7|17|15623.17|0.09|0.08|R|F|1993-09-04|1993-07-31|1993-09-23|DELIVER IN PERSON|SHIP|c accounts haggle blithel|
+2528|1|2|1|10|9010.00|0.02|0.03|R|F|1994-12-12|1994-12-29|1994-12-28|COLLECT COD|REG AIR|ely. fluffily even re|
+2528|74|3|2|13|12662.91|0.00|0.03|A|F|1994-11-27|1995-01-20|1994-12-03|TAKE BACK RETURN|REG AIR|ggle furiously. slyly final asympt|
+2528|175|6|3|35|37630.95|0.10|0.00|R|F|1994-12-19|1995-02-04|1995-01-15|NONE|MAIL|, even excuses. even,|
+2528|65|4|4|37|35707.22|0.00|0.01|A|F|1994-12-25|1995-02-02|1994-12-31|COLLECT COD|AIR|ng the pending excuses haggle after the bl|
+2529|131|7|1|4|4124.52|0.07|0.07|N|O|1996-10-19|1996-11-18|1996-10-24|DELIVER IN PERSON|SHIP|al dependencies haggle slyly alongsi|
+2530|21|2|1|9|8289.18|0.09|0.03|R|F|1994-05-10|1994-04-30|1994-05-24|TAKE BACK RETURN|REG AIR|lyly ironic|
+2530|93|7|2|42|41709.78|0.04|0.08|R|F|1994-03-27|1994-05-20|1994-03-29|NONE|RAIL|ng platelets wake s|
+2530|108|1|3|8|8064.80|0.10|0.08|A|F|1994-05-02|1994-05-08|1994-05-24|DELIVER IN PERSON|MAIL|ial asymptotes snooze slyly regular |
+2531|148|7|1|9|9433.26|0.03|0.07|N|O|1996-07-27|1996-07-03|1996-08-01|DELIVER IN PERSON|AIR|t the dogged, un|
+2531|157|2|2|3|3171.45|0.07|0.06|N|O|1996-07-20|1996-06-20|1996-08-10|NONE|MAIL|he quickly ev|
+2531|86|7|3|20|19721.60|0.06|0.04|N|O|1996-07-18|1996-06-25|1996-07-29|TAKE BACK RETURN|TRUCK|into beans. furious|
+2531|191|5|4|36|39282.84|0.08|0.01|N|O|1996-06-11|1996-07-26|1996-06-27|NONE|MAIL|y ironic, bold packages. blithely e|
+2531|56|4|5|28|26769.40|0.03|0.07|N|O|1996-07-06|1996-07-31|1996-07-19|TAKE BACK RETURN|REG AIR|its. busily|
+2531|145|4|6|46|48076.44|0.10|0.08|N|O|1996-07-03|1996-06-27|1996-07-12|TAKE BACK RETURN|REG AIR|e final, bold pains. ir|
+2532|53|4|1|3|2859.15|0.06|0.07|N|O|1995-12-14|1995-11-28|1995-12-15|COLLECT COD|FOB|unusual sentiments. even pinto|
+2532|160|2|2|33|34985.28|0.06|0.05|N|O|1995-11-23|1996-01-04|1995-12-16|DELIVER IN PERSON|TRUCK|rve carefully slyly ironic accounts! fluf|
+2532|135|1|3|1|1035.13|0.00|0.06|N|O|1996-01-27|1995-11-23|1996-01-29|DELIVER IN PERSON|REG AIR|ely final ideas cajole despite the ca|
+2532|78|8|4|50|48903.50|0.02|0.02|N|O|1995-11-13|1996-01-01|1995-11-26|NONE|TRUCK|yly after the fluffily regul|
+2532|114|1|5|9|9126.99|0.09|0.04|N|O|1995-11-30|1995-11-23|1995-12-12|DELIVER IN PERSON|TRUCK|cial ideas haggle slyly pending request|
+2532|150|1|6|20|21003.00|0.09|0.05|N|O|1995-12-02|1995-11-26|1995-12-08|TAKE BACK RETURN|AIR|er the slyly pending|
+2533|54|9|1|36|34345.80|0.06|0.04|N|O|1997-06-10|1997-04-28|1997-07-01|NONE|REG AIR|ss requests sleep neve|
+2533|198|10|2|5|5490.95|0.10|0.04|N|O|1997-05-26|1997-06-02|1997-06-24|NONE|FOB|ccounts. ironic, special accounts boo|
+2533|183|4|3|37|40077.66|0.00|0.08|N|O|1997-05-10|1997-04-26|1997-05-28|COLLECT COD|SHIP| haggle carefully |
+2533|30|5|4|17|15810.51|0.06|0.02|N|O|1997-05-23|1997-05-10|1997-06-18|NONE|FOB|ackages. blith|
+2533|126|1|5|38|38992.56|0.09|0.00|N|O|1997-05-10|1997-06-02|1997-05-28|TAKE BACK RETURN|REG AIR|of the regular accounts. even packages caj|
+2533|184|5|6|20|21683.60|0.05|0.08|N|O|1997-07-04|1997-04-30|1997-07-05|COLLECT COD|FOB|thless excuses are b|
+2533|94|7|7|14|13917.26|0.06|0.04|N|O|1997-07-06|1997-05-08|1997-08-03|COLLECT COD|FOB|ut the pending, special depos|
+2534|139|5|1|29|30134.77|0.07|0.07|N|O|1996-08-09|1996-09-29|1996-08-11|COLLECT COD|TRUCK|ugouts haggle slyly. final|
+2534|27|6|2|49|45423.98|0.08|0.08|N|O|1996-09-01|1996-08-20|1996-09-06|NONE|SHIP|sometimes regular requests. blithely unus|
+2534|1|4|3|50|45050.00|0.10|0.06|N|O|1996-09-25|1996-10-07|1996-10-09|TAKE BACK RETURN|AIR|ideas. deposits use. slyly regular pa|
+2534|75|3|4|43|41928.01|0.09|0.02|N|O|1996-10-25|1996-09-30|1996-11-05|TAKE BACK RETURN|REG AIR|ngly final depos|
+2534|165|2|5|14|14912.24|0.05|0.02|N|O|1996-08-12|1996-09-26|1996-08-28|COLLECT COD|MAIL|eposits doze quickly final|
+2534|116|10|6|12|12193.32|0.02|0.02|N|O|1996-07-29|1996-10-12|1996-08-14|TAKE BACK RETURN|AIR|sual depos|
+2534|173|3|7|17|18243.89|0.02|0.07|N|O|1996-07-22|1996-09-15|1996-08-03|NONE|SHIP|riously regular |
+2535|199|2|1|5|5495.95|0.06|0.01|A|F|1993-09-07|1993-07-25|1993-09-29|DELIVER IN PERSON|REG AIR|, unusual reque|
+2535|39|5|2|12|11268.36|0.08|0.05|A|F|1993-07-17|1993-08-17|1993-07-31|TAKE BACK RETURN|FOB|uses sleep among the packages. excuses |
+2535|54|5|3|5|4770.25|0.09|0.06|R|F|1993-07-28|1993-08-14|1993-08-11|DELIVER IN PERSON|SHIP| across the express requests. silent, eve|
+2535|160|5|4|19|20143.04|0.01|0.02|A|F|1993-06-01|1993-08-01|1993-06-19|DELIVER IN PERSON|FOB|ructions. final requests|
+2535|174|3|5|25|26854.25|0.07|0.04|A|F|1993-07-19|1993-08-07|1993-07-27|NONE|REG AIR|ions believe ab|
+2560|169|10|1|41|43835.56|0.07|0.01|R|F|1992-10-23|1992-11-11|1992-11-22|NONE|SHIP| after the accounts. regular foxes are be|
+2560|4|9|2|27|24408.00|0.00|0.01|R|F|1992-12-03|1992-11-16|1992-12-30|NONE|MAIL| against the carefully|
+2560|46|5|3|31|29327.24|0.01|0.05|A|F|1992-11-14|1992-10-14|1992-12-11|DELIVER IN PERSON|AIR|to beans. blithely regular Tiresias int|
+2560|72|1|4|36|34994.52|0.01|0.02|A|F|1992-10-18|1992-10-30|1992-11-05|TAKE BACK RETURN|MAIL|accounts alongside of the excuses are |
+2560|42|1|5|9|8478.36|0.04|0.02|A|F|1992-10-23|1992-10-29|1992-11-02|COLLECT COD|REG AIR| deposits affix quickly. unusual, eve|
+2560|108|9|6|13|13105.30|0.03|0.06|A|F|1992-09-07|1992-10-21|1992-09-24|COLLECT COD|FOB|slyly final accoun|
+2561|25|4|1|32|29600.64|0.02|0.01|N|O|1998-01-05|1997-12-28|1998-01-26|DELIVER IN PERSON|REG AIR|bold packages wake slyly. slyly|
+2561|98|1|2|5|4990.45|0.07|0.04|N|O|1997-12-27|1998-01-23|1998-01-13|TAKE BACK RETURN|AIR|p ironic, regular pinto beans.|
+2561|173|4|3|47|50438.99|0.04|0.02|N|O|1997-11-19|1998-01-21|1997-12-03|DELIVER IN PERSON|REG AIR|larly pending t|
+2561|108|9|4|39|39315.90|0.08|0.06|N|O|1998-01-20|1997-12-16|1998-02-05|TAKE BACK RETURN|MAIL|equests are furiously against the|
+2561|150|3|5|2|2100.30|0.04|0.08|N|O|1998-03-14|1998-01-21|1998-03-27|DELIVER IN PERSON|TRUCK|s are. silently silent foxes sleep about|
+2561|51|6|6|14|13314.70|0.02|0.03|N|O|1998-03-07|1998-02-04|1998-03-21|COLLECT COD|RAIL|ep unusual, ironic accounts|
+2562|53|5|1|28|26685.40|0.04|0.03|R|F|1992-10-04|1992-09-24|1992-10-09|COLLECT COD|MAIL|ans haggle special, special packages. |
+2562|148|9|2|1|1048.14|0.01|0.06|R|F|1992-10-16|1992-09-18|1992-10-17|NONE|TRUCK| slyly final ideas haggle car|
+2562|66|7|3|25|24151.50|0.05|0.03|A|F|1992-11-23|1992-10-08|1992-12-19|DELIVER IN PERSON|REG AIR| accounts-- silent, unusual ideas a|
+2562|148|1|4|37|38781.18|0.08|0.03|R|F|1992-10-29|1992-10-06|1992-11-09|COLLECT COD|FOB|. slyly regular ideas according to the fl|
+2562|160|8|5|29|30744.64|0.05|0.08|A|F|1992-11-01|1992-09-29|1992-11-13|TAKE BACK RETURN|MAIL|eep against the furiously r|
+2562|50|7|6|17|16150.85|0.01|0.06|A|F|1992-10-15|1992-10-08|1992-10-26|DELIVER IN PERSON|TRUCK|lar pinto beans. blithely ev|
+2563|65|4|1|10|9650.60|0.07|0.04|A|F|1994-01-26|1993-12-19|1994-01-28|DELIVER IN PERSON|AIR|tealthily abo|
+2563|167|4|2|28|29880.48|0.04|0.03|R|F|1994-03-17|1994-02-04|1994-04-13|TAKE BACK RETURN|RAIL|hely regular depe|
+2563|119|9|3|39|39745.29|0.07|0.00|R|F|1994-02-10|1993-12-31|1994-02-19|COLLECT COD|FOB|lent requests should integrate; carefully e|
+2563|90|1|4|50|49504.50|0.01|0.01|A|F|1994-01-26|1994-01-03|1994-02-09|DELIVER IN PERSON|SHIP|ly regular, regular excuses. bold plate|
+2563|15|6|5|42|38430.42|0.06|0.08|R|F|1994-02-21|1994-02-14|1994-03-04|DELIVER IN PERSON|AIR|ymptotes nag furiously slyly even inst|
+2563|121|2|6|5|5105.60|0.10|0.00|R|F|1993-12-27|1993-12-19|1994-01-02|DELIVER IN PERSON|REG AIR| the quickly final theodolite|
+2564|112|3|1|4|4048.44|0.02|0.00|R|F|1994-11-12|1994-10-29|1994-12-04|NONE|MAIL|y express requests sleep furi|
+2565|144|5|1|42|43853.88|0.04|0.08|N|O|1998-04-07|1998-04-02|1998-05-04|NONE|AIR|ngly silent |
+2565|189|10|2|26|28318.68|0.05|0.08|N|O|1998-05-07|1998-04-09|1998-05-15|DELIVER IN PERSON|TRUCK| pinto beans about the slyly regula|
+2565|115|5|3|34|34513.74|0.06|0.06|N|O|1998-03-19|1998-04-12|1998-04-17|DELIVER IN PERSON|SHIP|nstructions was carefu|
+2565|17|7|4|25|22925.25|0.10|0.08|N|O|1998-06-27|1998-05-20|1998-07-13|DELIVER IN PERSON|RAIL|, express accounts. final id|
+2565|76|7|5|26|25377.82|0.08|0.03|N|O|1998-03-05|1998-04-11|1998-03-11|TAKE BACK RETURN|AIR|ites wake. ironic acco|
+2565|141|4|6|48|49974.72|0.08|0.07|N|O|1998-06-18|1998-05-06|1998-07-13|DELIVER IN PERSON|TRUCK|r instructions sleep qui|
+2566|148|5|1|19|19914.66|0.06|0.07|R|F|1992-12-21|1992-11-24|1992-12-22|DELIVER IN PERSON|MAIL|ests. silent|
+2566|181|2|2|42|45409.56|0.08|0.02|R|F|1992-12-20|1992-12-22|1992-12-29|COLLECT COD|MAIL|ously ironic accounts|
+2566|23|8|3|18|16614.36|0.09|0.02|A|F|1992-11-16|1992-12-24|1992-12-16|COLLECT COD|FOB| braids according t|
+2566|42|9|4|3|2826.12|0.05|0.02|A|F|1992-11-04|1992-12-30|1992-12-04|TAKE BACK RETURN|FOB|ckages are ironic Tiresias. furious|
+2566|22|3|5|9|8298.18|0.04|0.03|R|F|1992-12-14|1992-12-28|1992-12-16|NONE|FOB|blithely bold accounts? quickl|
+2566|128|3|6|1|1028.12|0.07|0.03|A|F|1992-10-28|1992-11-20|1992-11-22|TAKE BACK RETURN|AIR|theodolites wake pending|
+2567|26|9|1|39|36114.78|0.03|0.04|N|O|1998-05-10|1998-05-10|1998-05-21|NONE|SHIP|ns. furiously final dependencies cajo|
+2567|112|3|2|50|50605.50|0.06|0.05|N|O|1998-05-05|1998-04-18|1998-05-09|DELIVER IN PERSON|TRUCK|. carefully pending foxes are furi|
+2567|52|10|3|6|5712.30|0.03|0.06|N|O|1998-04-21|1998-04-14|1998-05-11|NONE|RAIL|s cajole regular, final acco|
+2567|158|6|4|50|52907.50|0.05|0.03|N|O|1998-03-27|1998-05-25|1998-04-23|DELIVER IN PERSON|FOB|pinto beans? r|
+2567|81|2|5|46|45129.68|0.07|0.02|N|O|1998-06-02|1998-04-30|1998-06-13|COLLECT COD|AIR|efully pending epitaphs. carefully reg|
+2567|100|3|6|32|32003.20|0.01|0.07|N|O|1998-05-24|1998-04-30|1998-06-14|NONE|RAIL| the even, iro|
+2567|135|6|7|43|44510.59|0.06|0.02|N|O|1998-05-11|1998-04-15|1998-05-29|NONE|RAIL|requests. final courts cajole |
+2592|90|1|1|7|6930.63|0.10|0.04|R|F|1993-03-13|1993-04-25|1993-04-01|NONE|REG AIR| carefully special theodolites integrate |
+2592|66|1|2|2|1932.12|0.10|0.00|A|F|1993-03-24|1993-04-05|1993-04-16|DELIVER IN PERSON|RAIL|side of the b|
+2593|105|2|1|37|37188.70|0.08|0.06|R|F|1993-12-14|1993-10-08|1994-01-04|NONE|SHIP|s wake bravel|
+2593|90|1|2|28|27722.52|0.08|0.03|A|F|1993-10-30|1993-10-18|1993-11-06|DELIVER IN PERSON|SHIP|y even escapades shall|
+2593|128|3|3|6|6168.72|0.04|0.05|A|F|1993-11-28|1993-10-04|1993-12-28|TAKE BACK RETURN|REG AIR|ular packages. re|
+2593|161|10|4|44|46691.04|0.02|0.08|A|F|1993-09-05|1993-10-23|1993-09-29|NONE|RAIL|ents impress furiously; unusual theodoli|
+2593|4|5|5|3|2712.00|0.03|0.00|A|F|1993-12-16|1993-11-01|1993-12-29|COLLECT COD|SHIP|the furiously |
+2593|175|6|6|1|1075.17|0.08|0.08|A|F|1993-11-23|1993-10-25|1993-12-04|DELIVER IN PERSON|RAIL| accounts wake slyly |
+2593|192|5|7|11|12014.09|0.00|0.07|R|F|1993-11-01|1993-11-19|1993-11-28|TAKE BACK RETURN|RAIL|express packages sleep bold re|
+2594|72|3|1|7|6804.49|0.06|0.02|R|F|1993-03-26|1993-03-05|1993-04-24|DELIVER IN PERSON|FOB|arls cajole |
+2594|124|9|2|13|13313.56|0.10|0.05|R|F|1993-02-06|1993-03-01|1993-02-23|TAKE BACK RETURN|TRUCK|fully special accounts use courts|
+2594|126|1|3|24|24626.88|0.03|0.00|A|F|1993-01-31|1993-03-10|1993-02-04|COLLECT COD|REG AIR|lar accounts sleep fur|
+2594|144|7|4|46|48030.44|0.00|0.08|R|F|1993-04-17|1993-03-06|1993-04-21|TAKE BACK RETURN|SHIP|beans. instructions across t|
+2595|61|2|1|42|40364.52|0.08|0.02|N|O|1996-03-24|1996-01-28|1996-04-10|DELIVER IN PERSON|MAIL|ggle furiou|
+2595|88|9|2|30|29642.40|0.05|0.01|N|O|1996-03-05|1996-02-23|1996-03-19|NONE|AIR|ctions. regula|
+2595|24|3|3|19|17556.38|0.01|0.05|N|O|1995-12-23|1996-03-02|1996-01-17|COLLECT COD|MAIL|ns are neve|
+2595|159|1|4|29|30715.35|0.07|0.05|N|O|1996-01-01|1996-02-13|1996-01-18|TAKE BACK RETURN|RAIL|ronic accounts haggle carefully fin|
+2595|86|7|5|30|29582.40|0.09|0.07|N|O|1996-03-16|1996-01-31|1996-04-05|TAKE BACK RETURN|FOB|. final orbits cajole |
+2595|82|3|6|31|30444.48|0.06|0.04|N|O|1996-02-07|1996-02-10|1996-03-05|DELIVER IN PERSON|AIR|tipliers w|
+2596|170|5|1|6|6421.02|0.05|0.01|N|O|1996-12-15|1996-11-02|1996-12-29|TAKE BACK RETURN|TRUCK|ily special re|
+2596|139|10|2|43|44682.59|0.07|0.03|N|O|1996-09-03|1996-10-26|1996-09-15|NONE|FOB|ial packages haggl|
+2596|39|5|3|19|17841.57|0.10|0.00|N|O|1996-09-02|1996-11-03|1996-09-06|COLLECT COD|AIR|ias mold! sp|
+2596|105|6|4|10|10051.00|0.06|0.05|N|O|1996-08-25|1996-11-05|1996-09-13|DELIVER IN PERSON|REG AIR| instructions shall have|
+2597|84|5|1|24|23617.92|0.07|0.00|A|F|1993-05-15|1993-03-06|1993-05-25|TAKE BACK RETURN|FOB|pending packages. enticingly fi|
+2598|7|4|1|12|10884.00|0.00|0.01|N|O|1996-06-17|1996-04-12|1996-06-24|COLLECT COD|TRUCK|express packages nag sly|
+2598|148|7|2|40|41925.60|0.07|0.02|N|O|1996-05-11|1996-05-19|1996-06-08|TAKE BACK RETURN|AIR|the enticing|
+2598|104|9|3|4|4016.40|0.03|0.03|N|O|1996-05-23|1996-05-13|1996-05-25|COLLECT COD|AIR| across the furiously fi|
+2598|23|2|4|19|17537.38|0.02|0.00|N|O|1996-04-09|1996-05-30|1996-04-17|TAKE BACK RETURN|RAIL|nic packages. even accounts|
+2598|106|3|5|12|12073.20|0.01|0.08|N|O|1996-04-14|1996-04-24|1996-04-21|TAKE BACK RETURN|REG AIR|eposits cajol|
+2599|101|4|1|11|11012.10|0.08|0.08|N|O|1997-02-01|1996-12-14|1997-02-27|TAKE BACK RETURN|FOB| express accoun|
+2599|42|5|2|26|24493.04|0.03|0.04|N|O|1996-11-08|1996-12-21|1996-11-24|TAKE BACK RETURN|AIR|nag carefully |
+2599|99|10|3|29|28973.61|0.09|0.03|N|O|1997-01-10|1996-12-10|1997-02-02|COLLECT COD|RAIL|ly express dolphins. special, |
+2624|63|10|1|15|14445.90|0.03|0.07|N|O|1997-02-28|1997-02-19|1997-03-21|DELIVER IN PERSON|AIR|le. quickly pending requests|
+2624|189|10|2|12|13070.16|0.07|0.00|N|O|1997-02-24|1997-02-22|1997-02-27|DELIVER IN PERSON|SHIP|er the quickly unu|
+2625|20|1|1|42|38640.84|0.02|0.04|R|F|1992-10-18|1992-11-17|1992-10-23|DELIVER IN PERSON|AIR| even accounts haggle furiously|
+2626|22|5|1|45|41490.90|0.09|0.04|N|O|1995-11-22|1995-11-01|1995-11-23|NONE|AIR|deposits wake blithely according to |
+2626|175|3|2|2|2150.34|0.05|0.07|N|O|1995-10-19|1995-11-09|1995-10-24|TAKE BACK RETURN|FOB|uffy accounts haggle furiously above|
+2626|154|2|3|40|42166.00|0.05|0.07|N|O|1995-09-28|1995-12-03|1995-10-10|NONE|REG AIR|eans. ironic deposits haggle. depo|
+2627|131|7|1|28|28871.64|0.09|0.02|R|F|1992-05-14|1992-05-09|1992-05-31|COLLECT COD|SHIP|ggedly final excuses nag packages. f|
+2628|106|9|1|44|44268.40|0.07|0.03|R|F|1994-01-11|1994-01-14|1994-01-13|DELIVER IN PERSON|SHIP|lyly final, pending ide|
+2628|106|9|2|14|14085.40|0.01|0.03|A|F|1994-01-28|1993-11-30|1994-02-20|TAKE BACK RETURN|SHIP|g the furiously unusual pi|
+2628|64|9|3|42|40490.52|0.00|0.00|A|F|1993-11-20|1994-01-04|1993-12-19|DELIVER IN PERSON|TRUCK|ld notornis alongside |
+2628|95|7|4|23|22887.07|0.08|0.04|A|F|1993-10-27|1994-01-08|1993-11-12|DELIVER IN PERSON|TRUCK|usual packages sleep about the fina|
+2628|90|1|5|50|49504.50|0.07|0.01|A|F|1994-01-13|1993-12-11|1994-01-14|NONE|AIR|posits serve carefully toward |
+2629|118|9|1|6|6108.66|0.06|0.05|N|O|1998-06-10|1998-05-29|1998-06-13|DELIVER IN PERSON|SHIP|dolites hinder bli|
+2629|124|7|2|31|31747.72|0.08|0.03|N|O|1998-05-24|1998-05-26|1998-06-10|COLLECT COD|AIR|ate blithely bold, regular deposits. bold|
+2629|128|9|3|29|29815.48|0.08|0.07|N|O|1998-07-09|1998-06-17|1998-07-12|TAKE BACK RETURN|AIR|eposits serve unusual, express i|
+2629|70|5|4|33|32012.31|0.06|0.03|N|O|1998-05-29|1998-05-14|1998-05-30|NONE|TRUCK|es. slowly express accounts are along the|
+2630|29|8|1|46|42734.92|0.05|0.03|R|F|1992-11-05|1992-12-17|1992-12-05|TAKE BACK RETURN|MAIL|uests cajole. e|
+2630|57|2|2|8|7656.40|0.09|0.07|A|F|1992-11-16|1993-01-01|1992-12-07|DELIVER IN PERSON|TRUCK|indle fluffily silent, ironic pi|
+2630|173|2|3|45|48292.65|0.08|0.07|A|F|1993-01-04|1993-01-11|1993-01-09|NONE|FOB|edly express ideas. carefully final |
+2630|162|9|4|29|30802.64|0.08|0.07|A|F|1992-12-03|1993-01-04|1992-12-12|DELIVER IN PERSON|SHIP|efully unusual dependencies. even i|
+2631|122|7|1|42|42929.04|0.00|0.03|A|F|1994-01-04|1993-12-01|1994-01-16|TAKE BACK RETURN|SHIP|ect carefully at the furiously final the|
+2631|67|4|2|4|3868.24|0.07|0.06|R|F|1993-11-03|1993-12-17|1993-11-05|COLLECT COD|AIR|special theodolites. a|
+2631|118|8|3|15|15271.65|0.06|0.05|A|F|1993-09-30|1993-11-06|1993-10-13|DELIVER IN PERSON|SHIP|y. furiously even pinto be|
+2656|181|2|1|10|10811.80|0.02|0.06|R|F|1993-06-28|1993-07-04|1993-07-12|TAKE BACK RETURN|TRUCK|s nag regularly about the deposits. slyly|
+2656|137|8|2|38|39410.94|0.07|0.02|A|F|1993-06-25|1993-06-04|1993-07-24|NONE|RAIL|structions wake along the furio|
+2656|2|5|3|19|17138.00|0.03|0.02|R|F|1993-08-03|1993-07-25|1993-08-20|TAKE BACK RETURN|MAIL|ts serve deposi|
+2656|110|3|4|40|40404.40|0.05|0.04|R|F|1993-06-09|1993-07-24|1993-06-21|DELIVER IN PERSON|RAIL|refully final pearls. final ideas wake. qu|
+2657|115|9|1|22|22332.42|0.02|0.03|N|O|1995-12-08|1995-12-28|1995-12-21|TAKE BACK RETURN|MAIL|r ideas. furiously special dolphins|
+2657|165|2|2|15|15977.40|0.08|0.05|N|O|1995-12-09|1995-12-16|1995-12-18|NONE|RAIL|ole carefully above the ironic ideas. b|
+2657|79|9|3|25|24476.75|0.02|0.04|N|O|1995-10-21|1995-12-12|1995-11-09|COLLECT COD|FOB|lly pinto beans. final |
+2657|55|7|4|11|10505.55|0.04|0.08|N|O|1995-11-19|1995-12-11|1995-11-24|COLLECT COD|TRUCK|ckly enticing requests. fur|
+2657|78|9|5|42|41078.94|0.06|0.03|N|O|1996-01-23|1995-11-22|1996-01-25|COLLECT COD|RAIL|ckly slyly even accounts. platelets x-ray|
+2657|194|7|6|31|33919.89|0.01|0.03|N|O|1995-11-10|1995-11-27|1995-12-06|COLLECT COD|RAIL|re blithely |
+2658|132|3|1|41|42317.33|0.05|0.04|N|O|1995-11-07|1995-11-04|1995-12-04|NONE|MAIL|eposits. furiously final theodolite|
+2658|29|4|2|22|20438.44|0.08|0.05|N|O|1995-11-12|1995-11-18|1995-11-14|DELIVER IN PERSON|TRUCK|ts cajole. pending packages affix|
+2658|18|5|3|13|11934.13|0.07|0.06|N|O|1995-10-24|1995-12-12|1995-11-14|COLLECT COD|FOB|s kindle blithely regular accounts.|
+2658|92|5|4|22|21825.98|0.04|0.04|N|O|1995-12-02|1995-11-03|1995-12-26|DELIVER IN PERSON|SHIP| dependencies. blithely pending foxes abou|
+2658|7|8|5|45|40815.00|0.03|0.01|N|O|1995-11-02|1995-11-08|1995-11-29|DELIVER IN PERSON|MAIL|e special requests. quickly ex|
+2658|147|4|6|27|28272.78|0.05|0.07|N|O|1995-09-26|1995-12-08|1995-09-30|NONE|AIR|ecial packages use abov|
+2659|42|1|1|28|26377.12|0.08|0.05|A|F|1994-03-17|1994-01-24|1994-03-19|NONE|FOB|idle tithes|
+2659|43|2|2|21|19803.84|0.00|0.00|A|F|1993-12-23|1994-02-10|1994-01-17|DELIVER IN PERSON|RAIL|y beyond the furiously even co|
+2659|135|1|3|24|24843.12|0.04|0.03|R|F|1994-03-28|1994-02-20|1994-04-05|DELIVER IN PERSON|REG AIR| haggle carefully |
+2659|119|6|4|2|2038.22|0.00|0.08|R|F|1994-02-19|1994-03-12|1994-02-21|NONE|MAIL|sts above the fluffily express fo|
+2659|7|4|5|9|8163.00|0.08|0.03|A|F|1994-02-07|1994-03-17|1994-03-04|DELIVER IN PERSON|AIR|ly final packages sleep ac|
+2660|48|7|1|17|16116.68|0.00|0.05|N|O|1995-08-18|1995-09-13|1995-09-17|NONE|SHIP|al pinto beans wake after the furious|
+2661|178|9|1|31|33423.27|0.03|0.02|N|O|1997-04-07|1997-03-10|1997-04-23|TAKE BACK RETURN|AIR|e ironicall|
+2661|103|8|2|22|22068.20|0.08|0.02|N|O|1997-03-14|1997-03-17|1997-04-08|COLLECT COD|REG AIR| foxes affix quickly ironic request|
+2661|67|6|3|11|10637.66|0.00|0.08|N|O|1997-04-14|1997-02-11|1997-05-05|TAKE BACK RETURN|FOB|equests are a|
+2661|137|8|4|41|42522.33|0.06|0.02|N|O|1997-03-06|1997-03-27|1997-03-15|DELIVER IN PERSON|AIR|iously ironically ironic requests. |
+2662|102|5|1|43|43090.30|0.09|0.07|N|O|1996-11-24|1996-11-04|1996-12-08|NONE|RAIL|. slyly specia|
+2662|128|9|2|8|8224.96|0.02|0.07|N|O|1996-09-10|1996-10-09|1996-09-21|TAKE BACK RETURN|REG AIR|ajole carefully. sp|
+2662|2|5|3|6|5412.00|0.02|0.00|N|O|1996-11-30|1996-09-20|1996-12-03|DELIVER IN PERSON|REG AIR|olites cajole quickly along the b|
+2662|30|1|4|34|31621.02|0.06|0.07|N|O|1996-10-04|1996-11-05|1996-10-19|NONE|SHIP|ding theodolites use carefully. p|
+2663|114|4|1|35|35493.85|0.02|0.01|N|O|1995-12-11|1995-10-16|1996-01-07|TAKE BACK RETURN|REG AIR|tect. slyly fina|
+2688|18|5|1|45|41310.45|0.08|0.08|R|F|1992-05-21|1992-04-14|1992-05-28|NONE|FOB|sits run carefully|
+2688|15|6|2|46|42090.46|0.01|0.01|R|F|1992-05-24|1992-04-01|1992-05-26|COLLECT COD|TRUCK|elets. regular reque|
+2688|89|10|3|30|29672.40|0.05|0.04|A|F|1992-04-18|1992-03-18|1992-05-18|TAKE BACK RETURN|RAIL|ithely final |
+2688|25|10|4|3|2775.06|0.00|0.03|R|F|1992-02-04|1992-03-18|1992-02-24|DELIVER IN PERSON|RAIL|e fluffily |
+2688|59|10|5|22|21099.10|0.02|0.05|R|F|1992-02-09|1992-04-09|1992-02-11|DELIVER IN PERSON|RAIL|press, ironic excuses wake carefully id|
+2688|149|10|6|42|44063.88|0.01|0.01|R|F|1992-04-29|1992-04-04|1992-05-17|TAKE BACK RETURN|FOB|lly even account|
+2689|6|1|1|45|40770.00|0.02|0.04|R|F|1992-04-29|1992-06-22|1992-04-30|COLLECT COD|SHIP|e quickly. carefully silent|
+2690|140|1|1|44|45766.16|0.05|0.06|N|O|1996-05-30|1996-05-19|1996-06-26|NONE|REG AIR|ly alongside of th|
+2690|51|2|2|50|47552.50|0.03|0.03|N|O|1996-06-13|1996-05-22|1996-06-14|DELIVER IN PERSON|MAIL| doubt careful|
+2690|125|6|3|45|46130.40|0.02|0.07|N|O|1996-05-23|1996-06-02|1996-05-29|DELIVER IN PERSON|MAIL|ounts. slyly regular dependencies wa|
+2690|195|6|4|12|13142.28|0.04|0.07|N|O|1996-07-18|1996-06-03|1996-07-25|NONE|AIR|nal, regular atta|
+2690|86|7|5|30|29582.40|0.01|0.08|N|O|1996-05-20|1996-06-01|1996-06-04|TAKE BACK RETURN|SHIP|d accounts above the express req|
+2690|189|10|6|3|3267.54|0.07|0.01|N|O|1996-07-04|1996-05-28|1996-07-06|TAKE BACK RETURN|RAIL|. final reques|
+2690|79|7|7|35|34267.45|0.05|0.06|N|O|1996-07-25|1996-05-14|1996-08-03|COLLECT COD|FOB|y silent pinto be|
+2691|91|3|1|11|10901.99|0.04|0.07|R|F|1992-06-21|1992-06-08|1992-07-09|COLLECT COD|FOB|leep alongside of the accounts. slyly ironi|
+2691|48|7|2|2|1896.08|0.00|0.07|R|F|1992-05-10|1992-06-04|1992-05-11|TAKE BACK RETURN|TRUCK|s cajole at the blithely ironic warthog|
+2691|162|3|3|16|16994.56|0.09|0.03|R|F|1992-06-11|1992-07-29|1992-06-29|NONE|RAIL|bove the even foxes. unusual theodoli|
+2691|166|3|4|1|1066.16|0.08|0.00|A|F|1992-08-11|1992-06-07|1992-08-16|NONE|SHIP|egular instructions b|
+2692|17|1|1|3|2751.03|0.10|0.04|N|O|1998-02-25|1998-01-29|1998-03-27|TAKE BACK RETURN|MAIL|equests. bold, even foxes haggle slyl|
+2692|114|1|2|21|21296.31|0.03|0.05|N|O|1998-03-11|1998-02-11|1998-03-19|NONE|SHIP|posits. final, express requests nag furi|
+2693|9|10|1|26|23634.00|0.04|0.00|N|O|1996-09-14|1996-10-07|1996-10-03|COLLECT COD|MAIL|cajole alo|
+2693|102|3|2|43|43090.30|0.03|0.04|N|O|1996-10-24|1996-10-24|1996-11-03|TAKE BACK RETURN|TRUCK|as are according to th|
+2694|153|1|1|30|31594.50|0.02|0.06|N|O|1996-06-20|1996-06-01|1996-07-15|NONE|TRUCK|oxes. never iro|
+2694|157|2|2|35|37000.25|0.07|0.03|N|O|1996-05-24|1996-06-01|1996-05-25|NONE|RAIL|atelets past the furiously final deposits |
+2694|19|3|3|15|13785.15|0.08|0.02|N|O|1996-06-30|1996-05-01|1996-07-25|TAKE BACK RETURN|REG AIR|e blithely even platelets. special wa|
+2694|20|10|4|12|11040.24|0.00|0.05|N|O|1996-04-24|1996-04-22|1996-05-14|DELIVER IN PERSON|RAIL|foxes atop the hockey pla|
+2694|108|9|5|10|10081.00|0.08|0.08|N|O|1996-06-23|1996-05-28|1996-06-27|COLLECT COD|REG AIR|fluffily fluffy accounts. even packages hi|
+2695|184|5|1|21|22767.78|0.07|0.00|N|O|1996-10-04|1996-11-02|1996-10-21|NONE|MAIL|y regular pinto beans. evenly regular packa|
+2695|19|9|2|44|40436.44|0.09|0.07|N|O|1996-10-05|1996-10-10|1996-11-01|NONE|MAIL|ts. busy platelets boost|
+2695|144|7|3|21|21926.94|0.02|0.07|N|O|1996-09-13|1996-09-25|1996-10-13|NONE|TRUCK|s. furiously ironic platelets ar|
+2695|58|6|4|16|15328.80|0.08|0.08|N|O|1996-11-16|1996-10-05|1996-11-22|NONE|TRUCK|its. theodolites sleep slyly|
+2695|86|7|5|40|39443.20|0.02|0.03|N|O|1996-11-02|1996-10-26|1996-11-14|NONE|FOB|ructions. pending|
+2720|45|6|1|5|4725.20|0.10|0.06|A|F|1993-06-24|1993-08-08|1993-07-08|NONE|FOB|ously ironic foxes thrash|
+2720|17|8|2|42|38514.42|0.09|0.03|R|F|1993-07-25|1993-07-23|1993-08-23|COLLECT COD|REG AIR|fter the inst|
+2720|120|1|3|50|51006.00|0.10|0.02|A|F|1993-08-10|1993-07-29|1993-09-06|NONE|SHIP|l requests. deposits nag furiously|
+2720|109|2|4|49|49445.90|0.06|0.02|A|F|1993-07-09|1993-07-14|1993-07-13|NONE|REG AIR| accounts. fluffily bold pack|
+2720|121|6|5|27|27570.24|0.04|0.00|R|F|1993-06-29|1993-08-06|1993-07-28|NONE|TRUCK|eas. carefully regular |
+2721|183|4|1|49|53075.82|0.00|0.08|N|O|1996-02-14|1996-04-26|1996-03-02|DELIVER IN PERSON|AIR|ounts poach carefu|
+2721|3|4|2|2|1806.00|0.02|0.05|N|O|1996-02-13|1996-03-14|1996-02-28|TAKE BACK RETURN|TRUCK| slyly final requests against |
+2722|124|7|1|21|21506.52|0.09|0.01|A|F|1994-07-29|1994-06-26|1994-08-09|NONE|RAIL|e carefully around the furiously ironic pac|
+2722|146|7|2|15|15692.10|0.05|0.03|R|F|1994-07-02|1994-06-01|1994-07-13|COLLECT COD|AIR|refully final asympt|
+2722|34|10|3|16|14944.48|0.04|0.06|R|F|1994-05-25|1994-06-09|1994-05-26|NONE|MAIL|ts besides the fluffy,|
+2723|13|7|1|47|42911.47|0.09|0.07|N|O|1995-12-05|1995-11-19|1995-12-11|TAKE BACK RETURN|AIR|furiously r|
+2723|32|3|2|10|9320.30|0.06|0.08|N|O|1995-11-27|1995-11-29|1995-12-12|DELIVER IN PERSON|MAIL|al, special r|
+2723|162|1|3|2|2124.32|0.10|0.01|N|O|1995-11-09|1995-11-10|1995-11-14|TAKE BACK RETURN|FOB| courts boost quickly about th|
+2723|82|3|4|12|11784.96|0.01|0.05|N|O|1995-12-24|1995-11-15|1996-01-17|DELIVER IN PERSON|RAIL|bold foxes are bold packages. regular, fin|
+2723|129|10|5|40|41164.80|0.09|0.05|N|O|1995-11-17|1995-11-22|1995-11-18|TAKE BACK RETURN|MAIL|unwind fluffily carefully regular realms.|
+2724|92|4|1|47|46628.23|0.09|0.01|A|F|1994-11-23|1994-11-13|1994-12-03|COLLECT COD|TRUCK|unusual patterns nag. special p|
+2724|147|8|2|21|21989.94|0.09|0.02|A|F|1994-11-25|1994-10-15|1994-12-07|COLLECT COD|RAIL|as. carefully regular dependencies wak|
+2724|50|3|3|22|20901.10|0.04|0.06|A|F|1994-09-19|1994-11-18|1994-10-17|TAKE BACK RETURN|TRUCK|express fo|
+2724|35|6|4|1|935.03|0.07|0.03|A|F|1994-12-26|1994-11-27|1995-01-07|NONE|MAIL|lyly carefully blithe theodolites-- pl|
+2724|149|2|5|29|30425.06|0.05|0.06|A|F|1995-01-10|1994-11-17|1995-02-04|COLLECT COD|MAIL|l requests hagg|
+2725|118|2|1|23|23416.53|0.10|0.08|R|F|1994-08-25|1994-06-22|1994-08-28|TAKE BACK RETURN|REG AIR|y regular deposits. brave foxes |
+2725|5|8|2|41|37105.00|0.01|0.00|R|F|1994-07-05|1994-06-29|1994-08-02|DELIVER IN PERSON|TRUCK|ns sleep furiously c|
+2725|189|10|3|15|16337.70|0.07|0.03|R|F|1994-08-06|1994-08-09|1994-08-15|TAKE BACK RETURN|AIR|? furiously regular a|
+2726|1|6|1|50|45050.00|0.00|0.06|R|F|1993-03-04|1993-01-29|1993-03-28|COLLECT COD|TRUCK| furiously bold theodolites|
+2727|151|6|1|3|3153.45|0.03|0.01|N|O|1998-06-18|1998-06-06|1998-06-23|NONE|RAIL| the carefully regular foxes u|
+2752|31|2|1|41|38172.23|0.02|0.05|A|F|1994-03-02|1994-01-31|1994-03-06|DELIVER IN PERSON|AIR|tructions hag|
+2752|7|2|2|29|26303.00|0.02|0.04|R|F|1994-01-22|1994-01-08|1994-01-28|COLLECT COD|TRUCK|gly blithely re|
+2752|56|7|3|4|3824.20|0.08|0.00|A|F|1993-12-14|1994-02-13|1994-01-05|DELIVER IN PERSON|TRUCK|telets haggle. regular, final |
+2752|24|7|4|40|36960.80|0.09|0.06|A|F|1994-01-24|1994-01-18|1994-02-22|DELIVER IN PERSON|MAIL|into beans are after the sly|
+2752|126|5|5|22|22574.64|0.03|0.04|A|F|1994-03-20|1994-02-08|1994-04-01|TAKE BACK RETURN|TRUCK|equests nag. regular dependencies are furio|
+2752|170|5|6|21|22473.57|0.09|0.05|R|F|1994-01-01|1994-01-24|1994-01-24|COLLECT COD|SHIP| along the quickly |
+2752|199|10|7|38|41769.22|0.08|0.00|R|F|1994-02-23|1993-12-23|1994-03-24|DELIVER IN PERSON|SHIP|es boost. slyly silent ideas|
+2753|13|3|1|6|5478.06|0.10|0.04|A|F|1993-12-30|1994-01-28|1994-01-29|COLLECT COD|TRUCK|s accounts|
+2753|48|7|2|40|37921.60|0.03|0.05|A|F|1994-01-06|1994-02-13|1994-02-03|DELIVER IN PERSON|SHIP|latelets kindle slyly final depos|
+2753|89|10|3|30|29672.40|0.00|0.07|A|F|1994-01-26|1994-01-29|1994-02-02|NONE|RAIL|ans wake fluffily blithely iro|
+2753|31|7|4|7|6517.21|0.07|0.03|R|F|1994-02-11|1994-01-22|1994-03-10|DELIVER IN PERSON|AIR|xpress ideas detect b|
+2753|137|8|5|36|37336.68|0.04|0.08|R|F|1994-03-15|1994-01-03|1994-04-03|DELIVER IN PERSON|SHIP|gle slyly final c|
+2753|50|1|6|17|16150.85|0.01|0.08|A|F|1994-03-08|1994-01-17|1994-03-11|TAKE BACK RETURN|REG AIR| carefully bold deposits sublate s|
+2753|148|9|7|20|20962.80|0.01|0.06|R|F|1994-02-24|1994-02-04|1994-03-23|DELIVER IN PERSON|FOB| express pack|
+2754|149|6|1|4|4196.56|0.05|0.08|A|F|1994-07-13|1994-05-15|1994-08-02|NONE|REG AIR|blithely silent requests. regular depo|
+2754|177|5|2|19|20466.23|0.01|0.07|A|F|1994-06-27|1994-05-06|1994-06-28|NONE|FOB|latelets hag|
+2755|92|4|1|19|18849.71|0.10|0.00|R|F|1992-02-11|1992-03-15|1992-02-14|TAKE BACK RETURN|MAIL|furiously special deposits|
+2755|24|3|2|11|10164.22|0.03|0.08|A|F|1992-04-12|1992-05-07|1992-04-21|COLLECT COD|RAIL|egular excuses sleep carefully.|
+2755|64|3|3|21|20245.26|0.08|0.04|R|F|1992-02-13|1992-04-20|1992-03-02|NONE|AIR|furious re|
+2755|131|7|4|5|5155.65|0.01|0.00|A|F|1992-02-27|1992-04-07|1992-03-09|TAKE BACK RETURN|AIR|e the furi|
+2755|116|7|5|48|48773.28|0.05|0.06|R|F|1992-03-22|1992-03-10|1992-04-14|DELIVER IN PERSON|MAIL|yly even epitaphs for the |
+2756|118|9|1|35|35633.85|0.03|0.02|R|F|1994-06-08|1994-06-01|1994-06-21|TAKE BACK RETURN|AIR| deposits grow bold sheaves; iro|
+2756|80|9|2|47|46063.76|0.06|0.01|R|F|1994-05-10|1994-05-25|1994-05-13|NONE|AIR|e final, f|
+2756|105|8|3|31|31158.10|0.01|0.07|A|F|1994-07-27|1994-07-06|1994-08-22|TAKE BACK RETURN|TRUCK|en instructions use quickly.|
+2756|72|2|4|30|29162.10|0.00|0.04|A|F|1994-06-05|1994-06-30|1994-06-14|DELIVER IN PERSON|TRUCK|ular packages. regular deposi|
+2757|148|5|1|26|27251.64|0.07|0.00|N|O|1995-08-19|1995-10-02|1995-09-06|DELIVER IN PERSON|MAIL|around the blithely|
+2757|22|7|2|12|11064.24|0.07|0.08|N|O|1995-08-01|1995-09-04|1995-08-08|TAKE BACK RETURN|SHIP| regular, eve|
+2757|73|3|3|17|16542.19|0.10|0.04|N|O|1995-09-06|1995-09-27|1995-09-22|DELIVER IN PERSON|AIR|er the furiously silent |
+2757|140|1|4|25|26003.50|0.08|0.01|N|O|1995-11-09|1995-09-12|1995-11-23|NONE|AIR|uickly regular |
+2757|70|7|5|14|13580.98|0.04|0.05|N|O|1995-09-01|1995-08-24|1995-09-03|TAKE BACK RETURN|SHIP|special deposits u|
+2758|121|10|1|20|20422.40|0.02|0.04|N|O|1998-07-27|1998-09-10|1998-08-21|TAKE BACK RETURN|AIR|ptotes sleep furiously|
+2758|23|8|2|17|15691.34|0.10|0.06|N|O|1998-09-25|1998-10-03|1998-10-25|NONE|MAIL| accounts! qui|
+2758|26|5|3|1|926.02|0.06|0.02|N|O|1998-10-09|1998-09-15|1998-10-16|NONE|TRUCK|ake furious|
+2759|59|1|1|10|9590.50|0.10|0.03|R|F|1993-12-14|1994-01-08|1994-01-01|COLLECT COD|FOB|s. busily ironic theodo|
+2759|113|10|2|37|37485.07|0.00|0.06|R|F|1994-03-05|1994-02-22|1994-03-18|DELIVER IN PERSON|REG AIR|lar Tiresias affix ironically carefully sp|
+2759|112|9|3|11|11133.21|0.03|0.08|A|F|1994-01-24|1994-01-16|1994-02-21|DELIVER IN PERSON|TRUCK|hely regular |
+2759|23|2|4|31|28613.62|0.02|0.05|A|F|1994-01-11|1994-01-15|1994-01-23|NONE|SHIP|ithely aft|
+2784|33|4|1|45|41986.35|0.03|0.01|N|O|1998-02-15|1998-04-07|1998-02-26|COLLECT COD|AIR|yly along the asymptotes. reque|
+2784|54|5|2|23|21943.15|0.03|0.05|N|O|1998-03-28|1998-02-07|1998-04-17|DELIVER IN PERSON|AIR|uests lose after |
+2784|175|4|3|40|43006.80|0.07|0.01|N|O|1998-04-28|1998-03-19|1998-05-03|DELIVER IN PERSON|TRUCK|deas nag furiously never unusual |
+2784|29|10|4|3|2787.06|0.04|0.03|N|O|1998-01-19|1998-04-05|1998-02-05|TAKE BACK RETURN|AIR|n packages. foxes haggle quickly sile|
+2785|100|3|1|34|34003.40|0.08|0.06|N|O|1995-08-07|1995-09-09|1995-09-05|NONE|RAIL|ly final packages haggl|
+2785|110|7|2|37|37374.07|0.08|0.04|N|O|1995-07-25|1995-09-12|1995-08-06|DELIVER IN PERSON|TRUCK|tructions. furiously |
+2785|65|10|3|33|31846.98|0.08|0.06|N|O|1995-10-16|1995-08-24|1995-11-02|DELIVER IN PERSON|MAIL|fter the furiously final p|
+2785|48|1|4|34|32233.36|0.00|0.02|N|O|1995-09-16|1995-09-09|1995-10-11|COLLECT COD|SHIP|kages wake carefully silent |
+2786|136|2|1|15|15541.95|0.03|0.04|A|F|1992-05-19|1992-05-08|1992-05-28|COLLECT COD|TRUCK|low deposits are ironic|
+2786|51|3|2|42|39944.10|0.10|0.04|R|F|1992-05-15|1992-04-22|1992-05-30|DELIVER IN PERSON|AIR|unts are against the furious|
+2786|156|1|3|41|43302.15|0.04|0.05|R|F|1992-07-01|1992-06-04|1992-07-13|COLLECT COD|RAIL|ix requests. bold requests a|
+2786|23|4|4|24|22152.48|0.05|0.02|A|F|1992-04-04|1992-06-09|1992-05-02|DELIVER IN PERSON|MAIL|ans. slyly unusual platelets detect. unus|
+2786|50|3|5|43|40852.15|0.06|0.03|R|F|1992-04-22|1992-05-13|1992-04-29|NONE|RAIL|ons. theodolites after|
+2786|162|1|6|21|22305.36|0.08|0.00|A|F|1992-05-03|1992-05-01|1992-05-14|COLLECT COD|AIR|slow instructi|
+2787|33|9|1|4|3732.12|0.04|0.04|N|O|1996-01-26|1995-11-26|1996-02-20|TAKE BACK RETURN|SHIP|ts. instructions nag furiously according |
+2788|177|8|1|16|17234.72|0.06|0.06|A|F|1994-10-04|1994-11-25|1994-10-18|DELIVER IN PERSON|AIR| requests wake carefully. carefully si|
+2789|163|8|1|16|17010.56|0.03|0.02|N|O|1998-04-18|1998-05-25|1998-05-12|DELIVER IN PERSON|REG AIR|o beans use carefully|
+2789|23|4|2|41|37843.82|0.02|0.05|N|O|1998-03-20|1998-05-15|1998-03-21|COLLECT COD|MAIL|d packages-- fluffily specia|
+2789|176|5|3|33|35513.61|0.06|0.02|N|O|1998-04-21|1998-05-02|1998-04-30|COLLECT COD|TRUCK|deposits. ironic |
+2789|16|3|4|47|43052.47|0.02|0.04|N|O|1998-03-29|1998-05-05|1998-04-07|NONE|RAIL|usly busy packages wake against the unusual|
+2789|197|1|5|23|25235.37|0.02|0.07|N|O|1998-03-25|1998-05-10|1998-04-24|COLLECT COD|RAIL|cording to the careful de|
+2789|144|5|6|16|16706.24|0.07|0.03|N|O|1998-05-11|1998-05-08|1998-05-24|TAKE BACK RETURN|RAIL|d the carefully iron|
+2789|133|4|7|42|43391.46|0.01|0.00|N|O|1998-04-28|1998-05-17|1998-05-24|TAKE BACK RETURN|AIR|ending packages shoul|
+2790|185|6|1|27|29299.86|0.06|0.08|R|F|1994-09-04|1994-09-27|1994-09-16|TAKE BACK RETURN|MAIL|ilent packages cajole. quickly ironic requ|
+2790|117|1|2|50|50855.50|0.00|0.06|A|F|1994-12-08|1994-11-17|1994-12-19|NONE|RAIL|fter the regular ideas. f|
+2790|184|5|3|19|20599.42|0.06|0.00|R|F|1994-10-23|1994-10-03|1994-10-26|TAKE BACK RETURN|RAIL|uffily even excuses. furiously thin|
+2790|197|8|4|24|26332.56|0.07|0.01|A|F|1994-12-04|1994-10-10|1994-12-25|NONE|MAIL|ments. slyly f|
+2790|148|9|5|11|11529.54|0.08|0.03|A|F|1994-09-28|1994-11-14|1994-10-04|TAKE BACK RETURN|AIR|lar requests poach slyly foxes|
+2790|73|3|6|13|12649.91|0.08|0.00|R|F|1994-09-20|1994-10-10|1994-10-20|COLLECT COD|SHIP|n deposits according to the regul|
+2790|4|1|7|32|28928.00|0.08|0.02|A|F|1994-09-25|1994-10-26|1994-10-01|NONE|SHIP|ully pending|
+2791|59|10|1|49|46993.45|0.10|0.04|A|F|1995-01-11|1994-11-10|1995-02-08|COLLECT COD|MAIL| accounts sleep at the bold, regular pinto |
+2791|63|4|2|4|3852.24|0.10|0.08|A|F|1995-01-02|1994-12-28|1995-01-29|NONE|SHIP|slyly bold packages boost. slyly|
+2791|133|9|3|44|45457.72|0.08|0.06|R|F|1994-11-17|1994-11-12|1994-12-14|NONE|FOB|heodolites use furio|
+2791|156|8|4|24|25347.60|0.04|0.02|R|F|1995-01-30|1994-11-20|1995-02-08|DELIVER IN PERSON|TRUCK|ilent forges. quickly special pinto beans |
+2791|105|2|5|8|8040.80|0.02|0.04|R|F|1995-01-30|1994-11-24|1995-02-13|NONE|FOB|se. close ideas alongs|
+2791|75|3|6|9|8775.63|0.08|0.02|R|F|1994-11-19|1994-12-14|1994-12-10|TAKE BACK RETURN|AIR|pendencies. blithely bold patterns acr|
+2791|29|2|7|26|24154.52|0.06|0.03|R|F|1995-02-06|1994-12-07|1995-02-23|DELIVER IN PERSON|AIR|uriously special instructio|
+2816|59|10|1|33|31648.65|0.00|0.07|R|F|1994-10-19|1994-11-10|1994-11-09|NONE|REG AIR|s; slyly even theodo|
+2816|142|3|2|4|4168.56|0.05|0.04|R|F|1994-12-11|1994-12-07|1995-01-03|NONE|FOB|. blithely pending id|
+2816|121|6|3|4|4084.48|0.02|0.06|R|F|1994-12-12|1994-12-05|1994-12-30|NONE|RAIL| requests print above the final deposits|
+2817|60|8|1|25|24001.50|0.07|0.01|R|F|1994-04-21|1994-06-20|1994-05-07|DELIVER IN PERSON|FOB|doze blithely.|
+2817|32|8|2|5|4660.15|0.03|0.04|A|F|1994-05-07|1994-05-31|1994-05-12|TAKE BACK RETURN|AIR|furiously unusual theodolites use furiou|
+2817|172|10|3|35|37525.95|0.01|0.07|A|F|1994-05-20|1994-06-03|1994-05-22|COLLECT COD|FOB|gular foxes|
+2817|161|2|4|4|4244.64|0.00|0.05|R|F|1994-06-04|1994-06-11|1994-06-10|NONE|TRUCK|n accounts wake across the fluf|
+2818|121|4|1|12|12253.44|0.10|0.03|A|F|1995-02-01|1995-03-10|1995-02-16|NONE|AIR|lms. quickly bold asymp|
+2818|199|2|2|22|24182.18|0.06|0.07|R|F|1995-02-28|1995-03-10|1995-03-06|TAKE BACK RETURN|RAIL|egrate toward the carefully iron|
+2818|45|6|3|11|10395.44|0.01|0.06|R|F|1995-02-18|1995-02-11|1995-03-19|TAKE BACK RETURN|TRUCK|ggle across the carefully blithe|
+2818|40|6|4|32|30081.28|0.08|0.08|R|F|1995-02-04|1995-03-05|1995-02-18|COLLECT COD|REG AIR|arefully! ac|
+2818|18|8|5|42|38556.42|0.08|0.04|A|F|1995-02-12|1995-02-19|1995-03-13|COLLECT COD|MAIL|ar accounts wake carefully a|
+2818|91|5|6|7|6937.63|0.06|0.03|R|F|1995-03-24|1995-03-09|1995-04-06|TAKE BACK RETURN|TRUCK|ly according to the r|
+2819|70|1|1|17|16491.19|0.08|0.08|A|F|1994-07-16|1994-07-15|1994-07-17|TAKE BACK RETURN|RAIL|en deposits above the f|
+2819|67|2|2|12|11604.72|0.03|0.08|R|F|1994-07-18|1994-06-24|1994-07-28|NONE|MAIL| regular, regular a|
+2819|5|2|3|28|25340.00|0.03|0.08|R|F|1994-05-09|1994-07-02|1994-05-15|NONE|RAIL|ckages sublate carefully closely regular |
+2819|153|4|4|5|5265.75|0.00|0.02|R|F|1994-05-29|1994-06-12|1994-06-28|NONE|TRUCK| fluffily unusual foxes sleep caref|
+2819|200|3|5|6|6601.20|0.03|0.01|A|F|1994-07-22|1994-08-02|1994-07-29|NONE|REG AIR|eas after the carefully express pack|
+2820|174|2|1|23|24705.91|0.04|0.08|R|F|1994-07-10|1994-08-08|1994-07-21|NONE|MAIL| was furiously. deposits among the ironic|
+2820|126|9|2|33|33861.96|0.08|0.06|A|F|1994-07-07|1994-08-17|1994-08-02|DELIVER IN PERSON|AIR|carefully even pinto beans. |
+2820|141|10|3|38|39563.32|0.03|0.08|A|F|1994-09-10|1994-08-07|1994-10-07|TAKE BACK RETURN|MAIL|ests despite the carefully unusual a|
+2820|197|9|4|40|43887.60|0.06|0.06|A|F|1994-08-08|1994-07-30|1994-08-21|TAKE BACK RETURN|REG AIR|g multipliers. final c|
+2821|181|2|1|4|4324.72|0.00|0.00|A|F|1993-09-15|1993-10-02|1993-09-17|TAKE BACK RETURN|TRUCK|nding foxes.|
+2821|72|1|2|4|3888.28|0.09|0.00|A|F|1993-11-19|1993-09-20|1993-11-27|TAKE BACK RETURN|TRUCK|ual multipliers. final deposits cajol|
+2821|164|1|3|27|28732.32|0.01|0.01|A|F|1993-11-27|1993-10-11|1993-12-08|COLLECT COD|TRUCK|requests. blit|
+2822|151|9|1|39|40994.85|0.04|0.02|R|F|1993-09-11|1993-08-29|1993-09-18|NONE|MAIL|kly about the sly|
+2823|86|7|1|45|44373.60|0.03|0.04|N|O|1995-12-28|1995-11-27|1996-01-02|DELIVER IN PERSON|SHIP|furiously special idea|
+2823|160|5|2|18|19082.88|0.00|0.03|N|O|1995-11-11|1995-10-30|1995-12-08|TAKE BACK RETURN|TRUCK| final deposits. furiously regular foxes u|
+2823|186|7|3|11|11947.98|0.07|0.02|N|O|1995-12-10|1995-11-24|1995-12-21|DELIVER IN PERSON|SHIP|bold requests nag blithely s|
+2823|139|10|4|48|49878.24|0.09|0.03|N|O|1995-11-21|1995-10-30|1995-11-27|NONE|SHIP|ously busily slow excus|
+2823|99|2|5|18|17983.62|0.04|0.06|N|O|1995-11-09|1995-10-30|1995-11-19|NONE|AIR|eas. decoys cajole deposi|
+2823|123|2|6|20|20462.40|0.07|0.00|N|O|1995-11-13|1995-12-06|1995-12-07|NONE|MAIL|its sleep between the unusual, ironic pac|
+2823|86|7|7|12|11832.96|0.02|0.04|N|O|1995-12-22|1995-11-20|1996-01-13|NONE|REG AIR|the slyly ironic dolphins; fin|
+2848|65|4|1|44|42462.64|0.01|0.05|R|F|1992-04-14|1992-05-09|1992-04-19|DELIVER IN PERSON|MAIL|ions. slyly express instructions n|
+2848|165|6|2|8|8521.28|0.07|0.01|A|F|1992-03-21|1992-05-18|1992-04-07|DELIVER IN PERSON|TRUCK|. silent, final ideas sublate packages. ir|
+2848|138|4|3|8|8305.04|0.07|0.08|A|F|1992-06-20|1992-04-12|1992-07-09|NONE|SHIP|sly regular foxes. |
+2848|125|6|4|34|34854.08|0.02|0.08|A|F|1992-03-15|1992-04-24|1992-04-12|TAKE BACK RETURN|RAIL|ts along the blithely regu|
+2848|195|7|5|18|19713.42|0.07|0.03|R|F|1992-04-10|1992-06-01|1992-05-05|DELIVER IN PERSON|TRUCK|osits haggle. stealthily ironic packa|
+2849|154|2|1|16|16866.40|0.09|0.08|N|O|1996-05-20|1996-07-23|1996-06-18|NONE|TRUCK|. furiously regular requ|
+2849|187|8|2|39|42400.02|0.10|0.03|N|O|1996-05-22|1996-07-18|1996-06-05|TAKE BACK RETURN|SHIP|s sleep furiously silently regul|
+2849|60|1|3|24|23041.44|0.01|0.05|N|O|1996-06-12|1996-07-10|1996-06-27|TAKE BACK RETURN|AIR|e slyly even asymptotes. slo|
+2849|55|7|4|48|45842.40|0.05|0.02|N|O|1996-05-03|1996-06-05|1996-05-28|NONE|AIR|mong the carefully regular theodol|
+2849|28|7|5|30|27840.60|0.10|0.06|N|O|1996-08-24|1996-07-08|1996-09-03|TAKE BACK RETURN|SHIP|ly. carefully silent|
+2849|69|4|6|30|29071.80|0.06|0.07|N|O|1996-06-20|1996-07-23|1996-07-06|NONE|FOB|yly furiously even id|
+2850|97|1|1|43|42874.87|0.02|0.05|N|O|1997-01-11|1996-11-03|1997-02-01|COLLECT COD|REG AIR|unusual accounts|
+2850|110|7|2|30|30303.30|0.09|0.01|N|O|1996-12-14|1996-11-29|1997-01-03|COLLECT COD|AIR|even ideas. busy pinto beans sleep above t|
+2850|105|6|3|49|49249.90|0.09|0.04|N|O|1996-10-07|1996-12-12|1996-10-12|TAKE BACK RETURN|MAIL| slyly unusual req|
+2850|199|3|4|4|4396.76|0.04|0.04|N|O|1996-10-28|1996-12-26|1996-11-07|COLLECT COD|RAIL|al deposits cajole carefully quickly |
+2851|148|5|1|8|8385.12|0.09|0.03|N|O|1997-11-12|1997-11-22|1997-12-11|NONE|REG AIR|y special theodolites. carefully|
+2852|177|6|1|6|6463.02|0.01|0.01|R|F|1993-03-02|1993-04-11|1993-03-11|TAKE BACK RETURN|RAIL| accounts above the furiously un|
+2852|41|10|2|24|22584.96|0.05|0.07|R|F|1993-01-18|1993-03-13|1993-02-14|DELIVER IN PERSON|MAIL| the blithe|
+2852|164|9|3|29|30860.64|0.09|0.05|R|F|1993-04-21|1993-03-22|1993-05-02|COLLECT COD|SHIP|lyly ironi|
+2852|100|3|4|12|12001.20|0.08|0.02|A|F|1993-02-25|1993-03-24|1993-03-07|TAKE BACK RETURN|TRUCK|le. request|
+2852|154|2|5|28|29516.20|0.05|0.03|R|F|1993-02-08|1993-03-30|1993-02-11|NONE|MAIL|e accounts. caref|
+2853|139|5|1|14|14547.82|0.07|0.05|R|F|1994-05-16|1994-07-01|1994-05-27|NONE|TRUCK|oach slyly along t|
+2853|134|10|2|26|26887.38|0.06|0.01|R|F|1994-06-26|1994-06-05|1994-07-02|TAKE BACK RETURN|MAIL|dolphins wake slyly. blith|
+2853|173|3|3|40|42926.80|0.06|0.04|A|F|1994-08-06|1994-06-24|1994-08-29|NONE|RAIL|lyly. pearls cajole. final accounts ca|
+2853|132|8|4|20|20642.60|0.02|0.04|A|F|1994-08-30|1994-06-16|1994-09-06|TAKE BACK RETURN|TRUCK|e slyly silent foxes. express deposits sno|
+2853|36|7|5|1|936.03|0.08|0.05|R|F|1994-09-01|1994-06-27|1994-09-12|TAKE BACK RETURN|FOB|refully slyly quick packages. final c|
+2854|181|2|1|46|49734.28|0.00|0.04|A|F|1994-09-22|1994-08-02|1994-09-30|COLLECT COD|AIR|. furiously regular deposits across th|
+2854|88|9|2|29|28654.32|0.09|0.07|R|F|1994-07-06|1994-08-26|1994-07-09|COLLECT COD|SHIP|y slyly ironic accounts. foxes haggle slyl|
+2854|160|8|3|20|21203.20|0.08|0.01|R|F|1994-09-18|1994-08-03|1994-10-12|COLLECT COD|AIR|rs impress after the deposits. |
+2854|170|1|4|34|36385.78|0.06|0.03|A|F|1994-09-06|1994-08-07|1994-09-22|NONE|REG AIR|age carefully|
+2854|102|3|5|7|7014.70|0.03|0.06|A|F|1994-09-23|1994-08-14|1994-10-10|DELIVER IN PERSON|REG AIR| the pending|
+2854|18|2|6|13|11934.13|0.04|0.03|R|F|1994-09-15|1994-08-18|1994-09-19|DELIVER IN PERSON|SHIP| excuses wak|
+2855|33|4|1|50|46651.50|0.03|0.07|A|F|1993-05-20|1993-06-28|1993-06-16|TAKE BACK RETURN|TRUCK|beans. deposits |
+2880|35|6|1|40|37401.20|0.09|0.00|A|F|1992-05-26|1992-06-01|1992-05-31|COLLECT COD|TRUCK|even requests. quick|
+2880|139|5|2|26|27017.38|0.07|0.07|R|F|1992-04-12|1992-04-15|1992-04-28|NONE|RAIL|ully among the regular warthogs|
+2880|115|9|3|42|42634.62|0.01|0.01|R|F|1992-06-17|1992-05-29|1992-07-11|NONE|REG AIR|ions. carefully final accounts are unusual,|
+2880|18|2|4|46|42228.46|0.02|0.02|A|F|1992-04-21|1992-06-05|1992-05-16|COLLECT COD|RAIL|eep quickly according to t|
+2881|180|10|1|16|17282.88|0.02|0.06|A|F|1992-06-21|1992-06-27|1992-07-03|TAKE BACK RETURN|TRUCK|usly bold |
+2881|10|1|2|1|910.01|0.09|0.03|A|F|1992-05-13|1992-07-21|1992-05-18|COLLECT COD|MAIL|final theodolites. quickly|
+2881|93|6|3|21|20854.89|0.07|0.03|A|F|1992-05-28|1992-07-03|1992-06-02|TAKE BACK RETURN|SHIP|hely express Tiresias. final dependencies |
+2881|140|6|4|7|7280.98|0.06|0.01|R|F|1992-08-03|1992-07-10|1992-08-27|NONE|REG AIR|ironic packages are carefully final ac|
+2882|4|7|1|14|12656.00|0.09|0.02|N|O|1995-09-28|1995-11-11|1995-10-18|TAKE BACK RETURN|MAIL|kly. even requests w|
+2882|42|1|2|30|28261.20|0.00|0.00|N|O|1995-10-15|1995-10-13|1995-10-25|NONE|REG AIR|among the furiously even theodolites. regu|
+2882|197|9|3|29|31818.51|0.10|0.08|N|O|1995-09-10|1995-11-01|1995-10-02|NONE|TRUCK|kages. furiously ironic|
+2882|78|6|4|27|26407.89|0.06|0.02|N|O|1995-09-04|1995-11-11|1995-09-12|DELIVER IN PERSON|MAIL|rding to the regu|
+2882|134|5|5|32|33092.16|0.07|0.03|N|O|1995-10-21|1995-11-10|1995-11-01|COLLECT COD|RAIL|sts. quickly regular e|
+2882|87|8|6|47|46392.76|0.06|0.03|N|O|1995-09-13|1995-09-21|1995-09-14|NONE|REG AIR|l, special|
+2883|1|4|1|33|29733.00|0.08|0.07|R|F|1995-02-26|1995-03-04|1995-03-01|NONE|RAIL|s. final i|
+2883|125|6|2|27|27678.24|0.00|0.02|A|F|1995-03-12|1995-03-10|1995-04-04|TAKE BACK RETURN|REG AIR|s. brave pinto beans nag furiously|
+2883|189|10|3|47|51191.46|0.05|0.04|R|F|1995-01-29|1995-04-19|1995-02-05|DELIVER IN PERSON|SHIP|ep carefully ironic|
+2883|98|2|4|23|22956.07|0.00|0.02|R|F|1995-02-03|1995-03-17|1995-02-19|TAKE BACK RETURN|AIR| even requests cajole. special, regular |
+2883|195|8|5|36|39426.84|0.07|0.06|A|F|1995-05-02|1995-03-14|1995-05-30|COLLECT COD|MAIL|ests detect slyly special packages|
+2884|71|2|1|41|39813.87|0.03|0.00|N|O|1998-01-02|1997-12-17|1998-01-20|DELIVER IN PERSON|TRUCK|ep. slyly even accounts a|
+2884|146|5|2|25|26153.50|0.09|0.08|N|O|1998-01-18|1997-12-06|1998-02-16|TAKE BACK RETURN|MAIL|onic theodolites with the instructi|
+2884|26|7|3|8|7408.16|0.08|0.08|N|O|1997-11-30|1997-11-28|1997-12-14|COLLECT COD|TRUCK|pending accounts about |
+2885|4|9|1|6|5424.00|0.10|0.01|A|F|1993-01-05|1992-12-12|1993-01-19|COLLECT COD|FOB|ctions solve. slyly regular requests n|
+2885|112|3|2|4|4048.44|0.07|0.00|A|F|1992-10-09|1992-12-17|1992-11-04|TAKE BACK RETURN|SHIP| pending packages wake. |
+2885|1|6|3|45|40545.00|0.10|0.04|A|F|1992-12-24|1992-10-30|1993-01-04|NONE|SHIP|ess ideas. regular, silen|
+2885|32|3|4|15|13980.45|0.03|0.04|R|F|1992-10-31|1992-11-24|1992-11-21|DELIVER IN PERSON|MAIL|odolites. boldly pending packages han|
+2885|175|5|5|43|46232.31|0.06|0.00|R|F|1992-11-17|1992-10-30|1992-12-04|DELIVER IN PERSON|SHIP|cial deposits use bold|
+2885|190|1|6|5|5450.95|0.01|0.02|R|F|1993-01-06|1992-11-13|1993-02-05|TAKE BACK RETURN|TRUCK|s. slyly express th|
+2885|50|9|7|40|38002.00|0.05|0.03|A|F|1992-09-23|1992-11-15|1992-10-07|TAKE BACK RETURN|AIR| express depos|
+2886|60|1|1|1|960.06|0.09|0.05|A|F|1995-02-01|1994-12-18|1995-02-28|COLLECT COD|REG AIR|eposits fr|
+2886|184|5|2|38|41198.84|0.02|0.04|A|F|1995-01-21|1995-01-08|1995-01-30|NONE|SHIP|old requests along the fur|
+2886|63|8|3|2|1926.12|0.04|0.07|A|F|1994-11-18|1995-01-31|1994-12-05|COLLECT COD|REG AIR|ar theodolites. e|
+2886|130|3|4|46|47385.98|0.03|0.08|A|F|1995-02-02|1995-01-26|1995-02-15|TAKE BACK RETURN|SHIP|ously final packages sleep blithely regular|
+2887|66|3|1|11|10626.66|0.06|0.00|N|O|1997-07-08|1997-07-17|1997-07-15|COLLECT COD|SHIP|ackages. unusual, speci|
+2887|112|6|2|17|17205.87|0.00|0.08|N|O|1997-08-31|1997-07-04|1997-09-17|DELIVER IN PERSON|SHIP|fily final packages. regula|
+2912|122|1|1|8|8176.96|0.06|0.04|A|F|1992-04-09|1992-04-19|1992-04-26|NONE|RAIL|hs cajole over the slyl|
+2912|115|9|2|18|18271.98|0.00|0.08|R|F|1992-03-13|1992-04-19|1992-03-30|TAKE BACK RETURN|RAIL|unts cajole reg|
+2913|123|6|1|39|39901.68|0.06|0.04|N|O|1997-08-28|1997-09-27|1997-09-02|TAKE BACK RETURN|AIR|. final packages a|
+2913|22|5|2|22|20284.44|0.10|0.07|N|O|1997-09-18|1997-08-11|1997-10-02|COLLECT COD|MAIL|riously pending realms. blithely even pac|
+2913|166|1|3|17|18124.72|0.07|0.04|N|O|1997-10-21|1997-09-25|1997-11-20|NONE|FOB|requests doze quickly. furious|
+2913|143|4|4|5|5215.70|0.10|0.07|N|O|1997-10-07|1997-08-25|1997-10-09|TAKE BACK RETURN|RAIL|haggle. even, bold instructi|
+2913|15|9|5|13|11895.13|0.03|0.01|N|O|1997-10-02|1997-08-20|1997-10-26|COLLECT COD|MAIL|inos are carefully alongside of the bol|
+2913|168|5|6|35|37385.60|0.06|0.08|N|O|1997-08-30|1997-08-21|1997-09-03|COLLECT COD|MAIL|es. quickly even braids against|
+2914|66|7|1|22|21253.32|0.05|0.06|R|F|1993-05-11|1993-04-09|1993-05-22|DELIVER IN PERSON|FOB| carefully about the fluffily ironic gifts|
+2914|163|10|2|25|26579.00|0.03|0.04|A|F|1993-05-14|1993-04-04|1993-05-22|NONE|SHIP|cross the carefully even accounts.|
+2914|35|1|3|4|3740.12|0.00|0.05|R|F|1993-06-11|1993-04-09|1993-06-14|TAKE BACK RETURN|SHIP|s integrate. bold deposits sleep req|
+2914|121|2|4|9|9190.08|0.06|0.01|R|F|1993-06-17|1993-05-26|1993-06-19|NONE|REG AIR|s. carefully final foxes ar|
+2915|175|5|1|28|30104.76|0.10|0.02|R|F|1994-04-17|1994-06-09|1994-05-10|NONE|MAIL|yly special |
+2915|94|7|2|12|11929.08|0.00|0.03|A|F|1994-07-18|1994-06-11|1994-07-27|TAKE BACK RETURN|RAIL|accounts. slyly final|
+2915|136|2|3|15|15541.95|0.07|0.00|A|F|1994-05-01|1994-06-12|1994-05-15|DELIVER IN PERSON|TRUCK|al requests haggle furiousl|
+2915|81|2|4|43|42186.44|0.06|0.05|R|F|1994-06-02|1994-05-24|1994-06-06|DELIVER IN PERSON|SHIP|into beans dazzle alongside of|
+2916|83|4|1|21|20644.68|0.06|0.04|N|O|1996-03-11|1996-02-21|1996-03-30|NONE|REG AIR|uickly express ideas over the slyly even |
+2917|93|4|1|36|35751.24|0.10|0.01|N|O|1998-04-07|1998-02-23|1998-05-01|DELIVER IN PERSON|RAIL|usly ironic d|
+2917|21|2|2|20|18420.40|0.06|0.03|N|O|1997-12-31|1998-01-22|1998-01-12|NONE|MAIL|slyly even ideas wa|
+2917|90|1|3|4|3960.36|0.02|0.07|N|O|1998-01-10|1998-01-18|1998-02-08|TAKE BACK RETURN|REG AIR|s. unusual instruct|
+2917|167|2|4|5|5335.80|0.05|0.01|N|O|1997-12-16|1998-01-26|1998-01-07|NONE|RAIL|bove the furiously silent packages. pend|
+2917|41|10|5|37|34818.48|0.04|0.01|N|O|1997-12-12|1998-02-03|1997-12-23|COLLECT COD|RAIL|dependencies. express |
+2917|194|8|6|7|7659.33|0.05|0.01|N|O|1998-03-21|1998-03-03|1998-03-25|NONE|REG AIR|ly about the regular accounts. carefully pe|
+2918|78|7|1|24|23473.68|0.10|0.03|N|O|1996-12-20|1996-10-28|1996-12-26|DELIVER IN PERSON|FOB| quickly. express requests haggle careful|
+2919|102|5|1|2|2004.20|0.03|0.05|R|F|1993-12-28|1994-02-23|1994-01-18|COLLECT COD|TRUCK|re slyly. regular ideas detect furiousl|
+2919|121|4|2|49|50034.88|0.07|0.02|R|F|1993-12-16|1994-02-28|1993-12-19|COLLECT COD|FOB|hely final inst|
+2919|46|5|3|44|41625.76|0.07|0.07|A|F|1994-04-01|1994-01-12|1994-04-07|TAKE BACK RETURN|TRUCK|final ideas haggle carefully fluff|
+2919|102|5|4|44|44092.40|0.00|0.05|R|F|1994-02-04|1994-02-03|1994-03-02|TAKE BACK RETURN|AIR|es doze around the furiously |
+2944|120|1|1|44|44885.28|0.08|0.05|N|O|1997-12-25|1997-10-28|1998-01-21|COLLECT COD|AIR|ickly special theodolit|
+2944|42|9|2|44|41449.76|0.06|0.02|N|O|1997-10-28|1997-11-22|1997-11-10|NONE|SHIP|ickly. regular requests haggle. idea|
+2944|170|5|3|2|2140.34|0.06|0.07|N|O|1997-12-13|1997-12-01|1998-01-08|DELIVER IN PERSON|REG AIR|luffily expr|
+2944|17|7|4|23|21091.23|0.02|0.03|N|O|1998-01-12|1997-12-03|1998-01-17|TAKE BACK RETURN|MAIL| excuses? regular platelets e|
+2944|75|4|5|18|17551.26|0.10|0.01|N|O|1998-01-07|1997-10-26|1998-01-27|TAKE BACK RETURN|FOB| furiously slyl|
+2944|60|2|6|17|16321.02|0.00|0.03|N|O|1997-10-18|1997-11-27|1997-10-29|TAKE BACK RETURN|SHIP|slyly final dolphins sleep silent the|
+2944|90|1|7|7|6930.63|0.01|0.06|N|O|1997-10-30|1997-11-03|1997-11-03|DELIVER IN PERSON|FOB|fluffily blithely express pea|
+2945|59|10|1|37|35484.85|0.00|0.02|N|O|1996-02-10|1996-03-20|1996-02-12|COLLECT COD|SHIP|l instructions. regular, regular |
+2945|72|2|2|30|29162.10|0.05|0.01|N|O|1996-01-19|1996-02-11|1996-01-26|NONE|TRUCK|ular instructions|
+2945|127|8|3|28|28759.36|0.06|0.02|N|O|1996-03-17|1996-03-13|1996-04-15|COLLECT COD|FOB|le slyly along the eve|
+2945|188|9|4|34|36998.12|0.08|0.06|N|O|1996-02-03|1996-03-17|1996-02-29|COLLECT COD|REG AIR|at the unusual theodolite|
+2945|173|1|5|10|10731.70|0.09|0.05|N|O|1996-03-13|1996-03-10|1996-04-06|COLLECT COD|FOB|thely. final courts could hang qu|
+2945|97|9|6|45|44869.05|0.07|0.00|N|O|1996-03-01|1996-03-25|1996-03-08|TAKE BACK RETURN|MAIL|ainst the final packages|
+2945|52|10|7|47|44746.35|0.07|0.05|N|O|1996-01-05|1996-02-11|1996-01-12|DELIVER IN PERSON|MAIL|quests use|
+2946|10|5|1|25|22750.25|0.05|0.02|N|O|1996-05-06|1996-04-23|1996-05-16|DELIVER IN PERSON|SHIP|ic deposits. furiously|
+2946|94|5|2|48|47716.32|0.03|0.07|N|O|1996-06-02|1996-03-31|1996-06-16|COLLECT COD|TRUCK|oss the platelets. furi|
+2946|3|6|3|35|31605.00|0.03|0.00|N|O|1996-03-15|1996-04-02|1996-03-26|NONE|REG AIR| sublate along the fluffily iron|
+2947|10|1|1|37|33670.37|0.09|0.07|N|O|1995-08-09|1995-07-05|1995-08-20|DELIVER IN PERSON|RAIL|e accounts: expres|
+2947|186|7|2|10|10861.80|0.09|0.07|A|F|1995-06-07|1995-06-26|1995-06-08|NONE|MAIL|lly special |
+2948|118|9|1|48|48869.28|0.00|0.04|R|F|1994-08-29|1994-10-23|1994-09-23|NONE|TRUCK|unusual excuses use about the |
+2948|92|3|2|49|48612.41|0.04|0.07|R|F|1994-12-16|1994-11-08|1995-01-07|DELIVER IN PERSON|MAIL|ress requests. furiously blithe foxes |
+2949|21|6|1|4|3684.08|0.06|0.06|A|F|1994-06-07|1994-06-17|1994-07-04|TAKE BACK RETURN|REG AIR|gular pinto beans wake alongside of the reg|
+2949|70|5|2|50|48503.50|0.05|0.04|A|F|1994-08-04|1994-06-23|1994-08-17|TAKE BACK RETURN|FOB|gular courts cajole across t|
+2949|180|9|3|38|41046.84|0.02|0.06|R|F|1994-05-22|1994-05-25|1994-05-27|COLLECT COD|REG AIR|se slyly requests. carefull|
+2950|130|1|1|32|32964.16|0.01|0.05|N|O|1997-09-21|1997-08-25|1997-10-08|DELIVER IN PERSON|REG AIR|its wake carefully slyly final ideas.|
+2950|66|7|2|18|17389.08|0.10|0.01|N|O|1997-07-19|1997-08-29|1997-08-17|COLLECT COD|TRUCK|uests cajole furio|
+2950|53|4|3|14|13342.70|0.01|0.02|N|O|1997-07-29|1997-08-05|1997-07-31|TAKE BACK RETURN|MAIL|ccounts haggle carefully according |
+2950|187|8|4|45|48923.10|0.08|0.00|N|O|1997-09-05|1997-09-23|1997-09-11|NONE|FOB|ides the b|
+2950|61|2|5|46|44208.76|0.02|0.05|N|O|1997-07-15|1997-09-30|1997-07-25|COLLECT COD|RAIL|to the regular accounts are slyly carefu|
+2950|174|5|6|27|29002.59|0.01|0.03|N|O|1997-10-01|1997-09-13|1997-10-08|NONE|TRUCK|are alongside of the carefully silent |
+2951|3|8|1|5|4515.00|0.03|0.03|N|O|1996-03-27|1996-04-16|1996-03-30|NONE|REG AIR|to beans wake ac|
+2951|136|2|2|24|24867.12|0.07|0.03|N|O|1996-03-24|1996-04-16|1996-04-08|NONE|SHIP| ironic multipliers. express, regular|
+2951|187|8|3|40|43487.20|0.02|0.07|N|O|1996-05-03|1996-04-20|1996-05-22|COLLECT COD|REG AIR|ial deposits wake fluffily about th|
+2951|73|3|4|21|20434.47|0.06|0.08|N|O|1996-04-12|1996-04-27|1996-04-14|DELIVER IN PERSON|REG AIR|nt instructions toward the f|
+2951|51|6|5|15|14265.75|0.07|0.00|N|O|1996-03-25|1996-04-23|1996-03-27|COLLECT COD|REG AIR|inal account|
+2951|138|4|6|18|18686.34|0.06|0.00|N|O|1996-04-04|1996-04-27|1996-04-06|COLLECT COD|FOB|ep about the final, even package|
+2976|9|4|1|32|29088.00|0.06|0.00|A|F|1994-01-26|1994-02-13|1994-02-10|NONE|MAIL|nding, ironic deposits sleep f|
+2976|4|5|2|24|21696.00|0.00|0.03|A|F|1994-03-19|1994-01-26|1994-04-18|COLLECT COD|TRUCK|ronic pinto beans. slyly bol|
+2976|10|5|3|35|31850.35|0.10|0.07|R|F|1993-12-19|1994-02-14|1994-01-11|NONE|RAIL|boost slyly about the regular, regular re|
+2976|82|3|4|22|21605.76|0.00|0.04|A|F|1994-02-08|1994-03-03|1994-02-12|TAKE BACK RETURN|FOB|ncies kindle furiously. carefull|
+2976|134|5|5|13|13443.69|0.00|0.06|A|F|1994-02-06|1994-02-02|1994-02-19|NONE|FOB| furiously final courts boost |
+2976|109|2|6|30|30273.00|0.08|0.03|R|F|1994-03-27|1994-02-01|1994-04-26|TAKE BACK RETURN|RAIL|c ideas! unusual|
+2977|70|5|1|25|24251.75|0.03|0.07|N|O|1996-09-21|1996-10-06|1996-10-13|TAKE BACK RETURN|RAIL|furiously pe|
+2978|90|1|1|29|28712.61|0.00|0.08|A|F|1995-06-03|1995-07-25|1995-06-06|NONE|SHIP|ecial ideas promise slyly|
+2978|127|2|2|42|43139.04|0.01|0.06|N|O|1995-08-19|1995-07-18|1995-09-07|DELIVER IN PERSON|MAIL|ial requests nag blithely alongside of th|
+2978|43|2|3|26|24519.04|0.07|0.05|N|O|1995-07-29|1995-07-22|1995-08-20|COLLECT COD|REG AIR|as haggle against the carefully express dep|
+2978|28|1|4|7|6496.14|0.00|0.00|N|O|1995-07-18|1995-07-03|1995-07-23|NONE|FOB|. final ideas are blithe|
+2978|29|2|5|33|30657.66|0.09|0.03|R|F|1995-05-06|1995-07-23|1995-05-16|COLLECT COD|FOB|s. blithely unusual pack|
+2978|168|7|6|4|4272.64|0.08|0.04|N|O|1995-07-06|1995-07-31|1995-07-19|COLLECT COD|AIR|ffily unusual |
+2979|9|6|1|8|7272.00|0.00|0.08|N|O|1996-06-18|1996-05-21|1996-07-06|COLLECT COD|REG AIR|st blithely; blithely regular gifts dazz|
+2979|11|2|2|47|42817.47|0.05|0.00|N|O|1996-03-25|1996-05-13|1996-04-04|TAKE BACK RETURN|SHIP|iously unusual dependencies wake across|
+2979|188|9|3|35|38086.30|0.04|0.03|N|O|1996-05-25|1996-06-11|1996-06-24|DELIVER IN PERSON|MAIL|old ideas beneath the blit|
+2979|165|4|4|28|29824.48|0.05|0.08|N|O|1996-06-04|1996-04-23|1996-06-24|DELIVER IN PERSON|FOB|ing, regular pinto beans. blithel|
+2980|37|3|1|2|1874.06|0.09|0.03|N|O|1996-11-18|1996-10-22|1996-11-27|TAKE BACK RETURN|SHIP|enly across the special, pending packag|
+2980|10|7|2|48|43680.48|0.04|0.05|N|O|1996-09-25|1996-12-09|1996-10-12|NONE|REG AIR|totes. regular pinto |
+2980|133|9|3|27|27894.51|0.08|0.08|N|O|1996-12-08|1996-12-03|1996-12-14|NONE|REG AIR| theodolites cajole blithely sl|
+2980|25|10|4|49|45325.98|0.03|0.02|N|O|1996-10-04|1996-12-04|1996-10-06|NONE|RAIL|hy packages sleep quic|
+2980|187|8|5|24|26092.32|0.05|0.04|N|O|1997-01-12|1996-10-27|1997-01-14|NONE|MAIL|elets. fluffily regular in|
+2980|109|4|6|43|43391.30|0.01|0.01|N|O|1996-12-07|1996-11-10|1997-01-02|COLLECT COD|AIR|sts. slyly regu|
+2981|14|4|1|17|15538.17|0.03|0.05|N|O|1998-10-17|1998-10-02|1998-10-21|DELIVER IN PERSON|RAIL|, unusual packages x-ray. furious|
+2981|176|4|2|8|8609.36|0.06|0.03|N|O|1998-08-21|1998-09-28|1998-09-05|DELIVER IN PERSON|MAIL|ng to the f|
+2981|37|3|3|14|13118.42|0.03|0.07|N|O|1998-08-30|1998-10-04|1998-09-04|DELIVER IN PERSON|MAIL|kages detect furiously express requests.|
+2982|112|6|1|21|21254.31|0.00|0.01|A|F|1995-04-03|1995-06-08|1995-04-18|DELIVER IN PERSON|AIR|ironic deposits. furiously ex|
+2982|99|2|2|13|12988.17|0.02|0.08|R|F|1995-03-31|1995-05-07|1995-04-18|TAKE BACK RETURN|RAIL|regular deposits unwind alongside |
+2982|70|5|3|21|20371.47|0.01|0.01|R|F|1995-04-19|1995-06-03|1995-04-28|COLLECT COD|SHIP|egular ideas use furiously? bl|
+2983|163|4|1|44|46779.04|0.03|0.06|R|F|1992-02-09|1992-03-07|1992-03-09|TAKE BACK RETURN|AIR|ly regular instruct|
+2983|49|8|2|11|10439.44|0.09|0.06|A|F|1992-04-29|1992-02-27|1992-05-26|NONE|MAIL|aids integrate s|
+3008|132|3|1|8|8257.04|0.10|0.04|N|O|1995-12-06|1996-01-12|1995-12-22|TAKE BACK RETURN|FOB|yly ironic foxes. regular requests h|
+3008|200|3|2|31|34106.20|0.05|0.06|N|O|1995-12-14|1995-12-11|1995-12-31|TAKE BACK RETURN|AIR| bold packages. quic|
+3008|24|5|3|40|36960.80|0.01|0.03|N|O|1995-12-18|1996-01-06|1996-01-11|COLLECT COD|AIR|esias. theodolites detect blithely |
+3008|60|1|4|48|46082.88|0.07|0.06|N|O|1996-01-23|1996-01-07|1996-02-09|COLLECT COD|SHIP|ld theodolites. fluffily bold theodolit|
+3008|105|10|5|31|31158.10|0.03|0.02|N|O|1995-12-01|1996-01-20|1995-12-28|COLLECT COD|RAIL|nts use thinly around the carefully iro|
+3009|45|8|1|48|45361.92|0.10|0.02|N|O|1997-03-19|1997-05-13|1997-04-11|TAKE BACK RETURN|TRUCK| dependencies sleep quickly a|
+3009|185|6|2|38|41236.84|0.00|0.01|N|O|1997-05-01|1997-04-10|1997-05-17|TAKE BACK RETURN|AIR|nal packages should haggle slyly. quickl|
+3009|130|3|3|26|26783.38|0.08|0.02|N|O|1997-05-15|1997-05-10|1997-06-13|TAKE BACK RETURN|SHIP|uriously specia|
+3010|138|4|1|23|23876.99|0.04|0.00|N|O|1996-03-08|1996-02-29|1996-03-27|NONE|TRUCK|ounts. pendin|
+3010|174|4|2|22|23631.74|0.09|0.06|N|O|1996-03-06|1996-04-06|1996-03-18|COLLECT COD|REG AIR| final deposit|
+3010|58|6|3|24|22993.20|0.04|0.07|N|O|1996-05-09|1996-03-14|1996-05-15|DELIVER IN PERSON|RAIL|ar, even reques|
+3010|24|7|4|28|25872.56|0.09|0.06|N|O|1996-03-05|1996-03-28|1996-04-03|DELIVER IN PERSON|FOB|ake carefully carefully even request|
+3010|104|5|5|9|9036.90|0.02|0.02|N|O|1996-04-28|1996-03-17|1996-05-18|NONE|SHIP|inal packages. quickly even pinto|
+3010|92|3|6|38|37699.42|0.05|0.07|N|O|1996-04-15|1996-03-16|1996-04-21|DELIVER IN PERSON|RAIL|accounts ar|
+3011|198|10|1|5|5490.95|0.02|0.04|R|F|1992-04-21|1992-02-23|1992-05-15|NONE|TRUCK|nusual sentiments. carefully bold idea|
+3011|123|4|2|42|42971.04|0.05|0.00|A|F|1992-02-01|1992-03-18|1992-02-29|NONE|TRUCK|osits haggle quickly pending, |
+3012|195|7|1|49|53664.31|0.00|0.00|A|F|1993-08-07|1993-07-01|1993-08-08|NONE|MAIL| quickly furious packages. silently unusua|
+3012|161|2|2|37|39262.92|0.06|0.03|A|F|1993-08-16|1993-06-07|1993-08-24|TAKE BACK RETURN|REG AIR|uickly permanent packages sleep caref|
+3013|94|6|1|31|30816.79|0.08|0.08|N|O|1997-05-03|1997-04-05|1997-05-25|NONE|AIR|y furious depen|
+3013|139|5|2|30|31173.90|0.05|0.06|N|O|1997-05-02|1997-03-09|1997-05-12|TAKE BACK RETURN|MAIL|ronic packages. slyly even|
+3013|120|10|3|35|35704.20|0.00|0.03|N|O|1997-04-02|1997-05-04|1997-04-16|COLLECT COD|MAIL|ely accord|
+3013|181|2|4|17|18380.06|0.01|0.07|N|O|1997-02-26|1997-05-02|1997-03-27|DELIVER IN PERSON|SHIP|fully unusual account|
+3013|60|5|5|20|19201.20|0.00|0.04|N|O|1997-05-06|1997-03-18|1997-05-12|COLLECT COD|RAIL|unts boost regular ideas. slyly pe|
+3013|72|2|6|19|18469.33|0.08|0.07|N|O|1997-05-11|1997-04-18|1997-05-15|COLLECT COD|REG AIR|fluffily pending packages nag furiously al|
+3014|163|4|1|36|38273.76|0.05|0.03|A|F|1992-11-16|1993-01-20|1992-11-28|TAKE BACK RETURN|FOB|ding accounts boost fu|
+3014|106|1|2|36|36219.60|0.00|0.08|R|F|1992-12-28|1992-12-29|1993-01-24|COLLECT COD|MAIL|iously ironic r|
+3014|151|9|3|48|50455.20|0.06|0.02|A|F|1992-12-19|1993-01-08|1992-12-25|DELIVER IN PERSON|REG AIR|y pending theodolites wake. reg|
+3014|114|1|4|14|14197.54|0.10|0.02|R|F|1992-11-19|1993-01-01|1992-12-17|DELIVER IN PERSON|SHIP|. slyly brave platelets nag. careful,|
+3014|75|5|5|28|27301.96|0.02|0.08|R|F|1993-01-09|1992-12-18|1993-01-10|TAKE BACK RETURN|FOB|es are. final braids nag slyly. fluff|
+3014|38|4|6|30|28140.90|0.04|0.01|R|F|1993-02-28|1993-01-02|1993-03-20|TAKE BACK RETURN|AIR| final foxes.|
+3015|3|8|1|5|4515.00|0.09|0.00|A|F|1993-01-10|1992-12-02|1993-01-19|TAKE BACK RETURN|RAIL| the furiously pendi|
+3015|18|2|2|17|15606.17|0.03|0.01|R|F|1992-10-16|1992-11-20|1992-10-28|COLLECT COD|AIR|s above the fluffily final t|
+3015|91|4|3|23|22795.07|0.03|0.05|A|F|1992-12-03|1992-11-19|1992-12-23|DELIVER IN PERSON|FOB|s are slyly carefully special pinto bea|
+3015|156|7|4|7|7393.05|0.10|0.03|A|F|1992-12-07|1992-12-17|1992-12-30|DELIVER IN PERSON|REG AIR| after the evenly special packages ca|
+3015|165|4|5|42|44736.72|0.04|0.02|R|F|1993-01-21|1992-11-07|1993-02-11|DELIVER IN PERSON|AIR|encies haggle furious|
+3015|66|7|6|18|17389.08|0.02|0.03|R|F|1992-10-10|1992-11-19|1992-10-18|TAKE BACK RETURN|MAIL|equests wake fluffil|
+3040|16|6|1|18|16488.18|0.08|0.04|R|F|1993-06-25|1993-07-06|1993-07-19|TAKE BACK RETURN|SHIP|ly thin accou|
+3040|133|9|2|9|9298.17|0.00|0.01|A|F|1993-06-12|1993-05-16|1993-06-14|NONE|RAIL|ges. pending packages wake. requests|
+3040|126|5|3|30|30783.60|0.01|0.01|A|F|1993-08-06|1993-05-18|1993-08-19|NONE|MAIL|x furiously bold packages. expres|
+3040|83|4|4|14|13763.12|0.05|0.04|A|F|1993-05-13|1993-05-18|1993-05-19|TAKE BACK RETURN|REG AIR| haggle carefully. express hocke|
+3040|52|3|5|43|40938.15|0.04|0.04|R|F|1993-05-21|1993-05-25|1993-05-26|NONE|MAIL|sts nag slyly alongside of the depos|
+3040|18|5|6|10|9180.10|0.08|0.04|R|F|1993-05-16|1993-06-24|1993-06-11|DELIVER IN PERSON|MAIL|ely regular foxes haggle dari|
+3041|181|2|1|5|5405.90|0.07|0.04|N|O|1997-07-20|1997-07-15|1997-08-17|COLLECT COD|FOB|posits dazzle special p|
+3041|146|9|2|9|9415.26|0.03|0.03|N|O|1997-06-29|1997-08-14|1997-07-19|COLLECT COD|AIR|iously across the silent pinto beans. furi|
+3041|68|5|3|9|8712.54|0.09|0.06|N|O|1997-08-28|1997-07-23|1997-09-16|TAKE BACK RETURN|FOB|scapades after the special|
+3042|105|2|1|30|30153.00|0.08|0.06|A|F|1995-01-12|1995-02-15|1995-01-24|DELIVER IN PERSON|SHIP|the requests detect fu|
+3042|102|3|2|28|28058.80|0.05|0.03|A|F|1994-11-24|1995-01-02|1994-12-06|TAKE BACK RETURN|MAIL|ng the furiously r|
+3042|14|8|3|34|31076.34|0.04|0.00|R|F|1994-12-11|1995-02-03|1994-12-21|TAKE BACK RETURN|TRUCK|can wake after the enticingly stealthy i|
+3042|48|1|4|19|18012.76|0.02|0.01|A|F|1995-03-05|1995-01-24|1995-03-17|COLLECT COD|TRUCK|e carefully. regul|
+3043|46|9|1|23|21758.92|0.07|0.04|R|F|1992-05-08|1992-07-22|1992-05-18|COLLECT COD|TRUCK|uickly above the pending,|
+3043|6|3|2|15|13590.00|0.03|0.05|A|F|1992-05-27|1992-06-03|1992-06-09|COLLECT COD|FOB|usly furiously|
+3043|60|1|3|42|40322.52|0.10|0.07|R|F|1992-07-15|1992-06-19|1992-07-23|NONE|MAIL|ide of the un|
+3043|91|2|4|5|4955.45|0.10|0.01|A|F|1992-05-22|1992-07-02|1992-06-20|TAKE BACK RETURN|TRUCK|ake blithely re|
+3044|101|2|1|10|10011.00|0.07|0.08|N|O|1996-07-13|1996-05-06|1996-07-21|TAKE BACK RETURN|REG AIR| slyly ironic requests. s|
+3044|168|7|2|3|3204.48|0.06|0.02|N|O|1996-07-27|1996-05-26|1996-08-15|TAKE BACK RETURN|AIR|ecoys haggle furiously pending requests.|
+3044|19|3|3|47|43193.47|0.09|0.00|N|O|1996-05-24|1996-06-22|1996-05-30|NONE|REG AIR|ly around the car|
+3045|88|9|1|41|40511.28|0.05|0.01|N|O|1995-09-30|1995-11-24|1995-10-03|TAKE BACK RETURN|MAIL|ely final foxes. carefully ironic pinto b|
+3045|69|6|2|48|46514.88|0.02|0.03|N|O|1995-10-01|1995-12-16|1995-10-10|TAKE BACK RETURN|MAIL|ole quickly outside th|
+3046|74|5|1|44|42859.08|0.03|0.03|N|O|1996-03-03|1996-02-25|1996-04-01|NONE|AIR| are quickly. blithe|
+3046|54|5|2|46|43886.30|0.03|0.08|N|O|1996-03-22|1996-02-28|1996-04-07|TAKE BACK RETURN|AIR|sits sleep furious|
+3046|2|9|3|31|27962.00|0.03|0.07|N|O|1996-03-24|1996-01-30|1996-03-26|NONE|RAIL|y pending somas alongside of the slyly iro|
+3047|104|5|1|17|17069.70|0.08|0.02|N|O|1997-06-14|1997-04-20|1997-06-23|COLLECT COD|FOB|onic instruction|
+3047|14|1|2|23|21022.23|0.00|0.04|N|O|1997-05-20|1997-06-14|1997-05-28|TAKE BACK RETURN|REG AIR| slyly ironi|
+3072|57|9|1|6|5742.30|0.09|0.05|R|F|1994-02-09|1994-03-24|1994-02-28|DELIVER IN PERSON|REG AIR|gular requests abov|
+3072|108|3|2|36|36291.60|0.07|0.02|R|F|1994-04-14|1994-04-22|1994-05-06|COLLECT COD|AIR| theodolites. blithely e|
+3072|97|8|3|7|6979.63|0.04|0.07|R|F|1994-05-09|1994-03-31|1994-05-19|COLLECT COD|TRUCK|uests. ironic, ironic depos|
+3072|83|4|4|39|38340.12|0.05|0.08|A|F|1994-05-27|1994-04-20|1994-06-14|COLLECT COD|MAIL|es; slyly spe|
+3072|88|9|5|1|988.08|0.01|0.08|R|F|1994-02-26|1994-03-14|1994-03-19|NONE|AIR| slyly ironic attainments. car|
+3073|194|7|1|16|17507.04|0.07|0.01|R|F|1994-03-02|1994-03-23|1994-03-31|DELIVER IN PERSON|AIR|n requests. ironi|
+3073|22|5|2|47|43334.94|0.09|0.00|R|F|1994-03-26|1994-02-12|1994-04-21|NONE|REG AIR|eposits. fluffily|
+3073|87|8|3|10|9870.80|0.03|0.00|R|F|1994-02-11|1994-03-24|1994-02-26|COLLECT COD|FOB| furiously caref|
+3073|29|4|4|14|13006.28|0.09|0.07|R|F|1994-03-24|1994-04-01|1994-04-07|NONE|RAIL|ilently quiet epitaphs.|
+3073|41|10|5|25|23526.00|0.00|0.07|R|F|1994-04-14|1994-03-07|1994-04-22|NONE|TRUCK|nag asymptotes. pinto beans sleep |
+3073|147|8|6|39|40838.46|0.09|0.02|R|F|1994-05-01|1994-02-16|1994-05-12|DELIVER IN PERSON|AIR|lar excuses across the furiously even |
+3073|44|5|7|11|10384.44|0.08|0.07|A|F|1994-05-01|1994-03-06|1994-05-08|COLLECT COD|SHIP|instructions sleep according to the |
+3074|37|8|1|50|46851.50|0.08|0.08|A|F|1993-01-31|1992-12-15|1993-02-20|NONE|AIR|furiously pending requests haggle s|
+3074|139|5|2|39|40526.07|0.03|0.00|R|F|1992-12-08|1993-01-28|1992-12-09|DELIVER IN PERSON|TRUCK|iously throu|
+3075|9|6|1|39|35451.00|0.02|0.03|A|F|1994-06-10|1994-06-21|1994-06-20|NONE|FOB|ing deposits nag |
+3075|52|10|2|2|1904.10|0.07|0.08|R|F|1994-06-14|1994-06-10|1994-06-25|TAKE BACK RETURN|AIR|. unusual, unusual accounts haggle furious|
+3076|85|6|1|44|43343.52|0.00|0.05|A|F|1993-09-14|1993-10-04|1993-09-17|TAKE BACK RETURN|FOB| instructions h|
+3076|106|1|2|22|22134.20|0.08|0.00|A|F|1993-09-05|1993-09-10|1993-09-27|NONE|REG AIR|packages wake furiou|
+3076|5|8|3|31|28055.00|0.06|0.06|A|F|1993-08-10|1993-09-17|1993-08-17|TAKE BACK RETURN|SHIP|regular depos|
+3077|72|2|1|25|24301.75|0.06|0.01|N|O|1997-09-14|1997-10-16|1997-10-06|NONE|TRUCK|lent account|
+3077|91|3|2|40|39643.60|0.05|0.06|N|O|1997-10-22|1997-09-19|1997-11-19|DELIVER IN PERSON|AIR|to the enticing packag|
+3077|78|7|3|13|12714.91|0.03|0.07|N|O|1997-09-09|1997-10-15|1997-09-19|NONE|TRUCK|luffily close depende|
+3077|115|5|4|23|23347.53|0.03|0.02|N|O|1997-11-05|1997-09-16|1997-11-20|NONE|MAIL|lly. fluffily pending dinos across|
+3078|132|3|1|25|25803.25|0.01|0.03|A|F|1993-04-22|1993-05-01|1993-04-28|TAKE BACK RETURN|AIR|express dinos. carefully ironic|
+3078|78|8|2|21|20539.47|0.09|0.07|A|F|1993-03-20|1993-03-21|1993-04-01|COLLECT COD|AIR|e fluffily. |
+3079|70|5|1|20|19401.40|0.05|0.00|N|O|1997-10-18|1997-10-26|1997-11-14|NONE|RAIL|ets are according to the quickly dari|
+3079|117|1|2|38|38650.18|0.08|0.07|N|O|1997-11-07|1997-11-25|1997-12-06|NONE|RAIL|e carefully regular realms|
+3079|17|8|3|40|36680.40|0.02|0.08|N|O|1997-09-26|1997-12-11|1997-10-09|NONE|RAIL|ide of the pending, special deposi|
+3079|24|5|4|2|1848.04|0.00|0.08|N|O|1998-01-05|1997-11-17|1998-01-28|NONE|FOB|ly busy requests believ|
+3079|188|9|5|2|2176.36|0.10|0.00|N|O|1997-12-27|1997-10-25|1998-01-08|COLLECT COD|SHIP|y regular asymptotes doz|
+3079|166|1|6|46|49043.36|0.00|0.00|N|O|1997-11-19|1997-11-04|1997-11-25|DELIVER IN PERSON|REG AIR|es. final, regula|
+3104|51|6|1|20|19021.00|0.01|0.08|A|F|1993-12-31|1993-11-24|1994-01-12|DELIVER IN PERSON|REG AIR|s are. furiously s|
+3104|48|1|2|47|44557.88|0.02|0.05|A|F|1993-12-25|1993-11-02|1994-01-12|COLLECT COD|RAIL|ily daring acc|
+3104|63|4|3|11|10593.66|0.02|0.03|A|F|1993-10-05|1993-11-30|1993-10-27|NONE|TRUCK| special deposits u|
+3104|38|9|4|26|24388.78|0.02|0.08|R|F|1994-01-02|1993-12-05|1994-01-31|TAKE BACK RETURN|TRUCK|es boost carefully. slyly |
+3105|184|5|1|11|11925.98|0.01|0.06|N|O|1997-02-07|1997-02-09|1997-03-01|NONE|FOB|kly bold depths caj|
+3105|45|6|2|9|8505.36|0.08|0.08|N|O|1996-12-25|1997-02-04|1997-01-09|COLLECT COD|SHIP|es wake among t|
+3105|25|4|3|48|44400.96|0.02|0.05|N|O|1997-02-28|1997-01-31|1997-03-18|DELIVER IN PERSON|REG AIR|ending platelets wake carefully ironic inst|
+3105|91|5|4|23|22795.07|0.04|0.07|N|O|1997-03-08|1996-12-14|1997-03-18|COLLECT COD|REG AIR| detect slyly. blithely unusual requests ar|
+3105|90|1|5|8|7920.72|0.07|0.07|N|O|1996-12-28|1996-12-28|1997-01-25|NONE|FOB|s. blithely unusual ideas was after|
+3105|47|6|6|30|28411.20|0.08|0.05|N|O|1997-03-03|1997-02-03|1997-03-05|NONE|FOB|ess accounts boost among t|
+3106|86|7|1|22|21693.76|0.03|0.02|N|O|1997-02-28|1997-02-12|1997-03-03|DELIVER IN PERSON|FOB|structions atop the blithely|
+3106|136|2|2|49|50770.37|0.06|0.06|N|O|1997-02-27|1997-03-11|1997-03-12|NONE|TRUCK|lets. quietly regular courts |
+3106|52|7|3|42|39986.10|0.09|0.07|N|O|1997-04-05|1997-03-17|1997-04-22|COLLECT COD|REG AIR|nstructions wake. furiously |
+3106|196|10|4|6|6577.14|0.10|0.07|N|O|1997-02-02|1997-04-11|1997-02-27|COLLECT COD|REG AIR|symptotes. slyly bold platelets cajol|
+3106|65|2|5|16|15440.96|0.09|0.08|N|O|1997-02-25|1997-04-10|1997-03-16|NONE|AIR|sits wake slyl|
+3107|149|6|1|16|16786.24|0.05|0.04|N|O|1997-08-30|1997-10-20|1997-09-20|TAKE BACK RETURN|REG AIR|regular pinto beans. ironic ideas haggle|
+3107|142|3|2|35|36474.90|0.05|0.06|N|O|1997-08-27|1997-11-19|1997-09-14|COLLECT COD|TRUCK|ets doubt furiously final ideas. final|
+3107|170|9|3|23|24613.91|0.03|0.06|N|O|1997-12-10|1997-11-11|1997-12-14|TAKE BACK RETURN|SHIP|atelets must ha|
+3107|87|8|4|27|26651.16|0.00|0.08|N|O|1997-11-15|1997-10-31|1997-11-28|DELIVER IN PERSON|FOB|furiously final |
+3108|109|2|1|37|37336.70|0.06|0.04|A|F|1993-10-16|1993-10-01|1993-11-09|DELIVER IN PERSON|RAIL| final requests. |
+3108|166|1|2|26|27720.16|0.08|0.05|A|F|1993-11-12|1993-10-05|1993-12-09|COLLECT COD|TRUCK| slyly slow foxes wake furious|
+3109|18|2|1|32|29376.32|0.08|0.03|A|F|1993-09-05|1993-10-06|1993-09-18|DELIVER IN PERSON|FOB|ecial orbits are furiou|
+3109|145|4|2|49|51211.86|0.08|0.06|R|F|1993-10-24|1993-09-30|1993-11-21|TAKE BACK RETURN|AIR| even pearls. furiously pending |
+3109|176|4|3|43|46275.31|0.04|0.07|R|F|1993-09-29|1993-09-06|1993-10-13|COLLECT COD|MAIL|ding to the foxes. |
+3109|79|10|4|26|25455.82|0.01|0.05|R|F|1993-11-16|1993-10-18|1993-12-06|TAKE BACK RETURN|TRUCK| sleep slyly according to t|
+3109|143|2|5|50|52157.00|0.01|0.08|A|F|1993-09-17|1993-10-16|1993-10-11|NONE|FOB| regular packages boost blithely even, re|
+3109|15|9|6|10|9150.10|0.10|0.04|A|F|1993-10-26|1993-10-03|1993-11-09|NONE|TRUCK|sits haggle carefully. regular, unusual ac|
+3110|89|10|1|1|989.08|0.02|0.07|A|F|1995-01-15|1995-01-20|1995-01-30|DELIVER IN PERSON|REG AIR|c theodolites a|
+3110|57|2|2|31|29668.55|0.01|0.06|R|F|1995-03-31|1995-03-07|1995-04-21|TAKE BACK RETURN|REG AIR|en deposits. ironic|
+3110|3|10|3|34|30702.00|0.02|0.02|A|F|1995-02-23|1995-01-27|1995-03-09|TAKE BACK RETURN|FOB|ly pending requests ha|
+3110|40|1|4|16|15040.64|0.04|0.04|A|F|1995-01-10|1995-02-06|1995-01-26|NONE|MAIL|across the regular acco|
+3110|140|6|5|39|40565.46|0.09|0.01|A|F|1995-02-09|1995-01-21|1995-02-21|NONE|MAIL|side of the blithely unusual courts. slyly |
+3111|137|8|1|22|22816.86|0.06|0.05|N|O|1995-09-21|1995-11-09|1995-10-17|COLLECT COD|REG AIR|quests. regular dolphins against the |
+3111|58|10|2|30|28741.50|0.06|0.05|N|O|1995-10-05|1995-11-15|1995-11-01|TAKE BACK RETURN|TRUCK|eas are furiously slyly special deposits.|
+3111|52|3|3|10|9520.50|0.02|0.02|N|O|1995-11-10|1995-11-02|1995-12-04|NONE|FOB|ng the slyly ironic inst|
+3111|132|3|4|31|31996.03|0.00|0.08|N|O|1995-10-26|1995-09-26|1995-11-02|TAKE BACK RETURN|MAIL|kages detect express attainments|
+3111|54|6|5|14|13356.70|0.05|0.04|N|O|1995-10-17|1995-10-19|1995-10-19|TAKE BACK RETURN|SHIP|re. pinto |
+3111|86|7|6|5|4930.40|0.03|0.08|N|O|1995-08-30|1995-10-16|1995-09-04|DELIVER IN PERSON|TRUCK|. carefully even ideas|
+3111|148|9|7|41|42973.74|0.09|0.05|N|O|1995-11-22|1995-11-01|1995-12-01|TAKE BACK RETURN|FOB|fily slow ideas. |
+3136|142|5|1|30|31264.20|0.02|0.08|R|F|1994-08-13|1994-10-02|1994-09-02|TAKE BACK RETURN|RAIL|leep blithel|
+3136|103|4|2|7|7021.70|0.05|0.07|A|F|1994-10-08|1994-09-14|1994-10-11|TAKE BACK RETURN|SHIP|ic pinto beans are slyly. f|
+3136|158|3|3|43|45500.45|0.00|0.07|A|F|1994-09-05|1994-09-25|1994-09-11|NONE|RAIL|. special theodolites ha|
+3136|116|6|4|26|26418.86|0.04|0.05|A|F|1994-10-13|1994-11-07|1994-11-05|TAKE BACK RETURN|AIR|eep fluffily. daringly silent attainments d|
+3136|67|8|5|2|1934.12|0.08|0.07|R|F|1994-11-21|1994-11-03|1994-11-26|DELIVER IN PERSON|TRUCK|? special, silent |
+3136|80|1|6|29|28422.32|0.08|0.07|A|F|1994-11-16|1994-10-03|1994-12-14|NONE|FOB|latelets. final |
+3137|3|4|1|6|5418.00|0.02|0.02|N|O|1995-09-19|1995-10-23|1995-10-16|NONE|SHIP|ly express as|
+3137|6|3|2|4|3624.00|0.06|0.04|N|O|1995-10-01|1995-09-11|1995-10-30|COLLECT COD|RAIL|posits wake. silent excuses boost about|
+3138|93|5|1|7|6951.63|0.05|0.05|R|F|1994-03-04|1994-03-14|1994-03-20|NONE|AIR|lithely quickly even packages. packages|
+3138|44|5|2|27|25489.08|0.09|0.01|R|F|1994-03-24|1994-03-23|1994-04-18|DELIVER IN PERSON|FOB|counts cajole fluffily carefully special i|
+3138|197|8|3|32|35110.08|0.00|0.01|R|F|1994-02-24|1994-05-07|1994-02-28|TAKE BACK RETURN|MAIL|inal foxes affix slyly. fluffily regul|
+3138|172|3|4|38|40742.46|0.07|0.04|R|F|1994-02-21|1994-03-21|1994-03-13|COLLECT COD|FOB|lithely fluffily un|
+3138|10|1|5|12|10920.12|0.09|0.02|A|F|1994-03-04|1994-04-11|1994-03-21|COLLECT COD|FOB|. bold pinto beans haggl|
+3138|44|7|6|25|23601.00|0.05|0.08|A|F|1994-05-19|1994-04-07|1994-06-17|TAKE BACK RETURN|AIR|dolites around the carefully busy the|
+3139|40|6|1|46|43241.84|0.08|0.03|R|F|1992-04-28|1992-03-04|1992-05-19|TAKE BACK RETURN|FOB|of the unusual, unusual re|
+3140|7|4|1|21|19047.00|0.08|0.02|R|F|1992-04-12|1992-05-31|1992-04-21|NONE|REG AIR| furiously sly excuses according to the|
+3140|89|10|2|10|9890.80|0.07|0.01|A|F|1992-05-30|1992-05-09|1992-06-09|COLLECT COD|RAIL|accounts. expres|
+3140|133|4|3|28|28927.64|0.06|0.00|R|F|1992-06-08|1992-07-07|1992-07-08|TAKE BACK RETURN|SHIP|lar ideas. slyly ironic d|
+3141|177|6|1|32|34469.44|0.06|0.00|N|O|1995-11-21|1995-12-18|1995-11-26|DELIVER IN PERSON|FOB|oxes are quickly about t|
+3141|10|7|2|37|33670.37|0.10|0.05|N|O|1996-01-24|1995-12-16|1996-01-27|DELIVER IN PERSON|AIR|press pinto beans. bold accounts boost b|
+3141|79|7|3|9|8811.63|0.09|0.02|N|O|1995-11-11|1995-12-10|1995-12-02|DELIVER IN PERSON|MAIL|uickly ironic, pendi|
+3141|46|9|4|47|44463.88|0.03|0.01|N|O|1995-11-29|1996-01-13|1995-12-10|TAKE BACK RETURN|TRUCK| are slyly pi|
+3142|120|7|1|15|15301.80|0.03|0.08|R|F|1992-08-15|1992-08-18|1992-08-22|DELIVER IN PERSON|AIR|instructions are. ironic packages doz|
+3143|90|1|1|22|21781.98|0.02|0.00|A|F|1993-05-11|1993-03-26|1993-05-20|TAKE BACK RETURN|MAIL|l, special instructions nag |
+3143|183|4|2|40|43327.20|0.03|0.08|A|F|1993-05-07|1993-03-29|1993-05-17|COLLECT COD|FOB|sly unusual theodolites. slyly ev|
+3143|183|4|3|22|23829.96|0.05|0.03|A|F|1993-03-18|1993-05-09|1993-04-14|DELIVER IN PERSON|MAIL|beans. fluf|
+3143|66|7|4|46|44438.76|0.05|0.08|R|F|1993-04-19|1993-03-21|1993-05-05|COLLECT COD|REG AIR|low forges haggle. even packages use bli|
+3168|60|8|1|46|44162.76|0.08|0.08|R|F|1992-02-14|1992-03-02|1992-03-02|TAKE BACK RETURN|SHIP|y across the express accounts. fluff|
+3168|154|5|2|1|1054.15|0.06|0.08|A|F|1992-05-27|1992-03-12|1992-06-09|TAKE BACK RETURN|SHIP|pinto beans. slyly regular courts haggle |
+3168|128|3|3|13|13365.56|0.09|0.02|A|F|1992-03-05|1992-04-29|1992-03-15|NONE|SHIP|ironic somas haggle quick|
+3168|165|10|4|11|11716.76|0.02|0.05|R|F|1992-04-12|1992-03-17|1992-05-12|COLLECT COD|SHIP|ously furious dependenc|
+3169|192|4|1|12|13106.28|0.01|0.04|R|F|1994-01-05|1994-03-18|1994-01-21|COLLECT COD|REG AIR| regular d|
+3169|200|3|2|17|18703.40|0.05|0.04|R|F|1994-03-02|1994-01-21|1994-03-03|DELIVER IN PERSON|TRUCK|usly regular packages. ironi|
+3169|188|9|3|12|13058.16|0.08|0.07|A|F|1994-04-18|1994-03-12|1994-05-08|TAKE BACK RETURN|FOB|atelets. pac|
+3169|105|6|4|26|26132.60|0.10|0.04|R|F|1994-04-08|1994-03-21|1994-04-29|NONE|TRUCK|ter the regular ideas. slyly iro|
+3169|108|9|5|6|6048.60|0.09|0.01|A|F|1994-03-24|1994-02-22|1994-04-04|TAKE BACK RETURN|AIR|ular instructions. ca|
+3169|177|8|6|46|49549.82|0.02|0.07|A|F|1994-02-01|1994-01-22|1994-02-24|DELIVER IN PERSON|RAIL|thely bold theodolites are fl|
+3170|40|6|1|12|11280.48|0.03|0.03|N|O|1998-02-12|1998-01-17|1998-02-24|NONE|TRUCK|ing accounts along the speci|
+3170|100|2|2|21|21002.10|0.01|0.00|N|O|1997-12-09|1998-01-31|1997-12-21|DELIVER IN PERSON|MAIL|o beans. carefully final requests dou|
+3170|89|10|3|27|26705.16|0.00|0.05|N|O|1998-02-25|1998-01-29|1998-02-27|COLLECT COD|AIR|efully bold foxes. regular, ev|
+3170|41|2|4|34|31995.36|0.05|0.04|N|O|1998-02-01|1998-01-11|1998-02-20|TAKE BACK RETURN|TRUCK|s about the fluffily final de|
+3170|90|1|5|32|31682.88|0.02|0.04|N|O|1997-11-24|1997-12-12|1997-12-15|COLLECT COD|SHIP|ggle about the furiously r|
+3170|110|5|6|43|43434.73|0.08|0.05|N|O|1998-01-05|1998-01-04|1998-01-14|NONE|REG AIR|. express dolphins use sly|
+3170|84|5|7|26|25586.08|0.10|0.05|N|O|1998-02-12|1997-12-22|1998-02-28|COLLECT COD|TRUCK|s engage furiously. |
+3171|47|4|1|34|32199.36|0.04|0.00|A|F|1993-05-30|1993-05-27|1993-06-06|DELIVER IN PERSON|REG AIR|r the final, even packages. quickly|
+3171|139|10|2|50|51956.50|0.01|0.04|A|F|1993-07-19|1993-05-15|1993-07-31|TAKE BACK RETURN|REG AIR|riously final foxes about the ca|
+3172|96|9|1|4|3984.36|0.06|0.07|A|F|1992-09-26|1992-08-15|1992-10-20|DELIVER IN PERSON|TRUCK|s are slyly thin package|
+3172|148|7|2|43|45070.02|0.05|0.07|R|F|1992-08-22|1992-07-07|1992-08-26|COLLECT COD|MAIL| final packages. |
+3172|132|3|3|13|13417.69|0.03|0.01|R|F|1992-07-06|1992-08-06|1992-08-05|DELIVER IN PERSON|MAIL|inal deposits haggle along the|
+3172|135|6|4|28|28983.64|0.08|0.04|R|F|1992-07-09|1992-07-14|1992-07-16|NONE|MAIL|regular ideas. packages are furi|
+3172|64|5|5|31|29885.86|0.05|0.08|A|F|1992-09-01|1992-08-27|1992-09-23|NONE|SHIP|. slyly regular dependencies haggle quiet|
+3173|195|6|1|35|38331.65|0.01|0.08|N|O|1996-09-09|1996-10-15|1996-10-04|TAKE BACK RETURN|RAIL| across the slyly even requests.|
+3173|178|7|2|5|5390.85|0.09|0.07|N|O|1996-12-06|1996-09-17|1996-12-07|DELIVER IN PERSON|REG AIR|express depo|
+3173|46|9|3|16|15136.64|0.06|0.01|N|O|1996-08-12|1996-09-21|1996-08-22|NONE|SHIP|e special,|
+3173|94|5|4|2|1988.18|0.00|0.00|N|O|1996-10-15|1996-11-06|1996-10-18|COLLECT COD|MAIL|ular pearls|
+3173|185|6|5|2|2170.36|0.00|0.06|N|O|1996-08-18|1996-09-21|1996-09-07|DELIVER IN PERSON|MAIL|fluffily above t|
+3174|186|7|1|6|6517.08|0.04|0.08|N|O|1996-03-13|1996-02-09|1996-03-22|DELIVER IN PERSON|AIR| furiously ironic|
+3174|194|7|2|4|4376.76|0.01|0.05|N|O|1995-11-17|1996-01-08|1995-11-27|DELIVER IN PERSON|RAIL|deas sleep thi|
+3174|92|4|3|21|20833.89|0.08|0.05|N|O|1996-02-20|1995-12-28|1996-03-17|NONE|MAIL|iously. idly bold theodolites a|
+3174|192|6|4|13|14198.47|0.08|0.06|N|O|1996-01-11|1996-01-26|1996-02-01|DELIVER IN PERSON|SHIP|leep quickly? slyly special platelets|
+3174|72|2|5|39|37910.73|0.02|0.06|N|O|1995-12-02|1996-02-08|1995-12-12|TAKE BACK RETURN|TRUCK| wake slyly foxes. bold requests p|
+3174|120|7|6|8|8160.96|0.07|0.08|N|O|1995-12-07|1996-01-08|1995-12-29|DELIVER IN PERSON|TRUCK|nic deposits among t|
+3175|120|10|1|28|28563.36|0.10|0.01|R|F|1994-09-27|1994-10-05|1994-10-04|NONE|FOB|ore the even, silent foxes. b|
+3175|1|4|2|38|34238.00|0.01|0.07|R|F|1994-10-10|1994-08-25|1994-10-28|NONE|MAIL|the quickly even dolph|
+3175|129|4|3|12|12349.44|0.09|0.07|R|F|1994-10-16|1994-09-15|1994-10-18|NONE|AIR|ter the pending deposits. slyly e|
+3175|85|6|4|14|13791.12|0.02|0.05|R|F|1994-10-21|1994-09-05|1994-11-15|NONE|MAIL|nt dependencies are quietly even |
+3175|18|8|5|47|43146.47|0.08|0.03|R|F|1994-08-08|1994-09-10|1994-08-21|COLLECT COD|REG AIR| final requests x-r|
+3175|175|6|6|44|47307.48|0.01|0.00|R|F|1994-09-26|1994-08-30|1994-10-24|TAKE BACK RETURN|MAIL|are carefully furiously ironic accounts. e|
+3175|1|4|7|32|28832.00|0.01|0.02|R|F|1994-09-29|1994-09-20|1994-10-10|TAKE BACK RETURN|SHIP|lites sleep|
+3200|116|6|1|17|17273.87|0.10|0.00|N|O|1996-06-06|1996-04-21|1996-06-14|DELIVER IN PERSON|AIR|side of the furiously pendin|
+3200|166|1|2|27|28786.32|0.03|0.00|N|O|1996-05-07|1996-05-01|1996-05-09|TAKE BACK RETURN|REG AIR|as haggle furiously against the fluff|
+3200|131|2|3|36|37120.68|0.01|0.01|N|O|1996-03-22|1996-03-19|1996-03-30|DELIVER IN PERSON|FOB|f the carefu|
+3200|30|9|4|11|10230.33|0.10|0.02|N|O|1996-03-18|1996-03-21|1996-04-14|COLLECT COD|RAIL|osits sleep fur|
+3200|198|9|5|16|17571.04|0.05|0.00|N|O|1996-02-28|1996-03-13|1996-03-11|NONE|RAIL|ly against the quiet packages. blith|
+3200|175|3|6|25|26879.25|0.10|0.01|N|O|1996-02-08|1996-04-11|1996-03-06|COLLECT COD|FOB| slyly regular hockey players! pinto beans |
+3201|46|7|1|11|10406.44|0.10|0.06|A|F|1993-09-27|1993-08-29|1993-10-18|NONE|TRUCK|ing to the furiously expr|
+3201|118|5|2|27|27488.97|0.08|0.02|R|F|1993-08-31|1993-08-24|1993-09-08|TAKE BACK RETURN|FOB|deposits are slyly along|
+3201|119|6|3|50|50955.50|0.00|0.08|R|F|1993-10-27|1993-09-30|1993-11-16|COLLECT COD|TRUCK| deposits. express, ir|
+3202|183|4|1|30|32495.40|0.09|0.02|A|F|1993-03-18|1993-03-10|1993-03-23|COLLECT COD|SHIP|ven platelets. furiously final|
+3202|20|4|2|22|20240.44|0.01|0.02|R|F|1993-02-16|1993-02-16|1993-03-16|TAKE BACK RETURN|MAIL|the express packages. fu|
+3203|144|5|1|23|24015.22|0.01|0.07|N|O|1998-01-04|1998-01-12|1998-01-24|COLLECT COD|SHIP|uses. fluffily ironic pinto bea|
+3203|188|9|2|22|23939.96|0.03|0.03|N|O|1998-02-12|1998-01-01|1998-02-18|TAKE BACK RETURN|REG AIR|e the blithely regular accounts boost f|
+3204|12|2|1|10|9120.10|0.10|0.07|R|F|1993-01-27|1993-03-08|1993-01-29|COLLECT COD|SHIP|counts. bold |
+3204|7|10|2|39|35373.00|0.10|0.03|R|F|1993-02-11|1993-03-19|1993-02-28|TAKE BACK RETURN|MAIL|sits sleep theodolites. slyly bo|
+3205|68|5|1|7|6776.42|0.09|0.00|R|F|1992-07-05|1992-06-17|1992-07-07|NONE|SHIP|ly alongsi|
+3205|29|10|2|32|29728.64|0.08|0.03|A|F|1992-06-01|1992-07-10|1992-06-06|TAKE BACK RETURN|RAIL|lar accoun|
+3205|103|6|3|38|38117.80|0.10|0.08|A|F|1992-07-31|1992-06-03|1992-08-20|DELIVER IN PERSON|AIR|usly quiet accounts. slyly pending pinto |
+3205|56|7|4|10|9560.50|0.01|0.07|A|F|1992-06-18|1992-07-04|1992-07-16|COLLECT COD|RAIL| deposits cajole careful|
+3205|70|9|5|18|17461.26|0.03|0.03|A|F|1992-07-04|1992-06-14|1992-08-03|TAKE BACK RETURN|RAIL|symptotes. slyly even deposits ar|
+3205|195|8|6|19|20808.61|0.07|0.08|R|F|1992-05-28|1992-05-30|1992-06-05|COLLECT COD|AIR|yly pending packages snooz|
+3205|69|8|7|36|34886.16|0.06|0.03|A|F|1992-05-31|1992-06-19|1992-06-03|TAKE BACK RETURN|SHIP|s. ironic platelets above the s|
+3206|176|4|1|1|1076.17|0.07|0.05|N|O|1996-11-22|1996-10-16|1996-12-07|TAKE BACK RETURN|FOB|y unusual foxes cajole ab|
+3206|111|5|2|37|37411.07|0.07|0.01|N|O|1996-09-06|1996-10-31|1996-09-25|COLLECT COD|SHIP| quick theodolites hagg|
+3206|186|7|3|24|26068.32|0.00|0.08|N|O|1996-08-25|1996-10-01|1996-09-04|COLLECT COD|TRUCK|encies sleep deposits--|
+3207|113|3|1|2|2026.22|0.10|0.03|N|O|1998-06-15|1998-04-20|1998-06-21|COLLECT COD|MAIL|among the ironic, even packages |
+3207|71|9|2|42|40784.94|0.00|0.00|N|O|1998-05-02|1998-05-10|1998-06-01|NONE|SHIP|to the quickly special accounts? ironically|
+3207|152|7|3|17|17886.55|0.03|0.04|N|O|1998-03-27|1998-04-06|1998-03-28|COLLECT COD|RAIL|eep against the instructions. gifts hag|
+3207|19|6|4|32|29408.32|0.00|0.03|N|O|1998-06-17|1998-04-26|1998-07-07|TAKE BACK RETURN|SHIP|y across the slyly express foxes. bl|
+3207|83|4|5|8|7864.64|0.00|0.06|N|O|1998-06-13|1998-04-26|1998-07-11|COLLECT COD|SHIP|y. final pint|
+3207|134|5|6|32|33092.16|0.03|0.05|N|O|1998-04-19|1998-05-01|1998-05-08|COLLECT COD|FOB|l deposits wake beyond the carefully|
+3232|14|5|1|22|20108.22|0.10|0.01|A|F|1992-11-30|1992-12-09|1992-12-04|NONE|RAIL|thely. furio|
+3232|135|1|2|34|35194.42|0.07|0.04|R|F|1993-01-09|1992-11-14|1993-02-03|NONE|SHIP|old packages integrate quickly |
+3232|181|2|3|3|3243.54|0.04|0.06|R|F|1992-12-14|1992-12-11|1992-12-29|DELIVER IN PERSON|FOB|ily blithely ironic acco|
+3233|51|2|1|23|21874.15|0.04|0.05|A|F|1994-12-07|1995-01-11|1994-12-26|NONE|AIR|pending instructions use after the carefu|
+3233|154|6|2|6|6324.90|0.02|0.08|A|F|1994-12-06|1994-12-05|1994-12-07|TAKE BACK RETURN|REG AIR|requests are quickly above the slyly p|
+3233|100|4|3|2|2000.20|0.04|0.06|R|F|1995-01-03|1995-01-02|1995-01-21|TAKE BACK RETURN|AIR| across the bold packages|
+3233|9|2|4|25|22725.00|0.04|0.07|A|F|1994-11-24|1995-01-07|1994-12-11|NONE|RAIL|oss the pl|
+3234|79|10|1|45|44058.15|0.01|0.04|N|O|1996-05-15|1996-05-09|1996-06-02|DELIVER IN PERSON|TRUCK| express packages are carefully. f|
+3234|84|5|2|23|22633.84|0.03|0.00|N|O|1996-05-29|1996-05-15|1996-06-17|DELIVER IN PERSON|AIR|d-- fluffily special packag|
+3234|75|4|3|16|15601.12|0.06|0.05|N|O|1996-06-10|1996-05-30|1996-06-18|COLLECT COD|RAIL|ithely ironic accounts wake along t|
+3234|122|1|4|50|51106.00|0.09|0.05|N|O|1996-06-11|1996-05-19|1996-06-18|NONE|MAIL|ly regular ideas according to the regula|
+3234|165|2|5|14|14912.24|0.01|0.07|N|O|1996-04-06|1996-05-30|1996-04-13|NONE|REG AIR|lithely regular f|
+3235|109|2|1|9|9081.90|0.07|0.00|N|O|1995-11-17|1995-12-24|1995-11-30|COLLECT COD|AIR|l courts sleep quickly slyly |
+3235|95|6|2|43|42788.87|0.10|0.07|N|O|1995-12-25|1996-01-23|1996-01-09|COLLECT COD|MAIL|ckly final instru|
+3235|138|9|3|29|30105.77|0.06|0.06|N|O|1996-01-28|1995-12-26|1996-02-12|DELIVER IN PERSON|RAIL|e fluffy pinto bea|
+3235|178|9|4|23|24797.91|0.00|0.01|N|O|1996-02-16|1996-01-05|1996-03-07|DELIVER IN PERSON|SHIP|ldly ironic pinto beans|
+3236|117|4|1|10|10171.10|0.06|0.05|N|O|1996-11-15|1996-12-14|1996-11-29|TAKE BACK RETURN|AIR|arefully. fluffily reg|
+3236|122|7|2|21|21464.52|0.01|0.07|N|O|1996-12-23|1996-12-12|1997-01-21|NONE|AIR| final pinto |
+3236|118|2|3|7|7126.77|0.07|0.01|N|O|1996-12-27|1996-12-18|1997-01-24|DELIVER IN PERSON|SHIP|dolites. slyly unus|
+3237|11|5|1|11|10021.11|0.02|0.07|A|F|1992-08-03|1992-07-31|1992-08-13|TAKE BACK RETURN|AIR|es. permanently express platelets besid|
+3238|72|3|1|12|11664.84|0.06|0.01|R|F|1993-03-06|1993-05-08|1993-04-01|DELIVER IN PERSON|AIR|ackages affix furiously. furiously bol|
+3238|173|2|2|26|27902.42|0.01|0.06|A|F|1993-02-25|1993-04-04|1993-03-20|TAKE BACK RETURN|REG AIR|g accounts sleep furiously ironic attai|
+3238|81|2|3|1|981.08|0.00|0.04|R|F|1993-05-17|1993-04-18|1993-05-27|NONE|SHIP|wake alongs|
+3239|45|8|1|50|47252.00|0.05|0.01|N|O|1998-02-09|1998-04-02|1998-02-22|NONE|FOB|d blithely stea|
+3239|45|8|2|43|40636.72|0.01|0.06|N|O|1998-01-15|1998-03-12|1998-01-29|COLLECT COD|REG AIR|y. bold pinto beans use |
+3239|13|7|3|13|11869.13|0.01|0.05|N|O|1998-02-10|1998-02-19|1998-02-25|DELIVER IN PERSON|MAIL|r deposits solve fluf|
+3239|195|6|4|26|28474.94|0.03|0.05|N|O|1998-01-21|1998-03-21|1998-02-08|DELIVER IN PERSON|SHIP|ngly pending platelets are fluff|
+3239|12|9|5|31|28272.31|0.10|0.08|N|O|1998-04-14|1998-03-24|1998-04-17|DELIVER IN PERSON|FOB|foxes. pendin|
+3264|200|1|1|39|42907.80|0.06|0.06|N|O|1996-11-07|1996-12-12|1996-11-20|TAKE BACK RETURN|REG AIR|sleep carefully after the slyly final|
+3264|131|2|2|34|35058.42|0.00|0.01|N|O|1997-01-03|1997-01-06|1997-01-29|TAKE BACK RETURN|REG AIR|rns haggle carefully. blit|
+3264|125|8|3|11|11276.32|0.09|0.03|N|O|1996-12-11|1996-12-19|1996-12-15|DELIVER IN PERSON|SHIP|regular packages|
+3264|109|10|4|24|24218.40|0.09|0.07|N|O|1997-01-07|1996-12-13|1997-01-11|TAKE BACK RETURN|RAIL|ctions. quick|
+3264|63|4|5|6|5778.36|0.04|0.03|N|O|1996-11-10|1996-12-05|1996-11-22|TAKE BACK RETURN|SHIP|press packages. ironical|
+3264|141|2|6|43|44769.02|0.06|0.06|N|O|1997-01-17|1997-01-24|1997-02-01|TAKE BACK RETURN|TRUCK|leep at the blithely bold|
+3265|25|4|1|8|7400.16|0.06|0.02|A|F|1992-09-01|1992-09-12|1992-09-27|DELIVER IN PERSON|TRUCK|thely ironic requests sleep slyly-- i|
+3265|72|2|2|7|6804.49|0.09|0.00|R|F|1992-09-16|1992-09-04|1992-10-14|DELIVER IN PERSON|MAIL|he forges. fluffily regular asym|
+3265|191|4|3|28|30553.32|0.09|0.08|A|F|1992-10-22|1992-08-23|1992-10-25|NONE|RAIL|n requests. quickly final dinos|
+3266|64|1|1|31|29885.86|0.09|0.02|N|O|1995-06-19|1995-05-04|1995-07-06|COLLECT COD|MAIL|grate among the quickly express deposits|
+3266|38|4|2|43|40335.29|0.06|0.07|R|F|1995-05-04|1995-05-30|1995-05-11|COLLECT COD|AIR|ular asymptotes use careful|
+3267|185|6|1|33|35810.94|0.06|0.01|N|O|1997-03-30|1997-03-25|1997-04-23|TAKE BACK RETURN|AIR|es boost. |
+3268|96|7|1|1|996.09|0.06|0.08|A|F|1994-09-12|1994-08-31|1994-09-16|NONE|TRUCK|. ironic, bold requests use carefull|
+3268|42|9|2|40|37681.60|0.08|0.01|R|F|1994-06-30|1994-08-22|1994-07-25|COLLECT COD|FOB|ly. bold, eve|
+3269|161|10|1|40|42446.40|0.02|0.07|N|O|1996-06-11|1996-05-06|1996-06-15|DELIVER IN PERSON|TRUCK|es. pending d|
+3269|38|4|2|46|43149.38|0.00|0.02|N|O|1996-04-21|1996-04-12|1996-05-10|DELIVER IN PERSON|MAIL|final asymptotes nag|
+3269|44|3|3|39|36817.56|0.02|0.03|N|O|1996-03-13|1996-05-26|1996-03-19|COLLECT COD|MAIL|he express packages?|
+3269|83|4|4|37|36373.96|0.07|0.05|N|O|1996-06-14|1996-04-27|1996-07-07|NONE|MAIL|egular requests. carefully un|
+3269|93|7|5|42|41709.78|0.09|0.05|N|O|1996-03-19|1996-04-24|1996-04-18|COLLECT COD|TRUCK| the special packages. |
+3269|131|7|6|16|16498.08|0.01|0.08|N|O|1996-03-03|1996-04-06|1996-03-06|NONE|RAIL|s cajole. silent deposits are f|
+3270|35|1|1|11|10285.33|0.07|0.06|N|O|1997-07-29|1997-08-11|1997-08-05|TAKE BACK RETURN|AIR| solve at the regular deposits. |
+3270|38|4|2|44|41273.32|0.10|0.05|N|O|1997-07-20|1997-08-15|1997-08-04|DELIVER IN PERSON|SHIP| accounts. carefully even |
+3270|65|4|3|20|19301.20|0.01|0.02|N|O|1997-08-26|1997-07-31|1997-08-30|DELIVER IN PERSON|FOB|en accounts among the c|
+3270|189|10|4|29|31586.22|0.06|0.05|N|O|1997-07-01|1997-07-23|1997-07-10|TAKE BACK RETURN|MAIL|sly regular asymptotes. slyly dog|
+3270|34|10|5|32|29888.96|0.03|0.00|N|O|1997-09-23|1997-08-17|1997-09-27|NONE|REG AIR|promise carefully.|
+3270|57|5|6|29|27754.45|0.01|0.04|N|O|1997-08-22|1997-08-17|1997-09-06|COLLECT COD|RAIL|ptotes nag above the quickly bold deposits|
+3270|117|1|7|9|9153.99|0.06|0.08|N|O|1997-08-14|1997-08-11|1997-09-09|DELIVER IN PERSON|SHIP|ual packages|
+3271|57|9|1|30|28711.50|0.01|0.04|A|F|1992-01-16|1992-03-20|1992-01-17|DELIVER IN PERSON|AIR|r the unusual Tiresia|
+3271|54|5|2|18|17172.90|0.09|0.06|R|F|1992-05-01|1992-03-28|1992-05-29|DELIVER IN PERSON|FOB| packages eat around the furiously regul|
+3271|95|6|3|14|13931.26|0.05|0.01|A|F|1992-02-24|1992-02-14|1992-03-23|NONE|AIR|ending, even packa|
+3271|64|1|4|29|27957.74|0.07|0.04|A|F|1992-03-10|1992-02-05|1992-03-14|COLLECT COD|MAIL|lar instructions. carefully regular|
+3296|84|5|1|12|11808.96|0.06|0.07|R|F|1994-12-08|1994-12-14|1994-12-24|COLLECT COD|AIR|y about the slyly bold pinto bea|
+3296|149|8|2|31|32523.34|0.08|0.00|R|F|1995-01-26|1994-12-25|1995-02-16|NONE|REG AIR|ainst the furi|
+3296|185|6|3|29|31470.22|0.02|0.04|A|F|1995-01-12|1994-11-26|1995-02-06|DELIVER IN PERSON|SHIP|ss ideas are reg|
+3296|140|1|4|47|48886.58|0.06|0.00|A|F|1994-11-08|1994-12-20|1994-11-30|NONE|FOB|egular deposits. quic|
+3296|177|6|5|16|17234.72|0.06|0.02|R|F|1995-01-11|1994-12-27|1995-01-12|DELIVER IN PERSON|SHIP|kages cajole carefully |
+3296|197|1|6|40|43887.60|0.00|0.04|A|F|1994-12-28|1994-12-08|1995-01-13|COLLECT COD|REG AIR|ronic ideas across the|
+3296|36|2|7|6|5616.18|0.02|0.01|R|F|1995-01-03|1994-12-23|1995-01-27|TAKE BACK RETURN|AIR|carefully fur|
+3297|134|10|1|10|10341.30|0.10|0.04|A|F|1992-12-14|1993-01-21|1992-12-26|NONE|SHIP|ironic idea|
+3298|149|6|1|9|9442.26|0.01|0.06|N|O|1996-08-15|1996-05-24|1996-09-12|COLLECT COD|REG AIR|ly final accou|
+3298|186|7|2|27|29326.86|0.06|0.06|N|O|1996-07-10|1996-05-21|1996-07-15|DELIVER IN PERSON|FOB|lar packages. regular deposit|
+3298|29|2|3|25|23225.50|0.10|0.08|N|O|1996-06-30|1996-05-31|1996-07-23|COLLECT COD|SHIP|ly express f|
+3298|191|5|4|1|1091.19|0.10|0.03|N|O|1996-07-31|1996-05-23|1996-08-24|TAKE BACK RETURN|FOB|refully regular requ|
+3299|183|4|1|40|43327.20|0.03|0.02|A|F|1994-03-21|1994-03-23|1994-04-12|COLLECT COD|AIR|lyly even request|
+3300|129|4|1|3|3087.36|0.07|0.02|N|O|1995-11-01|1995-10-02|1995-11-20|NONE|REG AIR|g according to the dugouts. caref|
+3300|149|10|2|23|24130.22|0.02|0.02|N|O|1995-08-17|1995-09-03|1995-09-04|COLLECT COD|TRUCK|he fluffily final a|
+3301|169|8|1|45|48112.20|0.04|0.05|A|F|1994-11-19|1994-10-27|1994-11-24|TAKE BACK RETURN|FOB|nusual, final excuses after the entici|
+3302|36|2|1|45|42121.35|0.09|0.00|N|O|1996-01-24|1995-12-16|1996-02-13|COLLECT COD|FOB|counts use quickl|
+3303|184|5|1|25|27104.50|0.06|0.01|N|O|1998-03-25|1998-01-31|1998-04-12|NONE|SHIP|lly regular pi|
+3303|21|2|2|15|13815.30|0.04|0.06|N|O|1998-01-29|1998-01-22|1998-02-21|COLLECT COD|SHIP| detect sly|
+3303|99|10|3|37|36966.33|0.05|0.02|N|O|1998-02-16|1998-03-07|1998-02-18|TAKE BACK RETURN|TRUCK| carefully ironic asympt|
+3303|36|2|4|26|24336.78|0.09|0.00|N|O|1998-01-18|1998-03-11|1998-02-11|DELIVER IN PERSON|REG AIR|ickly permanent requests w|
+3328|113|7|1|6|6078.66|0.03|0.08|A|F|1993-03-07|1993-01-25|1993-03-29|COLLECT COD|TRUCK|ffily even instructions detect b|
+3328|5|2|2|23|20815.00|0.01|0.06|R|F|1993-01-12|1993-02-07|1993-01-30|TAKE BACK RETURN|MAIL|y. careful|
+3328|139|10|3|44|45721.72|0.05|0.00|R|F|1992-12-03|1992-12-19|1992-12-09|TAKE BACK RETURN|FOB|dly quickly final foxes? re|
+3328|95|9|4|42|41793.78|0.01|0.05|R|F|1992-11-24|1992-12-20|1992-12-06|DELIVER IN PERSON|AIR|ronic requests|
+3328|131|7|5|25|25778.25|0.05|0.00|R|F|1993-01-28|1993-01-04|1993-01-31|NONE|RAIL|e unusual, r|
+3329|138|4|1|36|37372.68|0.09|0.08|N|O|1995-08-06|1995-08-03|1995-08-14|DELIVER IN PERSON|TRUCK|ts at the re|
+3329|6|3|2|9|8154.00|0.00|0.02|N|O|1995-07-24|1995-08-02|1995-08-01|COLLECT COD|MAIL|lly final depo|
+3329|123|4|3|1|1023.12|0.04|0.08|N|O|1995-08-22|1995-09-28|1995-09-09|COLLECT COD|REG AIR|regular packages are carefull|
+3330|20|7|1|49|45080.98|0.05|0.01|R|F|1995-03-02|1995-03-03|1995-03-16|DELIVER IN PERSON|TRUCK|haggle carefully alongside of the bold r|
+3331|64|9|1|9|8676.54|0.08|0.07|A|F|1993-07-18|1993-07-03|1993-08-16|TAKE BACK RETURN|AIR|odolites. bold accounts|
+3331|21|2|2|38|34998.76|0.06|0.04|R|F|1993-07-24|1993-06-22|1993-08-23|NONE|AIR|ymptotes haggle across the ca|
+3331|3|10|3|26|23478.00|0.09|0.05|A|F|1993-08-05|1993-07-17|1993-08-29|DELIVER IN PERSON|MAIL|p asymptotes. carefully unusual in|
+3332|84|5|1|28|27554.24|0.10|0.02|R|F|1994-12-30|1995-01-16|1995-01-16|COLLECT COD|FOB|s against the carefully special multipl|
+3332|136|2|2|21|21758.73|0.08|0.04|R|F|1995-02-04|1995-01-08|1995-02-06|COLLECT COD|MAIL| quick packages sle|
+3332|134|5|3|27|27921.51|0.03|0.02|A|F|1994-12-10|1995-01-14|1994-12-11|TAKE BACK RETURN|FOB|ording to the slyly regula|
+3333|150|9|1|27|28354.05|0.06|0.08|A|F|1992-12-06|1992-10-26|1992-12-07|COLLECT COD|SHIP|s dazzle fluffil|
+3333|199|3|2|36|39570.84|0.08|0.07|R|F|1992-11-20|1992-11-06|1992-12-16|TAKE BACK RETURN|FOB|foxes sleep neve|
+3333|108|1|3|38|38307.80|0.05|0.05|A|F|1992-10-30|1992-11-03|1992-11-04|NONE|MAIL|ccounts promise bl|
+3333|113|4|4|49|49642.39|0.07|0.07|R|F|1992-10-02|1992-11-30|1992-10-12|DELIVER IN PERSON|MAIL|riously ironic r|
+3333|43|2|5|45|42436.80|0.07|0.08|A|F|1992-10-04|1992-11-08|1992-10-27|COLLECT COD|SHIP|dolites. quickly r|
+3334|187|8|1|20|21743.60|0.04|0.03|N|O|1996-05-21|1996-04-08|1996-05-26|TAKE BACK RETURN|AIR|uses nag furiously. instructions are ca|
+3334|190|1|2|7|7631.33|0.09|0.07|N|O|1996-04-28|1996-04-08|1996-05-25|NONE|SHIP|nts sublate slyly express pack|
+3335|105|10|1|13|13066.30|0.06|0.07|N|O|1996-01-20|1995-12-20|1996-02-09|COLLECT COD|REG AIR|out the special asymptotes|
+3335|31|2|2|44|40965.32|0.07|0.02|N|O|1996-01-05|1995-12-25|1996-01-18|DELIVER IN PERSON|SHIP|r packages cajole ac|
+3335|140|6|3|16|16642.24|0.01|0.06|N|O|1995-10-18|1995-12-08|1995-11-03|DELIVER IN PERSON|SHIP|g packages. carefully regular reque|
+3335|90|1|4|47|46534.23|0.10|0.03|N|O|1995-12-02|1995-11-19|1995-12-27|NONE|MAIL| quickly special ideas.|
+3360|174|4|1|31|33299.27|0.08|0.04|N|O|1998-04-24|1998-04-12|1998-05-23|COLLECT COD|REG AIR|quests. carefully even deposits wake acros|
+3360|91|3|2|29|28741.61|0.00|0.06|N|O|1998-04-15|1998-02-25|1998-05-13|TAKE BACK RETURN|FOB|press asymptotes. furiously final |
+3360|82|3|3|39|38301.12|0.08|0.03|N|O|1998-04-09|1998-04-20|1998-05-05|DELIVER IN PERSON|REG AIR|s. blithely express pinto bean|
+3360|117|7|4|29|29496.19|0.10|0.01|N|O|1998-05-19|1998-03-03|1998-06-09|TAKE BACK RETURN|FOB|hely gifts. spe|
+3360|58|6|5|4|3832.20|0.08|0.07|N|O|1998-02-27|1998-03-23|1998-03-28|COLLECT COD|SHIP|ly busy inst|
+3360|71|1|6|42|40784.94|0.04|0.01|N|O|1998-05-07|1998-04-18|1998-06-04|DELIVER IN PERSON|FOB|ages cajole. pending, |
+3361|144|5|1|6|6264.84|0.02|0.02|R|F|1992-10-02|1992-10-25|1992-10-05|DELIVER IN PERSON|FOB| packages sleep. furiously unus|
+3361|171|10|2|33|35348.61|0.01|0.02|R|F|1992-11-09|1992-10-15|1992-11-11|TAKE BACK RETURN|MAIL|uriously ironic accounts. ironic, ir|
+3361|191|5|3|31|33826.89|0.06|0.04|R|F|1992-08-29|1992-10-13|1992-09-08|NONE|FOB|ts. pending, regular accounts sleep fur|
+3362|22|5|1|14|12908.28|0.06|0.05|N|O|1995-08-01|1995-09-06|1995-08-22|NONE|FOB|even Tires|
+3362|195|6|2|41|44902.79|0.05|0.03|N|O|1995-10-31|1995-09-04|1995-11-17|COLLECT COD|REG AIR|ake alongside of the |
+3362|115|9|3|40|40604.40|0.05|0.06|N|O|1995-08-19|1995-10-17|1995-09-05|TAKE BACK RETURN|FOB|packages haggle furi|
+3362|2|7|4|3|2706.00|0.03|0.01|N|O|1995-08-26|1995-09-02|1995-09-17|NONE|SHIP|its cajole blithely excuses. de|
+3362|138|9|5|36|37372.68|0.06|0.00|N|O|1995-10-05|1995-08-28|1995-11-03|TAKE BACK RETURN|RAIL|es against the quickly permanent pint|
+3362|188|9|6|46|50056.28|0.09|0.05|N|O|1995-08-02|1995-10-12|1995-08-28|COLLECT COD|REG AIR|ly bold packages. regular deposits cajol|
+3363|10|3|1|42|38220.42|0.00|0.08|N|O|1995-11-09|1995-11-25|1995-11-15|TAKE BACK RETURN|RAIL| blithely final ideas nag after|
+3363|191|4|2|21|22914.99|0.08|0.08|N|O|1995-12-10|1995-10-28|1995-12-28|COLLECT COD|RAIL|he regular, brave deposits. f|
+3363|159|7|3|2|2118.30|0.01|0.07|N|O|1996-01-22|1995-12-01|1996-02-18|TAKE BACK RETURN|SHIP|uickly bold ide|
+3363|113|3|4|20|20262.20|0.07|0.06|N|O|1995-12-11|1995-11-15|1995-12-21|COLLECT COD|MAIL|carefully quiet excuses wake. sl|
+3363|200|4|5|4|4400.80|0.00|0.08|N|O|1995-10-30|1995-11-17|1995-11-22|COLLECT COD|FOB| ironic dependencie|
+3364|90|1|1|49|48514.41|0.03|0.05|N|O|1997-09-17|1997-08-23|1997-10-06|NONE|SHIP|d accounts? caref|
+3364|111|2|2|38|38422.18|0.02|0.02|N|O|1997-08-30|1997-09-12|1997-09-27|COLLECT COD|REG AIR| slyly express|
+3364|156|4|3|10|10561.50|0.00|0.01|N|O|1997-08-10|1997-08-24|1997-08-15|TAKE BACK RETURN|SHIP|g the accounts. final, busy accounts wi|
+3364|160|5|4|7|7421.12|0.10|0.05|N|O|1997-07-09|1997-08-01|1997-07-16|NONE|TRUCK|furiously regular ideas haggle furiously b|
+3364|81|2|5|3|2943.24|0.01|0.00|N|O|1997-10-19|1997-08-15|1997-10-28|TAKE BACK RETURN|TRUCK|c theodolites. blithely ir|
+3365|151|6|1|37|38892.55|0.02|0.08|R|F|1994-12-22|1995-02-07|1995-01-20|TAKE BACK RETURN|SHIP|requests. quickly pending instructions a|
+3365|167|2|2|37|39484.92|0.07|0.08|A|F|1994-11-24|1995-01-09|1994-11-27|NONE|REG AIR|oze blithely. furiously ironic theodolit|
+3365|115|6|3|13|13196.43|0.09|0.02|R|F|1995-02-25|1995-01-31|1995-03-16|NONE|RAIL|pths wake r|
+3365|176|4|4|49|52732.33|0.02|0.07|R|F|1995-01-03|1995-01-01|1995-01-18|COLLECT COD|MAIL|lyly unusual asymptotes. final|
+3365|16|3|5|2|1832.02|0.00|0.03|R|F|1995-02-04|1994-12-30|1995-03-06|TAKE BACK RETURN|FOB|es cajole fluffily pe|
+3365|126|5|6|24|24626.88|0.01|0.00|R|F|1995-02-27|1995-01-09|1995-03-27|DELIVER IN PERSON|REG AIR|into beans? carefully regula|
+3366|40|1|1|4|3760.16|0.07|0.01|N|O|1997-05-20|1997-06-25|1997-06-03|DELIVER IN PERSON|AIR| carefully about |
+3366|136|2|2|9|9325.17|0.00|0.08|N|O|1997-06-02|1997-07-05|1997-06-26|COLLECT COD|REG AIR|ackages sleep carefully across the bli|
+3367|41|10|1|27|25408.08|0.01|0.03|A|F|1993-04-13|1993-03-16|1993-04-26|NONE|RAIL|kly even instructions caj|
+3367|141|10|2|34|35398.76|0.04|0.08|A|F|1993-03-30|1993-02-23|1993-04-11|COLLECT COD|MAIL| accounts wake slyly |
+3367|120|7|3|38|38764.56|0.03|0.03|R|F|1993-03-13|1993-02-12|1993-03-31|NONE|RAIL|even packages sleep blithely slyly expr|
+3392|171|10|1|40|42846.80|0.01|0.01|N|O|1996-02-18|1995-12-16|1996-02-26|COLLECT COD|MAIL|ress instructions affix carefully. fur|
+3392|123|2|2|13|13300.56|0.09|0.02|N|O|1995-11-26|1996-01-17|1995-12-01|NONE|MAIL|across the fluffily bold deposits.|
+3392|127|10|3|34|34922.08|0.10|0.08|N|O|1996-01-20|1996-01-21|1996-01-24|DELIVER IN PERSON|MAIL|e carefully even braids. |
+3392|124|3|4|7|7168.84|0.08|0.05|N|O|1995-12-07|1996-01-09|1995-12-29|TAKE BACK RETURN|RAIL|as. express, final accounts dou|
+3393|117|7|1|16|16273.76|0.01|0.00|N|O|1995-07-17|1995-08-19|1995-08-04|COLLECT COD|TRUCK|uses. instructions after the blithely |
+3393|125|4|2|44|45105.28|0.08|0.04|N|O|1995-10-16|1995-08-05|1995-11-01|NONE|AIR|ld requests hag|
+3393|97|1|3|25|24927.25|0.07|0.02|N|O|1995-10-17|1995-08-12|1995-11-11|DELIVER IN PERSON|MAIL|ng excuses|
+3393|72|2|4|48|46659.36|0.06|0.06|N|O|1995-07-12|1995-09-15|1995-08-02|NONE|FOB| blithely final reques|
+3393|178|7|5|37|39892.29|0.07|0.02|N|O|1995-10-16|1995-08-19|1995-10-19|COLLECT COD|AIR|ss the slyly ironic pinto beans. ironic,|
+3393|62|7|6|17|16355.02|0.04|0.01|N|O|1995-08-15|1995-09-07|1995-09-10|COLLECT COD|MAIL|kly ironic deposits could|
+3394|155|6|1|33|34819.95|0.07|0.08|N|O|1996-08-07|1996-07-17|1996-09-02|TAKE BACK RETURN|SHIP|ideas alongside of th|
+3394|146|3|2|43|44984.02|0.08|0.03|N|O|1996-08-23|1996-07-20|1996-08-25|COLLECT COD|RAIL|hockey players. slyly regular requests afte|
+3394|88|9|3|26|25690.08|0.01|0.00|N|O|1996-08-08|1996-06-12|1996-09-05|TAKE BACK RETURN|RAIL|its use furiously. even, even account|
+3394|81|2|4|14|13735.12|0.08|0.00|N|O|1996-06-02|1996-07-02|1996-06-19|COLLECT COD|MAIL|e furiously final theodolites. furio|
+3394|127|8|5|30|30813.60|0.04|0.06|N|O|1996-05-12|1996-07-24|1996-05-19|COLLECT COD|REG AIR|t ideas according to the fluffily iro|
+3394|184|5|6|14|15178.52|0.05|0.05|N|O|1996-06-18|1996-06-24|1996-07-17|NONE|REG AIR|arefully regular do|
+3395|142|3|1|21|21884.94|0.03|0.06|R|F|1994-12-19|1995-01-13|1994-12-25|TAKE BACK RETURN|SHIP| careful dep|
+3395|36|2|2|38|35569.14|0.01|0.07|R|F|1995-01-13|1995-01-13|1995-01-25|COLLECT COD|SHIP| silent accounts are blithely|
+3395|43|4|3|43|40550.72|0.06|0.07|A|F|1994-12-13|1995-01-07|1994-12-14|COLLECT COD|AIR|ckages above the furiously regu|
+3395|122|1|4|39|39862.68|0.05|0.07|R|F|1994-12-03|1995-01-17|1994-12-10|NONE|AIR|riously unusual theodolites. fur|
+3396|128|7|1|34|34956.08|0.00|0.06|A|F|1994-05-30|1994-08-16|1994-06-11|NONE|AIR|. slyly unusual packages wak|
+3396|49|6|2|43|40808.72|0.03|0.08|A|F|1994-07-03|1994-08-09|1994-07-14|TAKE BACK RETURN|MAIL|cial packages cajole blithely around the |
+3396|138|4|3|9|9343.17|0.01|0.06|R|F|1994-07-01|1994-08-18|1994-07-21|DELIVER IN PERSON|AIR|usly special foxes. accounts wake careful|
+3396|75|3|4|32|31202.24|0.06|0.02|R|F|1994-08-07|1994-08-10|1994-09-05|COLLECT COD|TRUCK|osits are slyly. final, bold foxes s|
+3396|126|5|5|27|27705.24|0.02|0.01|A|F|1994-09-14|1994-07-26|1994-09-28|DELIVER IN PERSON|FOB| theodolites |
+3396|39|10|6|18|16902.54|0.10|0.00|A|F|1994-07-27|1994-06-26|1994-08-25|TAKE BACK RETURN|REG AIR|l requests haggle furiously along the fur|
+3396|198|2|7|31|34043.89|0.05|0.06|A|F|1994-06-07|1994-06-23|1994-06-19|TAKE BACK RETURN|REG AIR|l, express pinto beans. quic|
+3397|195|8|1|8|8761.52|0.07|0.01|A|F|1994-08-05|1994-08-11|1994-08-08|DELIVER IN PERSON|RAIL|y final foxes|
+3397|13|3|2|11|10043.11|0.00|0.07|A|F|1994-07-29|1994-09-18|1994-08-12|DELIVER IN PERSON|REG AIR|iously careful packages. s|
+3397|184|5|3|1|1084.18|0.07|0.05|R|F|1994-08-03|1994-07-30|1994-08-28|NONE|RAIL| regular packag|
+3397|86|7|4|33|32540.64|0.05|0.01|R|F|1994-09-04|1994-08-06|1994-09-22|COLLECT COD|RAIL|gular accounts. blithely re|
+3397|132|3|5|28|28899.64|0.05|0.05|R|F|1994-07-13|1994-08-26|1994-07-17|NONE|TRUCK|counts around the final reques|
+3398|173|4|1|1|1073.17|0.01|0.08|N|O|1996-11-22|1996-11-16|1996-12-09|COLLECT COD|MAIL| blithely final deposits.|
+3399|134|5|1|28|28955.64|0.09|0.05|N|O|1995-06-29|1995-05-19|1995-07-12|COLLECT COD|AIR|oggedly final theodolites grow. fi|
+3399|55|6|2|8|7640.40|0.01|0.05|A|F|1995-05-15|1995-04-19|1995-06-05|COLLECT COD|TRUCK|s use carefully carefully ir|
+3399|67|4|3|3|2901.18|0.03|0.00|N|F|1995-06-16|1995-04-04|1995-06-23|NONE|SHIP|hely pending dugouts |
+3399|14|5|4|21|19194.21|0.09|0.06|A|F|1995-03-12|1995-05-18|1995-03-28|TAKE BACK RETURN|MAIL|se final courts. exc|
+3424|181|2|1|39|42166.02|0.06|0.07|N|O|1996-11-03|1996-11-08|1996-11-23|DELIVER IN PERSON|MAIL|bits boost closely slyly p|
+3425|120|1|1|11|11221.32|0.03|0.08|N|O|1996-04-24|1996-05-29|1996-05-23|DELIVER IN PERSON|FOB|ckly final deposits use quickly?|
+3425|79|7|2|37|36225.59|0.06|0.03|N|O|1996-06-04|1996-05-09|1996-06-12|NONE|SHIP|as sleep carefully into the caref|
+3425|14|4|3|8|7312.08|0.06|0.08|N|O|1996-07-22|1996-06-07|1996-07-26|TAKE BACK RETURN|AIR|iously regular theodolites wake. s|
+3425|19|10|4|37|34003.37|0.04|0.01|N|O|1996-07-10|1996-05-10|1996-08-02|NONE|SHIP|ngside of the furiously thin dol|
+3425|79|9|5|48|46995.36|0.08|0.04|N|O|1996-04-14|1996-05-25|1996-04-23|TAKE BACK RETURN|AIR|uctions wake fluffily. care|
+3425|148|9|6|24|25155.36|0.05|0.04|N|O|1996-04-22|1996-06-24|1996-04-25|TAKE BACK RETURN|AIR|ajole blithely sl|
+3426|110|5|1|20|20202.20|0.05|0.04|N|O|1996-11-10|1996-12-24|1996-12-01|COLLECT COD|FOB|sits cajole blit|
+3426|14|4|2|19|17366.19|0.10|0.08|N|O|1996-11-02|1997-01-13|1996-11-15|DELIVER IN PERSON|RAIL|slyly special packages oug|
+3426|67|6|3|19|18374.14|0.08|0.05|N|O|1996-12-07|1996-12-15|1996-12-14|DELIVER IN PERSON|FOB|c accounts cajole carefu|
+3426|6|7|4|9|8154.00|0.09|0.05|N|O|1996-12-24|1997-01-14|1997-01-13|NONE|FOB|pecial theodolites haggle fluf|
+3426|49|6|5|31|29420.24|0.07|0.08|N|O|1996-11-11|1996-12-10|1996-12-10|DELIVER IN PERSON|SHIP| even sentiment|
+3427|54|5|1|41|39116.05|0.10|0.01|N|O|1997-09-11|1997-07-03|1997-10-04|COLLECT COD|RAIL|s the carefully|
+3427|189|10|2|24|26140.32|0.02|0.04|N|O|1997-07-01|1997-07-28|1997-07-30|NONE|SHIP|y bold, sly deposits. pendi|
+3427|139|5|3|40|41565.20|0.06|0.05|N|O|1997-06-12|1997-08-19|1997-06-23|COLLECT COD|MAIL|patterns cajole ca|
+3427|119|6|4|31|31592.41|0.08|0.04|N|O|1997-08-12|1997-07-26|1997-08-25|COLLECT COD|RAIL|s are carefull|
+3428|198|9|1|4|4392.76|0.00|0.03|N|O|1996-05-09|1996-06-13|1996-06-02|NONE|REG AIR|sly pending requests int|
+3428|118|9|2|35|35633.85|0.02|0.03|N|O|1996-05-01|1996-06-07|1996-05-20|COLLECT COD|TRUCK|ly regular pinto beans sleep|
+3428|136|7|3|47|48698.11|0.07|0.05|N|O|1996-04-16|1996-06-08|1996-05-05|NONE|REG AIR|y final pinto |
+3429|137|8|1|48|49782.24|0.06|0.02|N|O|1997-04-08|1997-03-09|1997-04-25|TAKE BACK RETURN|SHIP| haggle furiously ir|
+3429|59|7|2|15|14385.75|0.03|0.04|N|O|1997-02-04|1997-03-09|1997-03-01|TAKE BACK RETURN|TRUCK|beans are fu|
+3429|69|4|3|10|9690.60|0.05|0.07|N|O|1997-01-19|1997-02-22|1997-01-25|TAKE BACK RETURN|REG AIR|ackages. quickly e|
+3429|89|10|4|28|27694.24|0.10|0.07|N|O|1997-01-30|1997-03-18|1997-02-17|TAKE BACK RETURN|AIR|nstructions boost. thin|
+3429|165|6|5|45|47932.20|0.10|0.00|N|O|1997-04-21|1997-03-08|1997-05-05|COLLECT COD|REG AIR|ites poach a|
+3430|189|10|1|2|2178.36|0.07|0.06|R|F|1995-03-07|1995-01-28|1995-03-30|TAKE BACK RETURN|MAIL|sh furiously according to the evenly e|
+3430|81|2|2|32|31394.56|0.08|0.00|R|F|1995-01-17|1995-01-28|1995-02-06|NONE|TRUCK|egular instruction|
+3430|97|8|3|41|40880.69|0.06|0.04|R|F|1995-02-18|1995-02-21|1995-03-11|TAKE BACK RETURN|AIR|cuses. silent excuses h|
+3430|65|2|4|50|48253.00|0.01|0.00|R|F|1994-12-15|1995-03-03|1994-12-24|COLLECT COD|REG AIR|ironic theodolites. carefully regular pac|
+3430|95|9|5|5|4975.45|0.05|0.05|A|F|1995-04-02|1995-02-12|1995-04-08|DELIVER IN PERSON|FOB|even accounts haggle slyly bol|
+3430|171|10|6|15|16067.55|0.08|0.07|A|F|1995-02-01|1995-03-12|1995-02-04|COLLECT COD|SHIP|cajole around the accounts. qui|
+3430|52|7|7|23|21897.15|0.09|0.08|A|F|1995-03-06|1995-03-01|1995-03-10|COLLECT COD|MAIL|eas according to the|
+3431|180|8|1|41|44287.38|0.03|0.06|A|F|1993-09-26|1993-10-13|1993-10-22|NONE|AIR| sleep carefully ironically special|
+3456|111|8|1|34|34377.74|0.10|0.06|A|F|1993-08-29|1993-08-26|1993-09-07|TAKE BACK RETURN|SHIP|usy pinto beans b|
+3457|182|3|1|29|31383.22|0.03|0.02|R|F|1995-05-12|1995-07-13|1995-06-05|NONE|TRUCK|refully final excuses wake|
+3457|106|7|2|22|22134.20|0.06|0.01|N|O|1995-06-23|1995-06-16|1995-06-29|NONE|SHIP|packages nag furiously against|
+3457|109|2|3|7|7063.70|0.07|0.08|N|O|1995-08-14|1995-07-06|1995-08-18|COLLECT COD|SHIP| pending accounts along the|
+3457|1|2|4|24|21624.00|0.07|0.07|N|O|1995-08-03|1995-05-30|1995-08-14|TAKE BACK RETURN|REG AIR|tructions haggle alongsid|
+3457|109|4|5|42|42382.20|0.05|0.01|A|F|1995-06-12|1995-06-14|1995-06-14|COLLECT COD|MAIL|riously final instruc|
+3457|144|1|6|45|46986.30|0.08|0.01|N|O|1995-08-12|1995-07-18|1995-08-23|TAKE BACK RETURN|SHIP| packages. care|
+3457|167|4|7|9|9604.44|0.04|0.00|R|F|1995-05-29|1995-06-30|1995-06-12|DELIVER IN PERSON|FOB|quests. foxes sleep quickly|
+3458|133|4|1|48|49590.24|0.06|0.04|R|F|1995-03-17|1995-01-25|1995-03-28|TAKE BACK RETURN|AIR|iously pending dep|
+3458|50|3|2|46|43702.30|0.06|0.06|R|F|1995-03-08|1995-01-21|1995-03-10|TAKE BACK RETURN|SHIP|nod across the boldly even instruct|
+3458|143|4|3|36|37553.04|0.01|0.06|R|F|1995-04-20|1995-02-14|1995-05-09|TAKE BACK RETURN|REG AIR|s lose. blithely ironic requests boost|
+3458|16|10|4|16|14656.16|0.09|0.03|R|F|1995-03-01|1995-02-25|1995-03-06|TAKE BACK RETURN|AIR|s grow carefully. express, final grouc|
+3458|157|5|5|2|2114.30|0.09|0.03|A|F|1995-02-05|1995-02-01|1995-03-07|COLLECT COD|FOB|ironic packages haggle past the furiously |
+3458|142|1|6|6|6252.84|0.09|0.04|A|F|1995-03-10|1995-02-02|1995-03-23|TAKE BACK RETURN|AIR|dolites; regular theodolites cajole |
+3459|179|7|1|31|33454.27|0.06|0.01|A|F|1994-09-05|1994-10-20|1994-10-03|NONE|REG AIR|y regular pain|
+3459|130|9|2|30|30903.90|0.04|0.08|R|F|1994-11-22|1994-09-12|1994-12-11|NONE|REG AIR|nic theodolites; evenly i|
+3459|41|8|3|45|42346.80|0.04|0.05|A|F|1994-07-31|1994-09-09|1994-08-02|TAKE BACK RETURN|REG AIR|ntly speci|
+3459|69|10|4|10|9690.60|0.05|0.06|A|F|1994-10-06|1994-09-16|1994-11-03|TAKE BACK RETURN|REG AIR| furiously silent dolphi|
+3459|189|10|5|10|10891.80|0.02|0.02|R|F|1994-08-01|1994-10-17|1994-08-11|TAKE BACK RETURN|FOB|. blithely ironic pinto beans above|
+3460|11|1|1|40|36440.40|0.10|0.06|N|O|1995-12-28|1995-12-14|1996-01-02|NONE|REG AIR|odolites are slyly bold deposits|
+3460|74|4|2|3|2922.21|0.06|0.00|N|O|1996-01-19|1995-12-28|1996-01-31|COLLECT COD|AIR|er quickly |
+3460|35|1|3|40|37401.20|0.08|0.07|N|O|1995-10-29|1995-11-10|1995-11-24|TAKE BACK RETURN|REG AIR|o the even deposits|
+3460|95|8|4|50|49754.50|0.02|0.07|N|O|1996-01-30|1995-12-10|1996-02-06|DELIVER IN PERSON|SHIP|e slyly about the sly|
+3460|130|1|5|47|48416.11|0.08|0.05|N|O|1995-12-09|1995-11-12|1995-12-22|TAKE BACK RETURN|SHIP|es haggle slyly regular accounts. fi|
+3460|63|10|6|46|44300.76|0.03|0.07|N|O|1996-01-27|1996-01-01|1996-02-01|NONE|TRUCK|uses run among the carefully even deposits|
+3460|45|2|7|28|26461.12|0.00|0.01|N|O|1995-10-28|1995-11-13|1995-11-17|COLLECT COD|SHIP|inal, ironic instructions. carefully|
+3461|100|4|1|49|49004.90|0.06|0.06|A|F|1993-03-09|1993-04-16|1993-03-13|DELIVER IN PERSON|RAIL|ual request|
+3461|63|4|2|27|26002.62|0.06|0.06|A|F|1993-02-10|1993-03-02|1993-03-04|COLLECT COD|SHIP|ely unusual deposits. quickly ir|
+3461|39|5|3|44|41317.32|0.09|0.06|A|F|1993-05-20|1993-04-03|1993-05-27|COLLECT COD|RAIL| haggle quickly even ideas. fin|
+3461|95|7|4|41|40798.69|0.09|0.02|R|F|1993-02-19|1993-04-20|1993-02-21|NONE|TRUCK|heodolites. blithely ironi|
+3461|90|1|5|16|15841.44|0.08|0.06|A|F|1993-05-09|1993-04-29|1993-05-26|TAKE BACK RETURN|TRUCK| pending deposi|
+3461|167|2|6|24|25611.84|0.10|0.00|A|F|1993-06-01|1993-03-12|1993-06-20|TAKE BACK RETURN|MAIL|thely. carefully re|
+3462|151|3|1|4|4204.60|0.09|0.04|N|O|1997-06-12|1997-07-31|1997-06-16|COLLECT COD|RAIL|ackages. fu|
+3462|40|1|2|43|40421.72|0.08|0.03|N|O|1997-08-01|1997-07-18|1997-08-29|NONE|RAIL| carefully. final, final ideas sleep slyly|
+3462|129|4|3|6|6174.72|0.05|0.04|N|O|1997-06-02|1997-08-09|1997-06-30|NONE|RAIL|iously regular fo|
+3462|99|3|4|2|1998.18|0.09|0.07|N|O|1997-09-10|1997-08-08|1997-09-19|NONE|AIR|nic packages. even accounts alongside |
+3462|38|4|5|14|13132.42|0.01|0.02|N|O|1997-05-31|1997-07-05|1997-06-24|COLLECT COD|MAIL|yly. blithely bold theodolites wa|
+3463|61|10|1|45|43247.70|0.02|0.02|A|F|1993-10-30|1993-11-04|1993-11-08|DELIVER IN PERSON|FOB|nts are slyly |
+3463|98|1|2|43|42917.87|0.04|0.02|A|F|1993-10-28|1993-09-24|1993-11-03|DELIVER IN PERSON|FOB| across the |
+3488|160|5|1|1|1060.16|0.04|0.01|A|F|1995-03-06|1995-02-16|1995-03-23|DELIVER IN PERSON|FOB| final excuses. carefully even waters hagg|
+3488|104|9|2|48|48196.80|0.00|0.03|A|F|1995-03-29|1995-03-26|1995-04-28|COLLECT COD|SHIP|sly? final requests |
+3488|160|1|3|11|11661.76|0.03|0.08|R|F|1995-03-25|1995-02-08|1995-04-16|COLLECT COD|TRUCK|unusual re|
+3488|42|9|4|12|11304.48|0.05|0.07|R|F|1995-04-27|1995-02-16|1995-05-09|DELIVER IN PERSON|RAIL|e slyly; furiously final packages wak|
+3488|156|1|5|18|19010.70|0.09|0.06|A|F|1995-03-18|1995-03-19|1995-03-29|DELIVER IN PERSON|FOB|s the carefully r|
+3489|186|7|1|19|20637.42|0.09|0.05|A|F|1993-07-31|1993-10-26|1993-08-15|NONE|SHIP|c deposits alongside of the pending, fu|
+3489|29|4|2|46|42734.92|0.00|0.00|A|F|1993-08-02|1993-10-09|1993-08-10|TAKE BACK RETURN|TRUCK|xcuses? quickly stealthy dependenci|
+3490|92|6|1|43|42659.87|0.05|0.05|N|O|1997-08-04|1997-08-06|1997-08-14|TAKE BACK RETURN|SHIP|. even requests cajol|
+3490|86|7|2|50|49304.00|0.05|0.07|N|O|1997-06-27|1997-08-15|1997-06-28|NONE|RAIL| haggle carefu|
+3490|93|7|3|8|7944.72|0.10|0.04|N|O|1997-08-11|1997-07-25|1997-08-28|COLLECT COD|MAIL|inal deposits use furiousl|
+3491|154|2|1|28|29516.20|0.04|0.03|N|O|1998-09-29|1998-09-08|1998-10-23|COLLECT COD|FOB|ccounts. sly|
+3491|122|3|2|22|22486.64|0.08|0.02|N|O|1998-08-19|1998-08-22|1998-09-03|TAKE BACK RETURN|REG AIR| grow against the boldly pending pinto bea|
+3492|156|7|1|3|3168.45|0.02|0.08|R|F|1994-11-26|1994-12-28|1994-12-19|COLLECT COD|REG AIR|the deposits. carefully |
+3492|126|9|2|7|7182.84|0.04|0.00|R|F|1995-03-10|1995-01-03|1995-03-16|COLLECT COD|FOB|thely regular dolphi|
+3492|109|10|3|34|34309.40|0.05|0.06|A|F|1994-12-07|1994-12-29|1994-12-24|COLLECT COD|AIR| unusual requests. ir|
+3492|147|6|4|30|31414.20|0.02|0.06|A|F|1995-01-29|1995-01-02|1995-02-13|DELIVER IN PERSON|MAIL| detect furiously permanent, unusual accou|
+3492|122|1|5|47|48039.64|0.09|0.07|R|F|1995-03-24|1994-12-28|1995-03-29|NONE|REG AIR|deposits. quickly express |
+3492|22|7|6|47|43334.94|0.04|0.07|R|F|1994-12-12|1995-01-18|1994-12-26|COLLECT COD|RAIL|ronic instructions u|
+3493|93|6|1|31|30785.79|0.06|0.07|R|F|1993-10-22|1993-10-12|1993-11-07|DELIVER IN PERSON|REG AIR|ructions. slyly regular accounts across the|
+3493|132|3|2|10|10321.30|0.02|0.06|R|F|1993-08-27|1993-10-07|1993-09-23|COLLECT COD|TRUCK|hall have to integ|
+3494|117|1|1|40|40684.40|0.05|0.04|R|F|1993-07-10|1993-06-01|1993-07-25|TAKE BACK RETURN|TRUCK|lites haggle furiously about the fin|
+3494|75|6|2|23|22426.61|0.10|0.01|A|F|1993-06-19|1993-06-04|1993-07-14|NONE|FOB|osits nag |
+3494|198|2|3|40|43927.60|0.02|0.08|A|F|1993-05-30|1993-07-02|1993-06-20|TAKE BACK RETURN|MAIL|uests cajole blithely|
+3494|77|8|4|30|29312.10|0.04|0.03|R|F|1993-07-01|1993-06-08|1993-07-15|TAKE BACK RETURN|TRUCK|ns are quickly regular, |
+3495|28|3|1|20|18560.40|0.10|0.03|N|O|1996-04-24|1996-05-18|1996-05-01|TAKE BACK RETURN|RAIL|posits are carefully; forges cajole qui|
+3495|173|1|2|24|25756.08|0.05|0.02|N|O|1996-03-22|1996-04-10|1996-04-07|DELIVER IN PERSON|RAIL|ic, final pains along the even request|
+3495|199|10|3|16|17587.04|0.08|0.02|N|O|1996-03-30|1996-04-02|1996-04-12|TAKE BACK RETURN|AIR|y bold dependencies; blithely idle sautern|
+3520|28|1|1|30|27840.60|0.04|0.02|N|O|1997-11-11|1997-10-02|1997-12-06|COLLECT COD|SHIP|deas should solve blithely among the ironi|
+3520|167|4|2|38|40552.08|0.00|0.04|N|O|1997-08-14|1997-10-26|1997-09-09|NONE|RAIL|yly final packages according to the quickl|
+3520|106|9|3|5|5030.50|0.01|0.02|N|O|1997-11-13|1997-09-22|1997-12-09|NONE|MAIL|ly even ideas haggle |
+3520|64|5|4|41|39526.46|0.01|0.01|N|O|1997-08-06|1997-09-20|1997-08-20|TAKE BACK RETURN|AIR| carefully pendi|
+3520|163|10|5|35|37210.60|0.02|0.02|N|O|1997-09-16|1997-09-03|1997-09-24|DELIVER IN PERSON|FOB|s nag carefully. sometimes unusual account|
+3521|59|4|1|48|46034.40|0.09|0.03|A|F|1993-01-03|1992-12-31|1993-01-22|NONE|AIR|ses use. furiously express ideas wake f|
+3521|131|2|2|2|2062.26|0.05|0.06|R|F|1993-01-29|1992-12-20|1993-02-23|NONE|MAIL|refully duri|
+3521|178|8|3|38|40970.46|0.00|0.08|A|F|1993-02-15|1992-12-10|1993-03-10|COLLECT COD|FOB|ges hang q|
+3521|144|7|4|26|27147.64|0.02|0.08|R|F|1993-01-04|1993-01-20|1993-01-17|DELIVER IN PERSON|AIR|onic dependencies haggle. fur|
+3521|36|7|5|28|26208.84|0.10|0.01|A|F|1993-01-06|1993-01-22|1993-02-02|TAKE BACK RETURN|FOB|e slyly above the slyly final|
+3522|4|9|1|6|5424.00|0.08|0.03|A|F|1995-01-21|1994-12-09|1995-01-23|NONE|SHIP|tes snooze |
+3522|87|8|2|48|47379.84|0.00|0.03|R|F|1994-12-05|1994-10-30|1994-12-26|TAKE BACK RETURN|SHIP|ve the quickly special packages|
+3522|157|2|3|46|48628.90|0.09|0.02|A|F|1994-11-12|1994-11-30|1994-11-20|NONE|AIR|d the express, silent foxes. blit|
+3522|130|9|4|7|7210.91|0.10|0.02|A|F|1994-10-31|1994-11-19|1994-11-28|NONE|TRUCK|e stealthil|
+3522|50|9|5|27|25651.35|0.02|0.05|R|F|1994-11-29|1994-12-15|1994-12-08|COLLECT COD|REG AIR|ic tithes. car|
+3522|158|10|6|18|19046.70|0.01|0.03|A|F|1994-11-16|1994-10-29|1994-11-29|COLLECT COD|RAIL|sits wake carefully pen|
+3523|25|6|1|15|13875.30|0.06|0.02|N|O|1998-06-26|1998-05-22|1998-07-24|COLLECT COD|REG AIR|se slyly pending, sp|
+3523|133|9|2|4|4132.52|0.03|0.06|N|O|1998-05-08|1998-05-18|1998-05-25|TAKE BACK RETURN|MAIL|ts. final accounts detect furiously along |
+3523|50|7|3|24|22801.20|0.07|0.04|N|O|1998-08-02|1998-06-22|1998-08-27|COLLECT COD|FOB|ke according to the doggedly re|
+3523|192|4|4|36|39318.84|0.06|0.08|N|O|1998-05-26|1998-06-04|1998-06-25|DELIVER IN PERSON|SHIP|accounts. fluffily regu|
+3523|134|5|5|48|49638.24|0.00|0.01|N|O|1998-07-22|1998-06-25|1998-08-19|DELIVER IN PERSON|AIR| regular requests|
+3524|137|8|1|5|5185.65|0.01|0.04|R|F|1992-05-23|1992-07-25|1992-06-19|DELIVER IN PERSON|RAIL|ts whithout the bold depende|
+3524|143|6|2|17|17733.38|0.09|0.08|A|F|1992-09-01|1992-07-17|1992-09-05|DELIVER IN PERSON|FOB|g, final epitaphs about the pinto |
+3525|46|7|1|12|11352.48|0.01|0.03|N|O|1996-03-08|1996-03-18|1996-03-16|NONE|TRUCK|lar excuses wake carefull|
+3525|138|9|2|27|28029.51|0.03|0.03|N|O|1995-12-30|1996-01-23|1996-01-02|DELIVER IN PERSON|SHIP|y slyly special asymptotes|
+3525|75|5|3|31|30227.17|0.00|0.03|N|O|1996-03-08|1996-02-27|1996-03-13|COLLECT COD|TRUCK|he careful|
+3525|184|5|4|28|30357.04|0.03|0.02|N|O|1996-01-22|1996-02-08|1996-01-27|COLLECT COD|FOB| nag according |
+3526|98|9|1|11|10978.99|0.02|0.03|R|F|1995-05-23|1995-05-28|1995-05-24|NONE|TRUCK|ges. furiously regular d|
+3526|117|7|2|23|23393.53|0.03|0.04|A|F|1995-05-01|1995-05-31|1995-05-25|DELIVER IN PERSON|FOB|special, regular packages cajole. |
+3526|33|9|3|20|18660.60|0.05|0.08|N|F|1995-06-16|1995-04-26|1995-06-22|DELIVER IN PERSON|REG AIR|kages. bold, special requests detect sl|
+3527|102|7|1|47|47098.70|0.07|0.02|N|O|1997-07-14|1997-07-29|1997-07-21|DELIVER IN PERSON|RAIL|unts. express re|
+3527|26|9|2|33|30558.66|0.01|0.02|N|O|1997-09-25|1997-09-17|1997-10-12|NONE|FOB|kly alongside of |
+3527|162|7|3|50|53108.00|0.09|0.07|N|O|1997-07-17|1997-08-03|1997-07-29|DELIVER IN PERSON|SHIP|e even accounts was about th|
+3527|128|3|4|17|17478.04|0.02|0.05|N|O|1997-07-30|1997-09-01|1997-08-17|COLLECT COD|MAIL|ular instruction|
+3552|197|8|1|18|19749.42|0.01|0.07|N|O|1997-08-11|1997-07-14|1997-08-15|DELIVER IN PERSON|TRUCK|s deposits against the blithely unusual pin|
+3552|90|1|2|44|43563.96|0.01|0.00|N|O|1997-08-08|1997-06-15|1997-08-29|COLLECT COD|FOB|ns after the blithely reg|
+3552|161|6|3|36|38201.76|0.04|0.08|N|O|1997-06-29|1997-06-24|1997-07-21|COLLECT COD|TRUCK|ly regular theodolites. fin|
+3553|143|10|1|4|4172.56|0.05|0.01|R|F|1994-06-13|1994-07-10|1994-07-03|COLLECT COD|RAIL|olites boost bli|
+3553|65|4|2|26|25091.56|0.05|0.08|A|F|1994-08-06|1994-07-30|1994-08-23|DELIVER IN PERSON|MAIL|fily special p|
+3553|22|5|3|18|16596.36|0.04|0.03|A|F|1994-07-03|1994-06-30|1994-07-07|COLLECT COD|RAIL|. quickly ironic|
+3553|32|8|4|40|37281.20|0.06|0.00|A|F|1994-09-14|1994-06-26|1994-09-25|NONE|RAIL| slyly pending asymptotes against the furi|
+3553|157|2|5|36|38057.40|0.06|0.08|R|F|1994-08-12|1994-06-25|1994-09-06|DELIVER IN PERSON|TRUCK| realms. pending, bold theodolites |
+3554|175|5|1|32|34405.44|0.01|0.05|N|O|1995-09-28|1995-09-01|1995-10-07|NONE|RAIL|. blithely ironic t|
+3554|145|6|2|18|18812.52|0.03|0.00|N|O|1995-09-11|1995-08-12|1995-10-04|DELIVER IN PERSON|REG AIR| haggle. furiously fluffy requests ac|
+3554|192|3|3|41|44779.79|0.02|0.01|N|O|1995-07-13|1995-08-28|1995-07-27|DELIVER IN PERSON|MAIL|ent dependencies. sly|
+3555|166|3|1|11|11727.76|0.05|0.02|N|O|1996-09-25|1996-10-01|1996-10-03|NONE|FOB|oost caref|
+3555|79|10|2|15|14686.05|0.03|0.08|N|O|1996-07-13|1996-09-01|1996-08-02|TAKE BACK RETURN|RAIL|y across the pending a|
+3555|43|2|3|25|23576.00|0.09|0.07|N|O|1996-10-01|1996-08-23|1996-10-24|TAKE BACK RETURN|MAIL|sual packages. quickly |
+3555|5|6|4|19|17195.00|0.00|0.05|N|O|1996-09-08|1996-09-14|1996-10-01|COLLECT COD|REG AIR|leep special theodolit|
+3555|33|4|5|29|27057.87|0.07|0.04|N|O|1996-08-02|1996-09-04|1996-08-08|DELIVER IN PERSON|TRUCK|deas. carefully s|
+3555|28|3|6|33|30624.66|0.04|0.08|N|O|1996-09-20|1996-09-23|1996-10-05|TAKE BACK RETURN|AIR|fluffily regular a|
+3555|126|5|7|9|9235.08|0.07|0.02|N|O|1996-10-13|1996-10-02|1996-10-22|NONE|SHIP|are. slyly final foxes acro|
+3556|142|9|1|45|46896.30|0.05|0.06|A|F|1992-10-14|1992-12-21|1992-10-16|NONE|TRUCK|ckages boost quickl|
+3556|31|2|2|43|40034.29|0.02|0.06|R|F|1993-01-18|1992-11-09|1993-02-04|NONE|FOB|wake carefull|
+3556|87|8|3|28|27638.24|0.10|0.04|A|F|1993-01-06|1992-11-27|1993-01-16|NONE|MAIL|refully final instructions? ironic packa|
+3557|175|3|1|41|44081.97|0.01|0.07|R|F|1993-01-30|1992-12-31|1993-02-18|COLLECT COD|FOB|ideas breach c|
+3557|129|10|2|37|38077.44|0.03|0.05|R|F|1993-02-16|1993-01-05|1993-03-15|DELIVER IN PERSON|RAIL|gside of the ca|
+3558|87|8|1|8|7896.64|0.01|0.03|N|O|1996-05-31|1996-05-26|1996-06-25|COLLECT COD|AIR|? even requests sle|
+3558|10|7|2|28|25480.28|0.02|0.08|N|O|1996-06-02|1996-04-18|1996-06-24|COLLECT COD|TRUCK|l deposits |
+3558|187|8|3|3|3261.54|0.03|0.06|N|O|1996-05-19|1996-04-28|1996-05-26|DELIVER IN PERSON|RAIL|l, final deposits haggle. fina|
+3558|91|5|4|22|21803.98|0.06|0.03|N|O|1996-04-27|1996-04-19|1996-04-30|DELIVER IN PERSON|SHIP|refully ironic theodolites are fu|
+3558|29|8|5|38|35302.76|0.03|0.08|N|O|1996-05-29|1996-05-02|1996-06-09|COLLECT COD|RAIL|refully permanently iron|
+3558|72|1|6|17|16525.19|0.07|0.07|N|O|1996-03-14|1996-05-04|1996-04-05|NONE|RAIL|ithely unusual packa|
+3559|90|1|1|29|28712.61|0.00|0.07|R|F|1992-12-10|1992-12-03|1992-12-20|COLLECT COD|REG AIR|l, regular accounts wake flu|
+3584|11|8|1|4|3644.04|0.04|0.08|N|O|1997-08-16|1997-10-31|1997-08-28|DELIVER IN PERSON|TRUCK|nal packag|
+3584|160|8|2|23|24383.68|0.00|0.03|N|O|1997-09-10|1997-10-15|1997-09-30|COLLECT COD|TRUCK|l platelets until the asymptotes |
+3584|24|5|3|6|5544.12|0.03|0.06|N|O|1997-10-28|1997-11-09|1997-11-24|TAKE BACK RETURN|MAIL|deposits across the|
+3584|146|5|4|11|11507.54|0.06|0.02|N|O|1997-11-27|1997-10-15|1997-12-08|NONE|REG AIR|lithely slyly |
+3584|18|5|5|39|35802.39|0.09|0.07|N|O|1997-09-20|1997-10-31|1997-10-06|COLLECT COD|AIR|eposits. carefu|
+3585|122|1|1|21|21464.52|0.05|0.04|A|F|1994-12-04|1994-12-25|1995-01-01|TAKE BACK RETURN|TRUCK|ounts use. express, final platelets us|
+3585|19|10|2|40|36760.40|0.03|0.00|R|F|1995-01-22|1995-01-17|1995-02-07|TAKE BACK RETURN|RAIL|elets affix. even asymptotes play care|
+3585|112|2|3|11|11133.21|0.01|0.04|R|F|1995-01-04|1995-02-14|1995-01-15|NONE|MAIL|even packages|
+3585|48|1|4|33|31285.32|0.08|0.08|A|F|1994-12-14|1995-01-19|1994-12-22|NONE|RAIL|ironic dependencies serve furi|
+3585|25|8|5|13|12025.26|0.06|0.07|R|F|1995-03-15|1995-01-22|1995-03-17|DELIVER IN PERSON|AIR|ccording to the foxes. slyly iro|
+3585|94|7|6|7|6958.63|0.10|0.02|A|F|1994-12-13|1995-01-20|1995-01-05|TAKE BACK RETURN|TRUCK|dependencies sleep un|
+3585|42|1|7|45|42391.80|0.03|0.00|A|F|1995-01-20|1995-02-19|1995-02-11|DELIVER IN PERSON|MAIL|are blithely c|
+3586|194|7|1|2|2188.38|0.03|0.08|R|F|1994-02-10|1994-01-07|1994-03-03|DELIVER IN PERSON|RAIL|he even, unusual decoy|
+3586|84|5|2|29|28538.32|0.04|0.07|R|F|1994-03-06|1994-03-02|1994-03-13|DELIVER IN PERSON|RAIL| slyly unusual i|
+3586|58|3|3|2|1916.10|0.03|0.06|R|F|1994-03-22|1994-02-20|1994-04-08|NONE|REG AIR|unts. slyly final ideas agai|
+3586|84|5|4|33|32474.64|0.06|0.01|R|F|1994-01-24|1994-02-09|1994-02-07|NONE|TRUCK|refully across the fur|
+3586|108|1|5|8|8064.80|0.06|0.02|A|F|1994-03-29|1994-02-26|1994-04-02|NONE|FOB|theodolites hagg|
+3586|99|1|6|8|7992.72|0.09|0.01|A|F|1994-03-18|1994-01-17|1994-04-06|DELIVER IN PERSON|RAIL| ironic pinto beans cajole carefully theo|
+3586|123|4|7|33|33762.96|0.05|0.04|A|F|1994-02-11|1994-01-15|1994-03-03|NONE|REG AIR|iously regular pinto beans integrate|
+3587|197|10|1|5|5485.95|0.09|0.07|N|O|1996-09-03|1996-07-05|1996-09-11|DELIVER IN PERSON|SHIP|ithely regular decoys above the |
+3587|132|8|2|48|49542.24|0.00|0.03|N|O|1996-08-02|1996-07-02|1996-08-05|TAKE BACK RETURN|MAIL|beans. blithely final depe|
+3587|151|3|3|36|37841.40|0.05|0.05|N|O|1996-07-26|1996-06-16|1996-08-23|TAKE BACK RETURN|MAIL|ully regular excuse|
+3587|124|9|4|31|31747.72|0.03|0.01|N|O|1996-07-21|1996-07-01|1996-07-23|COLLECT COD|SHIP|press fluffily regul|
+3587|70|7|5|12|11640.84|0.06|0.03|N|O|1996-08-30|1996-07-04|1996-09-22|DELIVER IN PERSON|RAIL|g the even pinto beans. special,|
+3587|107|2|6|16|16113.60|0.01|0.03|N|O|1996-05-11|1996-06-19|1996-06-04|COLLECT COD|FOB|y ruthless dolphins to |
+3587|74|2|7|23|22403.61|0.07|0.05|N|O|1996-08-30|1996-07-01|1996-09-10|COLLECT COD|FOB|l multipliers sleep theodolites-- slyly |
+3588|91|5|1|28|27750.52|0.04|0.08|R|F|1995-05-03|1995-05-03|1995-05-14|DELIVER IN PERSON|TRUCK|special pinto beans cajole slyly. slyly |
+3588|88|9|2|6|5928.48|0.06|0.08|A|F|1995-04-09|1995-05-30|1995-04-10|TAKE BACK RETURN|MAIL|s. fluffily fluf|
+3588|159|10|3|45|47661.75|0.04|0.02|R|F|1995-05-07|1995-05-04|1995-05-28|TAKE BACK RETURN|TRUCK|ecial pains integrate blithely. reques|
+3588|127|10|4|22|22596.64|0.05|0.00|A|F|1995-04-08|1995-05-06|1995-04-27|NONE|RAIL|inal accounts. pending, bo|
+3588|55|3|5|28|26741.40|0.03|0.03|A|F|1995-04-23|1995-05-25|1995-04-28|DELIVER IN PERSON|TRUCK| express sheaves. unusual theodo|
+3588|110|3|6|37|37374.07|0.08|0.04|N|F|1995-06-17|1995-05-25|1995-06-24|TAKE BACK RETURN|RAIL|xcuses sleep quickly along th|
+3588|39|5|7|46|43195.38|0.08|0.07|A|F|1995-06-06|1995-05-08|1995-06-08|NONE|AIR| slyly ironic deposits sublate ab|
+3589|37|3|1|42|39355.26|0.08|0.08|R|F|1994-08-11|1994-07-17|1994-08-23|DELIVER IN PERSON|AIR|he blithely unusual pac|
+3590|176|6|1|10|10761.70|0.08|0.00|N|O|1995-07-17|1995-06-26|1995-08-12|TAKE BACK RETURN|SHIP|t the quickly ironic|
+3590|95|6|2|19|18906.71|0.03|0.03|N|O|1995-08-02|1995-06-20|1995-08-08|NONE|SHIP|special pinto beans. blithely reg|
+3590|96|9|3|43|42831.87|0.07|0.06|N|O|1995-07-12|1995-07-25|1995-07-16|DELIVER IN PERSON|SHIP|s could have to use|
+3590|56|8|4|26|24857.30|0.01|0.03|N|O|1995-07-08|1995-06-17|1995-08-02|DELIVER IN PERSON|SHIP|arefully along th|
+3590|191|2|5|37|40374.03|0.00|0.08|N|O|1995-09-01|1995-06-29|1995-09-10|NONE|SHIP|ccounts above the silent waters thrash f|
+3590|119|10|6|31|31592.41|0.03|0.01|N|O|1995-06-24|1995-07-12|1995-06-25|DELIVER IN PERSON|REG AIR|ve furiously final instructions. slyly regu|
+3590|194|7|7|44|48144.36|0.05|0.04|N|F|1995-06-07|1995-06-15|1995-06-27|NONE|MAIL|s sleep after the regular platelets. blit|
+3591|29|8|1|21|19509.42|0.03|0.03|A|F|1994-02-25|1994-02-02|1994-03-05|DELIVER IN PERSON|TRUCK|structions against |
+3591|69|6|2|24|23257.44|0.04|0.04|R|F|1993-12-26|1994-01-07|1994-01-25|COLLECT COD|FOB|ages. slyly regular dependencies cajo|
+3591|164|9|3|4|4256.64|0.01|0.03|A|F|1994-04-04|1994-02-19|1994-05-02|DELIVER IN PERSON|RAIL|he final packages. deposits serve quick|
+3591|153|4|4|49|51604.35|0.01|0.00|A|F|1994-03-21|1994-01-26|1994-03-28|COLLECT COD|AIR| mold slyly. bl|
+3616|197|9|1|30|32915.70|0.01|0.00|A|F|1994-05-05|1994-04-24|1994-05-12|TAKE BACK RETURN|FOB|ly ironic accounts unwind b|
+3616|138|9|2|28|29067.64|0.08|0.06|R|F|1994-02-20|1994-04-18|1994-03-05|DELIVER IN PERSON|REG AIR|ironic packages. furiously ev|
+3617|117|8|1|46|46787.06|0.03|0.02|N|O|1996-05-19|1996-05-14|1996-06-11|NONE|RAIL|ar theodolites. regu|
+3617|98|9|2|16|15969.44|0.05|0.02|N|O|1996-05-08|1996-06-03|1996-05-19|COLLECT COD|RAIL| slyly on th|
+3617|98|2|3|32|31938.88|0.00|0.06|N|O|1996-04-20|1996-06-07|1996-05-19|DELIVER IN PERSON|MAIL|uriously against the express accounts. ex|
+3617|41|10|4|22|20702.88|0.10|0.05|N|O|1996-07-11|1996-05-02|1996-07-25|NONE|REG AIR|uffily even accounts. packages sleep blithe|
+3617|137|8|5|11|11408.43|0.08|0.05|N|O|1996-07-16|1996-04-23|1996-07-28|COLLECT COD|MAIL|ly quickly even requests. final|
+3618|140|1|1|38|39525.32|0.08|0.00|N|O|1997-12-22|1998-02-23|1998-01-03|TAKE BACK RETURN|TRUCK|nts haggle fluffily above the regular |
+3618|144|5|2|48|50118.72|0.04|0.00|N|O|1998-03-12|1998-02-13|1998-03-29|DELIVER IN PERSON|TRUCK|tructions atop the ironi|
+3618|63|2|3|24|23113.44|0.01|0.04|N|O|1998-01-26|1998-01-15|1998-02-17|TAKE BACK RETURN|AIR|xpress acc|
+3618|161|2|4|26|27590.16|0.01|0.05|N|O|1998-03-23|1998-01-24|1998-04-15|DELIVER IN PERSON|AIR|iously regular deposits cajole ruthless|
+3619|96|7|1|49|48808.41|0.01|0.08|N|O|1997-01-22|1996-12-21|1997-02-17|TAKE BACK RETURN|MAIL| waters. furiously even deposits |
+3619|116|10|2|27|27434.97|0.08|0.04|N|O|1996-12-12|1997-01-18|1996-12-18|TAKE BACK RETURN|SHIP|pecial accounts haggle care|
+3619|48|7|3|46|43609.84|0.08|0.03|N|O|1997-01-31|1997-01-27|1997-02-11|NONE|SHIP|press, expres|
+3619|93|6|4|18|17875.62|0.04|0.02|N|O|1997-03-18|1996-12-24|1997-03-21|COLLECT COD|AIR|eodolites |
+3619|120|10|5|38|38764.56|0.05|0.08|N|O|1996-12-08|1997-02-03|1997-01-07|NONE|RAIL|theodolites detect abo|
+3619|152|3|6|43|45242.45|0.01|0.01|N|O|1997-01-25|1997-01-06|1997-02-07|COLLECT COD|RAIL| bold, even|
+3620|59|7|1|41|39321.05|0.03|0.08|N|O|1997-03-21|1997-04-20|1997-03-30|COLLECT COD|FOB|t attainments cajole qui|
+3620|167|4|2|16|17074.56|0.00|0.06|N|O|1997-05-17|1997-05-08|1997-06-03|COLLECT COD|SHIP|s. even, pending in|
+3621|17|8|1|29|26593.29|0.02|0.06|A|F|1993-08-03|1993-07-08|1993-08-10|DELIVER IN PERSON|FOB|al requests. fl|
+3621|93|5|2|13|12910.17|0.09|0.04|R|F|1993-08-30|1993-06-30|1993-09-01|NONE|REG AIR|r the unusual packages. brave theodoli|
+3621|164|9|3|45|47887.20|0.07|0.07|R|F|1993-08-09|1993-06-18|1993-09-05|DELIVER IN PERSON|AIR| doubt about the bold deposits. carefully|
+3621|44|3|4|20|18880.80|0.05|0.04|R|F|1993-05-27|1993-07-04|1993-06-22|TAKE BACK RETURN|SHIP|gular accounts use carefully with|
+3622|175|6|1|47|50532.99|0.09|0.00|N|O|1996-02-24|1996-02-22|1996-03-12|TAKE BACK RETURN|TRUCK|are careful|
+3622|89|10|2|4|3956.32|0.04|0.04|N|O|1996-02-03|1996-02-19|1996-02-16|TAKE BACK RETURN|TRUCK|lithely brave foxes. furi|
+3622|190|1|3|46|50148.74|0.07|0.07|N|O|1995-12-18|1996-01-23|1996-01-12|TAKE BACK RETURN|AIR|sits wake. blithe|
+3622|177|8|4|9|9694.53|0.08|0.05|N|O|1995-12-12|1996-02-09|1995-12-13|TAKE BACK RETURN|SHIP|arefully. furiously regular ideas n|
+3623|80|10|1|32|31362.56|0.05|0.00|N|O|1997-04-18|1997-03-15|1997-05-09|COLLECT COD|SHIP| courts. furiously regular ideas b|
+3623|117|4|2|33|33564.63|0.08|0.01|N|O|1997-03-17|1997-02-13|1997-04-02|TAKE BACK RETURN|TRUCK|odolites. blithely spe|
+3623|24|7|3|21|19404.42|0.02|0.02|N|O|1997-01-19|1997-03-18|1997-01-24|NONE|FOB|ress ideas are furio|
+3623|165|2|4|42|44736.72|0.05|0.06|N|O|1997-01-11|1997-03-24|1997-01-21|COLLECT COD|RAIL|g to the slyly regular packa|
+3623|88|9|5|30|29642.40|0.10|0.04|N|O|1997-04-04|1997-03-03|1997-05-01|NONE|RAIL| ironic somas sleep fluffily|
+3623|186|7|6|7|7603.26|0.01|0.02|N|O|1997-01-05|1997-03-26|1997-01-26|NONE|TRUCK|aves. slyly special packages cajole. fu|
+3623|140|6|7|13|13521.82|0.03|0.08|N|O|1997-01-02|1997-02-26|1997-01-26|DELIVER IN PERSON|SHIP|deas. furiously expres|
+3648|144|5|1|16|16706.24|0.02|0.06|A|F|1993-08-14|1993-08-14|1993-08-15|COLLECT COD|FOB|s nag packages.|
+3648|105|2|2|30|30153.00|0.00|0.01|R|F|1993-08-31|1993-09-06|1993-09-06|DELIVER IN PERSON|FOB| above the somas boost furious|
+3648|46|7|3|34|32165.36|0.10|0.00|A|F|1993-08-21|1993-07-25|1993-09-15|DELIVER IN PERSON|FOB| deposits are furiously. careful, |
+3648|13|10|4|16|14608.16|0.06|0.03|R|F|1993-07-27|1993-08-26|1993-08-24|DELIVER IN PERSON|FOB|uriously stealthy deposits haggle furi|
+3648|117|7|5|25|25427.75|0.06|0.03|R|F|1993-08-15|1993-08-25|1993-09-09|TAKE BACK RETURN|TRUCK|s requests. silent asymp|
+3648|169|10|6|14|14968.24|0.08|0.06|R|F|1993-10-02|1993-08-26|1993-10-09|COLLECT COD|AIR|sly pending excuses. carefully i|
+3648|195|6|7|49|53664.31|0.09|0.03|R|F|1993-06-27|1993-07-27|1993-07-24|TAKE BACK RETURN|FOB|egular instructions. slyly regular pinto|
+3649|5|6|1|25|22625.00|0.10|0.04|A|F|1994-10-27|1994-08-23|1994-11-05|TAKE BACK RETURN|TRUCK|special re|
+3649|89|10|2|23|22748.84|0.08|0.00|R|F|1994-09-26|1994-10-01|1994-09-28|NONE|REG AIR|rs promise blithe|
+3649|70|7|3|14|13580.98|0.02|0.04|A|F|1994-09-19|1994-08-17|1994-10-12|DELIVER IN PERSON|TRUCK|ithely bold accounts wake |
+3649|76|4|4|40|39042.80|0.00|0.08|R|F|1994-07-20|1994-08-30|1994-08-14|TAKE BACK RETURN|RAIL|luffy somas sleep quickly-- ironic de|
+3649|100|1|5|24|24002.40|0.05|0.03|A|F|1994-07-07|1994-08-20|1994-07-27|TAKE BACK RETURN|FOB|c accounts. quickly final theodo|
+3649|122|3|6|3|3066.36|0.10|0.04|A|F|1994-07-17|1994-08-10|1994-08-03|NONE|FOB|lly bold requests nag; |
+3650|136|2|1|30|31083.90|0.10|0.00|A|F|1992-08-26|1992-07-05|1992-09-01|DELIVER IN PERSON|SHIP|ckly special platelets. furiously sil|
+3650|128|9|2|43|44209.16|0.05|0.05|A|F|1992-09-07|1992-08-12|1992-09-10|COLLECT COD|TRUCK|gside of the quick|
+3650|2|9|3|1|902.00|0.04|0.06|A|F|1992-06-23|1992-07-18|1992-07-08|NONE|REG AIR|re about the pinto |
+3650|63|2|4|31|29854.86|0.10|0.08|R|F|1992-06-15|1992-07-01|1992-07-15|DELIVER IN PERSON|RAIL| against the ironic accounts cajol|
+3650|187|8|5|19|20656.42|0.05|0.04|R|F|1992-08-29|1992-08-09|1992-09-21|DELIVER IN PERSON|AIR|y even forges. fluffily furious accounts|
+3650|94|8|6|27|26840.43|0.07|0.08|A|F|1992-07-03|1992-07-23|1992-07-13|COLLECT COD|MAIL|ular requests snooze fluffily regular pi|
+3650|70|7|7|43|41713.01|0.10|0.07|A|F|1992-06-25|1992-07-09|1992-07-22|DELIVER IN PERSON|RAIL|structions use caref|
+3651|19|9|1|20|18380.20|0.01|0.04|N|O|1998-06-10|1998-06-06|1998-06-23|NONE|SHIP|tect quickly among the r|
+3651|155|7|2|24|25323.60|0.09|0.04|N|O|1998-06-22|1998-07-17|1998-07-10|DELIVER IN PERSON|RAIL|excuses haggle according to th|
+3651|113|10|3|41|41537.51|0.00|0.05|N|O|1998-05-10|1998-07-09|1998-05-13|NONE|RAIL|blithely. furiously |
+3651|110|5|4|27|27272.97|0.05|0.03|N|O|1998-05-03|1998-06-30|1998-05-05|DELIVER IN PERSON|RAIL| sleep blithely furiously do|
+3652|180|8|1|24|25924.32|0.05|0.03|N|O|1997-06-07|1997-04-07|1997-06-12|COLLECT COD|MAIL|the final p|
+3652|137|8|2|37|38373.81|0.02|0.05|N|O|1997-05-11|1997-04-06|1997-06-05|COLLECT COD|MAIL|osits haggle carefu|
+3652|163|8|3|39|41463.24|0.01|0.02|N|O|1997-03-10|1997-04-03|1997-03-21|NONE|REG AIR|y express instructions. un|
+3652|80|9|4|1|980.08|0.01|0.04|N|O|1997-04-20|1997-05-03|1997-05-18|DELIVER IN PERSON|SHIP| bold dependencies sublate. r|
+3653|145|4|1|38|39715.32|0.08|0.05|A|F|1994-06-26|1994-05-13|1994-07-13|NONE|REG AIR|ainst the |
+3653|64|1|2|29|27957.74|0.07|0.01|A|F|1994-04-11|1994-06-11|1994-04-29|COLLECT COD|RAIL|ording to the special, final|
+3653|181|2|3|17|18380.06|0.09|0.03|R|F|1994-06-24|1994-06-02|1994-07-17|DELIVER IN PERSON|RAIL|gle slyly regular|
+3653|186|7|4|9|9775.62|0.10|0.07|R|F|1994-04-03|1994-05-19|1994-04-10|COLLECT COD|FOB|slyly silent account|
+3653|188|9|5|41|44615.38|0.08|0.01|A|F|1994-06-18|1994-05-18|1994-06-20|COLLECT COD|RAIL|onic packages affix sly|
+3653|43|4|6|9|8487.36|0.05|0.03|A|F|1994-07-21|1994-05-31|1994-08-17|NONE|MAIL|tes: blithely bo|
+3653|49|6|7|2|1898.08|0.06|0.03|R|F|1994-06-02|1994-05-31|1994-06-29|NONE|FOB|n accounts. fina|
+3654|165|2|1|46|48997.36|0.08|0.05|A|F|1992-06-05|1992-08-19|1992-06-06|DELIVER IN PERSON|FOB|usly regular foxes. furio|
+3654|93|4|2|29|28799.61|0.07|0.06|A|F|1992-09-11|1992-07-20|1992-10-04|DELIVER IN PERSON|FOB|odolites detect. quickly r|
+3654|2|7|3|37|33374.00|0.07|0.05|A|F|1992-09-22|1992-07-20|1992-10-19|TAKE BACK RETURN|RAIL|unts doze bravely ab|
+3654|168|9|4|11|11749.76|0.08|0.00|A|F|1992-07-20|1992-07-30|1992-07-23|TAKE BACK RETURN|SHIP|quickly along the express, ironic req|
+3654|94|5|5|34|33799.06|0.04|0.00|R|F|1992-07-26|1992-08-26|1992-08-12|TAKE BACK RETURN|REG AIR| the quick|
+3654|107|4|6|20|20142.00|0.03|0.02|A|F|1992-07-30|1992-07-05|1992-08-05|COLLECT COD|SHIP|s sleep about the slyly |
+3654|173|1|7|45|48292.65|0.01|0.07|A|F|1992-09-15|1992-07-04|1992-09-20|DELIVER IN PERSON|FOB|sly ironic notornis nag slyly|
+3655|184|5|1|5|5420.90|0.03|0.04|R|F|1993-01-17|1992-12-31|1993-01-23|DELIVER IN PERSON|TRUCK|riously bold pinto be|
+3655|97|10|2|1|997.09|0.10|0.06|R|F|1992-10-24|1992-12-18|1992-11-07|DELIVER IN PERSON|AIR|arefully slow pinto beans are|
+3655|30|5|3|35|32551.05|0.01|0.04|R|F|1992-12-20|1992-11-16|1993-01-15|TAKE BACK RETURN|MAIL|blithely even accounts! furiously regular|
+3655|72|3|4|35|34022.45|0.04|0.07|R|F|1992-10-17|1992-12-23|1992-10-28|COLLECT COD|MAIL|ng foxes cajole fluffily slyly final fo|
+3680|177|6|1|48|51704.16|0.00|0.06|R|F|1993-01-16|1993-01-23|1993-01-19|COLLECT COD|FOB|packages. quickly fluff|
+3680|5|8|2|41|37105.00|0.00|0.04|A|F|1993-01-06|1993-03-02|1993-01-08|NONE|FOB|iously ironic platelets in|
+3680|56|4|3|33|31549.65|0.09|0.08|R|F|1993-03-16|1993-02-19|1993-04-05|NONE|FOB|ts. ironic, fina|
+3681|106|9|1|35|35213.50|0.03|0.08|R|F|1992-07-31|1992-05-18|1992-08-07|COLLECT COD|FOB|lyly special pinto |
+3682|61|10|1|6|5766.36|0.07|0.02|N|O|1997-05-06|1997-04-04|1997-05-11|NONE|AIR|ronic deposits wake slyly. ca|
+3682|116|7|2|18|18289.98|0.06|0.06|N|O|1997-04-30|1997-03-21|1997-05-10|NONE|FOB|regular dependencies|
+3682|47|10|3|17|16099.68|0.03|0.05|N|O|1997-02-12|1997-04-04|1997-02-22|COLLECT COD|FOB|, ironic packages wake a|
+3682|57|5|4|30|28711.50|0.09|0.05|N|O|1997-04-16|1997-04-16|1997-04-29|NONE|MAIL|he requests cajole quickly pending package|
+3683|101|4|1|35|35038.50|0.05|0.03|A|F|1993-05-31|1993-04-17|1993-06-14|NONE|SHIP| the furiously expr|
+3683|49|8|2|41|38910.64|0.01|0.06|A|F|1993-03-26|1993-05-06|1993-04-09|NONE|TRUCK|ress instructions. slyly express a|
+3683|100|3|3|23|23002.30|0.00|0.08|R|F|1993-07-02|1993-05-16|1993-07-30|NONE|TRUCK|xpress accounts sleep slyly re|
+3684|126|7|1|48|49253.76|0.04|0.06|A|F|1993-08-20|1993-09-02|1993-09-10|DELIVER IN PERSON|REG AIR|its boost alongside|
+3684|46|7|2|6|5676.24|0.06|0.08|R|F|1993-08-09|1993-10-05|1993-09-06|DELIVER IN PERSON|FOB|he silent requests. packages sleep fu|
+3684|163|8|3|19|20200.04|0.04|0.02|A|F|1993-10-19|1993-08-25|1993-11-02|COLLECT COD|FOB|e slyly carefully pending foxes. d|
+3684|135|1|4|13|13456.69|0.02|0.05|A|F|1993-07-23|1993-09-16|1993-08-06|NONE|TRUCK|ing, unusual pinto beans! thinly p|
+3685|47|4|1|37|35040.48|0.02|0.03|R|F|1992-03-11|1992-04-09|1992-04-05|DELIVER IN PERSON|TRUCK|ress attai|
+3685|58|6|2|7|6706.35|0.05|0.00|R|F|1992-05-16|1992-02-23|1992-05-17|DELIVER IN PERSON|FOB|sits. special asymptotes about the r|
+3685|134|5|3|38|39296.94|0.08|0.03|A|F|1992-05-17|1992-03-16|1992-06-06|TAKE BACK RETURN|TRUCK|thely unusual pack|
+3685|192|5|4|39|42595.41|0.10|0.05|R|F|1992-02-19|1992-04-06|1992-03-02|COLLECT COD|FOB|ic courts nag carefully after the |
+3685|56|7|5|37|35373.85|0.00|0.01|A|F|1992-03-02|1992-04-10|1992-03-04|NONE|FOB|. carefully sly requests are regular, regu|
+3686|122|5|1|7|7154.84|0.02|0.04|N|O|1998-07-15|1998-08-22|1998-07-30|DELIVER IN PERSON|TRUCK| furiously unusual accou|
+3686|200|2|2|38|41807.60|0.06|0.03|N|O|1998-09-04|1998-08-11|1998-09-19|DELIVER IN PERSON|AIR|y silent foxes! carefully ruthless cour|
+3686|45|6|3|31|29296.24|0.10|0.06|N|O|1998-09-09|1998-08-28|1998-10-09|COLLECT COD|MAIL|gle across the courts. furiously regu|
+3686|117|1|4|7|7119.77|0.10|0.01|N|O|1998-07-16|1998-09-02|1998-07-22|NONE|FOB|ake carefully carefully q|
+3687|145|4|1|32|33444.48|0.03|0.06|R|F|1993-05-07|1993-04-05|1993-05-25|DELIVER IN PERSON|AIR|deas cajole fo|
+3687|81|2|2|2|1962.16|0.00|0.08|R|F|1993-02-23|1993-03-25|1993-03-11|NONE|TRUCK| express requests. slyly regular depend|
+3687|174|4|3|10|10741.70|0.01|0.02|A|F|1993-02-11|1993-03-22|1993-03-09|NONE|FOB|ing pinto beans|
+3687|162|9|4|19|20181.04|0.02|0.05|A|F|1993-05-14|1993-04-24|1993-06-01|DELIVER IN PERSON|MAIL|ly final asymptotes according to t|
+3687|119|9|5|31|31592.41|0.07|0.08|A|F|1993-05-28|1993-03-20|1993-06-05|DELIVER IN PERSON|FOB|foxes cajole quickly about the furiously f|
+3712|141|4|1|27|28110.78|0.01|0.05|R|F|1992-02-01|1992-02-26|1992-03-02|TAKE BACK RETURN|SHIP|ctions. even accounts haggle alongside |
+3712|185|6|2|13|14107.34|0.03|0.03|R|F|1992-04-30|1992-02-11|1992-05-30|DELIVER IN PERSON|FOB|s around the furiously ironic account|
+3712|64|1|3|44|42418.64|0.01|0.01|A|F|1992-03-26|1992-02-19|1992-04-18|TAKE BACK RETURN|FOB|ously permanently regular req|
+3712|148|7|4|38|39829.32|0.01|0.06|A|F|1992-01-15|1992-03-24|1992-01-27|COLLECT COD|RAIL|s nag carefully-- even, reg|
+3713|112|6|1|41|41496.51|0.07|0.08|N|O|1998-05-11|1998-07-17|1998-05-22|COLLECT COD|RAIL|eposits wake blithely fina|
+3713|177|7|2|19|20466.23|0.04|0.04|N|O|1998-06-25|1998-07-24|1998-07-08|DELIVER IN PERSON|AIR|tructions serve blithely around the furi|
+3713|180|1|3|19|20523.42|0.03|0.02|N|O|1998-05-19|1998-07-06|1998-06-09|DELIVER IN PERSON|REG AIR|quests cajole careful|
+3713|169|10|4|45|48112.20|0.06|0.04|N|O|1998-06-15|1998-07-30|1998-07-14|DELIVER IN PERSON|MAIL|al pinto beans affix after the slyly |
+3713|90|1|5|46|45544.14|0.10|0.04|N|O|1998-08-22|1998-06-27|1998-08-31|NONE|MAIL|totes. carefully special theodolites s|
+3713|182|3|6|29|31383.22|0.09|0.03|N|O|1998-08-04|1998-06-13|1998-08-21|NONE|RAIL|the regular dugouts wake furiously sil|
+3713|130|1|7|14|14421.82|0.04|0.00|N|O|1998-07-19|1998-07-02|1998-07-28|DELIVER IN PERSON|SHIP|eposits impress according|
+3714|69|6|1|13|12597.78|0.07|0.03|N|O|1998-06-26|1998-06-17|1998-07-07|TAKE BACK RETURN|REG AIR| the furiously final|
+3714|146|3|2|14|14645.96|0.02|0.05|N|O|1998-05-30|1998-06-30|1998-05-31|DELIVER IN PERSON|RAIL|ending ideas. thinly unusual theodo|
+3714|159|10|3|16|16946.40|0.00|0.02|N|O|1998-05-25|1998-07-07|1998-06-17|TAKE BACK RETURN|AIR|ccounts cajole fu|
+3714|30|9|4|44|40921.32|0.04|0.02|N|O|1998-07-18|1998-07-10|1998-07-22|DELIVER IN PERSON|AIR|s. quickly ironic dugouts sublat|
+3715|97|1|1|13|12962.17|0.00|0.03|N|O|1996-05-11|1996-04-25|1996-06-09|TAKE BACK RETURN|SHIP|e quickly ironic|
+3715|169|6|2|16|17106.56|0.01|0.06|N|O|1996-06-28|1996-04-22|1996-06-30|TAKE BACK RETURN|AIR|usly regular pearls haggle final packages|
+3715|12|3|3|37|33744.37|0.05|0.02|N|O|1996-05-03|1996-04-30|1996-05-17|NONE|SHIP|ut the carefully expr|
+3716|32|8|1|10|9320.30|0.09|0.04|N|O|1997-12-02|1997-11-09|1997-12-14|TAKE BACK RETURN|SHIP|ts. quickly sly ideas slee|
+3716|194|5|2|39|42673.41|0.02|0.08|N|O|1997-11-27|1997-10-23|1997-12-24|COLLECT COD|REG AIR|even deposits.|
+3716|107|8|3|42|42298.20|0.02|0.08|N|O|1997-12-03|1997-10-12|1997-12-15|NONE|TRUCK| of the pend|
+3716|165|10|4|19|20238.04|0.05|0.08|N|O|1997-09-25|1997-10-18|1997-10-12|NONE|TRUCK|arefully unusual accounts. flu|
+3716|182|3|5|25|27054.50|0.06|0.05|N|O|1997-11-23|1997-10-24|1997-11-24|COLLECT COD|REG AIR|fully unusual accounts. carefu|
+3717|153|8|1|45|47391.75|0.07|0.04|N|O|1998-08-09|1998-08-18|1998-08-14|TAKE BACK RETURN|TRUCK|ests wake whithout the blithely final pl|
+3717|53|5|2|3|2859.15|0.01|0.07|N|O|1998-06-09|1998-07-31|1998-06-14|NONE|REG AIR|nside the regular packages sleep|
+3717|196|7|3|45|49328.55|0.05|0.08|N|O|1998-09-19|1998-07-22|1998-09-28|DELIVER IN PERSON|MAIL|s the blithely unu|
+3717|69|6|4|5|4845.30|0.06|0.03|N|O|1998-09-02|1998-08-20|1998-09-26|TAKE BACK RETURN|AIR|quickly among |
+3717|16|7|5|7|6412.07|0.09|0.02|N|O|1998-09-08|1998-07-18|1998-09-10|DELIVER IN PERSON|RAIL| after the packa|
+3717|64|1|6|38|36634.28|0.01|0.07|N|O|1998-07-10|1998-07-08|1998-07-29|COLLECT COD|RAIL|ly about the car|
+3717|106|7|7|28|28170.80|0.03|0.01|N|O|1998-07-25|1998-08-12|1998-08-16|COLLECT COD|RAIL|ts sleep q|
+3718|21|10|1|40|36840.80|0.01|0.04|N|O|1996-11-20|1996-12-17|1996-12-03|DELIVER IN PERSON|MAIL|out the express deposits|
+3718|163|8|2|16|17010.56|0.02|0.06|N|O|1996-11-11|1996-12-25|1996-11-12|COLLECT COD|TRUCK|slyly even accounts. blithely special acco|
+3718|70|5|3|8|7760.56|0.05|0.03|N|O|1996-12-06|1996-12-06|1996-12-15|TAKE BACK RETURN|AIR| the even deposits sleep carefully b|
+3719|22|5|1|35|32270.70|0.06|0.08|N|O|1997-06-11|1997-04-03|1997-06-15|TAKE BACK RETURN|TRUCK|ly foxes. pending braids haggle furio|
+3719|174|4|2|2|2148.34|0.02|0.08|N|O|1997-02-17|1997-04-25|1997-03-03|NONE|REG AIR|ccounts boost carefu|
+3719|182|3|3|12|12986.16|0.05|0.06|N|O|1997-06-10|1997-05-04|1997-07-09|TAKE BACK RETURN|REG AIR|grate according to the |
+3719|90|1|4|13|12871.17|0.02|0.00|N|O|1997-05-03|1997-04-16|1997-05-27|TAKE BACK RETURN|SHIP|iously. regular dep|
+3719|78|8|5|19|18583.33|0.06|0.08|N|O|1997-05-22|1997-03-20|1997-06-12|COLLECT COD|TRUCK|he regular ideas integrate acros|
+3719|142|5|6|43|44812.02|0.03|0.08|N|O|1997-05-08|1997-04-15|1997-06-06|COLLECT COD|RAIL|the furiously special pinto bean|
+3719|19|10|7|16|14704.16|0.10|0.01|N|O|1997-03-02|1997-03-18|1997-03-28|TAKE BACK RETURN|RAIL| express asymptotes. ir|
+3744|195|8|1|30|32855.70|0.05|0.06|A|F|1992-05-07|1992-02-12|1992-05-17|TAKE BACK RETURN|FOB|nts among |
+3745|137|8|1|18|18668.34|0.01|0.05|A|F|1993-10-17|1993-11-16|1993-11-13|DELIVER IN PERSON|SHIP| slyly bold pinto beans according to |
+3746|165|6|1|37|39410.92|0.07|0.00|A|F|1994-12-29|1994-10-25|1995-01-03|COLLECT COD|FOB|e of the careful|
+3746|144|7|2|28|29235.92|0.06|0.08|R|F|1994-09-20|1994-10-21|1994-09-27|DELIVER IN PERSON|FOB|s after the even, special requests|
+3746|188|9|3|3|3264.54|0.10|0.01|R|F|1994-11-03|1994-12-10|1994-11-12|NONE|MAIL| the silent ideas cajole carefully |
+3746|28|7|4|11|10208.22|0.00|0.05|R|F|1994-10-02|1994-11-19|1994-10-10|COLLECT COD|SHIP| ironic theodolites are among th|
+3747|141|10|1|42|43727.88|0.05|0.05|N|O|1996-11-10|1996-10-19|1996-11-19|TAKE BACK RETURN|REG AIR|y. blithely fina|
+3747|170|1|2|33|35315.61|0.01|0.03|N|O|1996-10-14|1996-11-12|1996-11-11|NONE|REG AIR| regular p|
+3747|139|10|3|30|31173.90|0.00|0.07|N|O|1996-12-16|1996-11-15|1996-12-17|NONE|RAIL|! furiously f|
+3747|33|9|4|21|19593.63|0.00|0.06|N|O|1996-11-18|1996-09-23|1996-11-26|TAKE BACK RETURN|AIR|ithely bold orbits mold furiously blit|
+3747|126|5|5|32|32835.84|0.08|0.05|N|O|1996-09-10|1996-11-04|1996-10-10|DELIVER IN PERSON|MAIL|quests shall h|
+3747|154|5|6|14|14758.10|0.08|0.07|N|O|1996-11-03|1996-10-29|1996-11-06|TAKE BACK RETURN|AIR|packages cajole carefu|
+3747|118|2|7|23|23416.53|0.00|0.04|N|O|1996-11-08|1996-11-10|1996-12-03|NONE|REG AIR|kages are ironic|
+3748|104|7|1|12|12049.20|0.06|0.01|N|O|1998-04-17|1998-04-15|1998-05-12|NONE|AIR|old reques|
+3748|165|4|2|24|25563.84|0.08|0.04|N|O|1998-06-07|1998-05-02|1998-06-21|DELIVER IN PERSON|TRUCK|al deposits. blithely|
+3748|197|1|3|19|20846.61|0.05|0.01|N|O|1998-04-23|1998-05-17|1998-05-23|COLLECT COD|RAIL|pinto beans run carefully quic|
+3748|187|8|4|5|5435.90|0.00|0.07|N|O|1998-06-29|1998-05-06|1998-07-12|DELIVER IN PERSON|MAIL| regular accounts sleep quickly-- furious|
+3748|147|4|5|21|21989.94|0.07|0.08|N|O|1998-03-30|1998-04-07|1998-04-05|TAKE BACK RETURN|MAIL|fix carefully furiously express ideas. furi|
+3749|173|3|1|11|11804.87|0.07|0.05|N|O|1995-06-25|1995-05-23|1995-07-10|TAKE BACK RETURN|RAIL|egular requests along the |
+3749|129|8|2|9|9262.08|0.08|0.05|A|F|1995-04-23|1995-04-18|1995-04-26|NONE|REG AIR|uses cajole blithely pla|
+3749|199|2|3|31|34074.89|0.00|0.05|N|F|1995-06-11|1995-05-20|1995-06-27|COLLECT COD|REG AIR|s. foxes sleep slyly unusual grouc|
+3749|131|2|4|7|7217.91|0.07|0.06|A|F|1995-03-31|1995-04-05|1995-04-11|NONE|TRUCK|he slyly ironic packages|
+3749|183|4|5|14|15164.52|0.02|0.00|N|F|1995-06-11|1995-05-19|1995-07-11|DELIVER IN PERSON|SHIP|press instruc|
+3749|54|6|6|10|9540.50|0.10|0.03|N|O|1995-06-24|1995-05-24|1995-07-18|COLLECT COD|SHIP|essly. regular pi|
+3750|134|10|1|37|38262.81|0.04|0.03|N|O|1995-07-08|1995-07-28|1995-07-28|DELIVER IN PERSON|REG AIR|usly busy account|
+3750|152|3|2|33|34720.95|0.05|0.03|N|O|1995-06-27|1995-06-20|1995-07-03|TAKE BACK RETURN|REG AIR|theodolites haggle. slyly pendin|
+3750|80|10|3|20|19601.60|0.09|0.05|N|F|1995-06-17|1995-06-06|1995-06-28|TAKE BACK RETURN|REG AIR|ss, ironic requests! fur|
+3750|166|1|4|33|35183.28|0.04|0.03|N|F|1995-06-15|1995-06-04|1995-06-29|COLLECT COD|RAIL|ep blithely according to the flu|
+3750|83|4|5|1|983.08|0.05|0.01|N|O|1995-07-24|1995-06-25|1995-08-21|DELIVER IN PERSON|REG AIR|l dolphins against the slyly|
+3750|113|7|6|47|47616.17|0.01|0.08|R|F|1995-05-11|1995-06-13|1995-06-02|TAKE BACK RETURN|FOB|slowly regular accounts. blithely ev|
+3751|172|2|1|37|39670.29|0.00|0.04|R|F|1994-04-30|1994-05-30|1994-05-30|NONE|REG AIR|ly express courts |
+3751|141|8|2|32|33316.48|0.03|0.05|R|F|1994-05-05|1994-07-02|1994-06-02|COLLECT COD|MAIL|rthogs could have to slee|
+3751|65|2|3|45|43427.70|0.08|0.06|R|F|1994-05-27|1994-06-19|1994-06-14|NONE|RAIL|according to |
+3751|14|4|4|39|35646.39|0.07|0.01|A|F|1994-08-16|1994-07-11|1994-09-12|COLLECT COD|TRUCK|refully according to the iro|
+3751|58|3|5|12|11496.60|0.02|0.03|A|F|1994-08-09|1994-06-30|1994-08-12|TAKE BACK RETURN|TRUCK|accounts wake furious|
+3751|76|5|6|39|38066.73|0.02|0.08|R|F|1994-08-01|1994-06-01|1994-08-26|COLLECT COD|SHIP|to beans. pending, express packages c|
+3776|3|10|1|39|35217.00|0.05|0.01|R|F|1993-01-03|1993-02-05|1993-01-08|COLLECT COD|FOB|yly blithely pending packages|
+3776|159|4|2|14|14828.10|0.06|0.08|R|F|1992-12-30|1993-02-12|1993-01-27|DELIVER IN PERSON|RAIL|y special ideas. express packages pr|
+3776|141|8|3|49|51015.86|0.01|0.08|R|F|1992-12-03|1993-02-16|1992-12-28|TAKE BACK RETURN|RAIL|equests. final, thin grouches |
+3776|92|6|4|49|48612.41|0.08|0.05|A|F|1993-02-11|1993-01-06|1993-02-27|COLLECT COD|MAIL|es: careful warthogs haggle fluffi|
+3777|100|4|1|11|11001.10|0.02|0.03|A|F|1994-04-09|1994-06-05|1994-04-14|NONE|FOB|ld ideas. even theodolites|
+3777|8|5|2|10|9080.00|0.03|0.01|R|F|1994-05-22|1994-05-29|1994-06-13|COLLECT COD|RAIL|le. ironic depths a|
+3777|166|7|3|18|19190.88|0.10|0.06|R|F|1994-05-04|1994-05-23|1994-05-22|COLLECT COD|REG AIR|eful packages use slyly: even deposits |
+3777|18|9|4|35|32130.35|0.10|0.04|A|F|1994-05-25|1994-05-26|1994-06-13|COLLECT COD|AIR|s. carefully express asymptotes accordi|
+3777|98|10|5|14|13973.26|0.04|0.05|R|F|1994-05-06|1994-06-24|1994-05-31|NONE|TRUCK|ording to the iro|
+3778|57|2|1|21|20098.05|0.01|0.06|R|F|1993-05-27|1993-07-10|1993-06-03|COLLECT COD|REG AIR|ts. blithely special theodoli|
+3778|29|10|2|32|29728.64|0.09|0.00|A|F|1993-06-22|1993-08-18|1993-07-03|TAKE BACK RETURN|MAIL|tes affix carefully above the |
+3778|94|6|3|41|40757.69|0.05|0.00|R|F|1993-06-21|1993-07-27|1993-07-15|COLLECT COD|FOB|e the furiously ironi|
+3778|169|4|4|28|29936.48|0.03|0.05|R|F|1993-08-18|1993-07-10|1993-09-06|TAKE BACK RETURN|REG AIR|y silent orbits print carefully against |
+3778|98|2|5|28|27946.52|0.01|0.06|R|F|1993-09-02|1993-08-08|1993-10-02|DELIVER IN PERSON|FOB|r deposits. theodol|
+3778|20|7|6|26|23920.52|0.00|0.01|A|F|1993-09-24|1993-07-06|1993-10-22|NONE|TRUCK| against the fluffily|
+3778|105|6|7|49|49249.90|0.02|0.04|A|F|1993-06-13|1993-08-08|1993-07-04|DELIVER IN PERSON|MAIL|ans. furiously |
+3779|46|5|1|28|26489.12|0.04|0.05|N|O|1997-05-06|1997-04-01|1997-05-18|TAKE BACK RETURN|AIR|s. close requests sleep|
+3779|110|3|2|5|5050.55|0.07|0.03|N|O|1997-01-07|1997-03-26|1997-02-05|DELIVER IN PERSON|AIR|heodolites. slyly regular a|
+3780|127|8|1|25|25678.00|0.08|0.04|N|O|1996-06-27|1996-07-02|1996-07-22|NONE|AIR|l, unusual |
+3780|190|1|2|40|43607.60|0.10|0.04|N|O|1996-06-06|1996-05-29|1996-07-01|COLLECT COD|SHIP|gular deposits-- furiously regular |
+3781|14|5|1|48|43872.48|0.02|0.06|N|O|1996-08-22|1996-08-13|1996-09-15|NONE|REG AIR|equests may cajole careful|
+3781|188|9|2|39|42439.02|0.10|0.00|N|O|1996-08-20|1996-08-16|1996-09-01|DELIVER IN PERSON|REG AIR|unts are carefully. ir|
+3781|30|1|3|17|15810.51|0.01|0.03|N|O|1996-06-23|1996-09-04|1996-07-19|TAKE BACK RETURN|REG AIR|. theodolite|
+3781|31|2|4|15|13965.45|0.05|0.00|N|O|1996-08-23|1996-08-08|1996-09-06|TAKE BACK RETURN|AIR| carefully blithe|
+3781|16|6|5|23|21068.23|0.09|0.08|N|O|1996-09-05|1996-08-18|1996-09-27|DELIVER IN PERSON|SHIP|pendencies are b|
+3782|27|10|1|29|26883.58|0.01|0.07|N|O|1996-09-17|1996-10-03|1996-10-07|DELIVER IN PERSON|REG AIR|quickly unusual pinto beans. carefully fina|
+3782|153|1|2|10|10531.50|0.03|0.05|N|O|1996-09-07|1996-11-19|1996-10-04|COLLECT COD|FOB|ven pinto b|
+3782|136|7|3|30|31083.90|0.06|0.06|N|O|1996-12-19|1996-10-31|1997-01-14|TAKE BACK RETURN|MAIL|slyly even pinto beans hag|
+3782|117|7|4|34|34581.74|0.02|0.06|N|O|1996-11-07|1996-10-22|1996-11-19|DELIVER IN PERSON|MAIL|gage after the even|
+3782|130|3|5|40|41205.20|0.09|0.04|N|O|1996-12-16|1996-11-22|1997-01-01|COLLECT COD|AIR|s instructions. regular accou|
+3783|167|4|1|36|38417.76|0.04|0.08|R|F|1993-12-17|1994-02-26|1994-01-03|DELIVER IN PERSON|SHIP|ites haggle among the carefully unusu|
+3783|73|3|2|36|35030.52|0.02|0.02|R|F|1994-03-02|1994-02-09|1994-03-15|COLLECT COD|TRUCK|egular accounts|
+3783|85|6|3|50|49254.00|0.04|0.01|R|F|1994-03-14|1994-01-09|1994-04-10|DELIVER IN PERSON|FOB|he furiously regular deposits. |
+3783|27|6|4|37|34299.74|0.10|0.05|R|F|1993-12-09|1994-02-17|1993-12-30|COLLECT COD|REG AIR|ing to the ideas. regular accounts de|
+3808|43|10|1|28|26405.12|0.02|0.01|R|F|1994-05-27|1994-06-18|1994-06-22|TAKE BACK RETURN|FOB|lly final accounts alo|
+3808|127|6|2|47|48274.64|0.04|0.08|R|F|1994-06-12|1994-06-03|1994-07-02|COLLECT COD|TRUCK|fully for the quickly final deposits: flu|
+3808|31|2|3|45|41896.35|0.00|0.03|R|F|1994-07-03|1994-05-29|1994-07-14|TAKE BACK RETURN|REG AIR| carefully special|
+3808|100|1|4|34|34003.40|0.07|0.04|R|F|1994-08-13|1994-07-22|1994-08-31|DELIVER IN PERSON|FOB| pearls will have to |
+3808|155|7|5|29|30599.35|0.08|0.03|A|F|1994-06-22|1994-05-26|1994-07-06|TAKE BACK RETURN|TRUCK| deposits across the pac|
+3808|168|5|6|44|46999.04|0.06|0.06|A|F|1994-06-07|1994-06-04|1994-06-25|NONE|REG AIR|the blithely regular foxes. even, final |
+3809|191|3|1|17|18550.23|0.10|0.04|N|O|1996-08-14|1996-07-05|1996-09-04|DELIVER IN PERSON|FOB|es detect furiously sil|
+3809|133|4|2|32|33060.16|0.01|0.02|N|O|1996-07-03|1996-06-01|1996-07-25|COLLECT COD|SHIP|xcuses would boost against the fluffily eve|
+3809|105|6|3|46|46234.60|0.10|0.06|N|O|1996-08-20|1996-06-01|1996-08-24|TAKE BACK RETURN|TRUCK|l asymptotes. special |
+3809|178|9|4|43|46361.31|0.00|0.04|N|O|1996-05-06|1996-06-22|1996-06-05|TAKE BACK RETURN|TRUCK|yly ironic decoys; regular, iron|
+3810|184|5|1|49|53124.82|0.05|0.01|R|F|1992-11-27|1992-10-30|1992-12-16|COLLECT COD|AIR|cajole. fur|
+3810|169|8|2|18|19244.88|0.01|0.04|A|F|1992-11-28|1992-11-15|1992-12-27|DELIVER IN PERSON|SHIP|s. furiously careful deposi|
+3810|137|3|3|41|42522.33|0.08|0.08|A|F|1992-10-26|1992-10-27|1992-11-05|COLLECT COD|SHIP|l requests boost slyly along the slyl|
+3810|182|3|4|11|11903.98|0.06|0.04|A|F|1992-12-18|1992-12-11|1993-01-15|DELIVER IN PERSON|MAIL| the pending pinto beans. expr|
+3811|164|3|1|24|25539.84|0.04|0.02|N|O|1998-07-13|1998-05-16|1998-08-12|TAKE BACK RETURN|TRUCK|deposits. slyly regular accounts cajo|
+3811|166|5|2|2|2132.32|0.01|0.08|N|O|1998-06-16|1998-06-16|1998-06-23|NONE|MAIL|slyly fluff|
+3811|43|6|3|19|17917.76|0.02|0.06|N|O|1998-07-20|1998-06-14|1998-07-29|NONE|MAIL|s boost blithely furiou|
+3811|171|1|4|50|53558.50|0.08|0.03|N|O|1998-07-28|1998-07-06|1998-08-16|COLLECT COD|FOB|ts are slyly fluffy ideas. furiou|
+3811|182|3|5|23|24890.14|0.00|0.04|N|O|1998-08-13|1998-07-09|1998-08-29|COLLECT COD|AIR|nstructions sleep quickly. slyly final |
+3811|2|7|6|35|31570.00|0.04|0.07|N|O|1998-04-17|1998-06-30|1998-04-25|NONE|REG AIR|yly final dolphins? quickly ironic frets|
+3812|145|4|1|33|34489.62|0.00|0.05|N|O|1996-10-10|1996-10-05|1996-10-15|TAKE BACK RETURN|MAIL|posits engage. ironic, regular p|
+3812|173|2|2|33|35414.61|0.06|0.03|N|O|1996-10-05|1996-10-13|1996-10-22|TAKE BACK RETURN|MAIL|inal excuses d|
+3813|176|7|1|37|39818.29|0.05|0.04|N|O|1998-10-13|1998-09-19|1998-10-28|NONE|REG AIR|ravely special packages haggle p|
+3813|123|2|2|39|39901.68|0.05|0.00|N|O|1998-08-30|1998-08-12|1998-09-29|COLLECT COD|FOB|y ideas. final ideas about the sp|
+3814|131|7|1|7|7217.91|0.02|0.02|R|F|1995-05-01|1995-05-09|1995-05-28|DELIVER IN PERSON|REG AIR|es sleep furiou|
+3814|173|3|2|14|15024.38|0.01|0.00|R|F|1995-03-17|1995-05-10|1995-04-16|DELIVER IN PERSON|AIR|sits along the final, ironic deposit|
+3814|168|7|3|36|38453.76|0.06|0.02|N|O|1995-06-19|1995-04-18|1995-06-28|COLLECT COD|SHIP|beans cajole quickly sl|
+3814|66|7|4|20|19321.20|0.04|0.07|R|F|1995-02-23|1995-03-26|1995-03-04|DELIVER IN PERSON|SHIP|. doggedly ironic deposits will have to wa|
+3814|107|2|5|15|15106.50|0.03|0.04|N|O|1995-06-23|1995-03-25|1995-07-09|COLLECT COD|SHIP| carefully final deposits haggle slyly|
+3814|83|4|6|47|46204.76|0.09|0.05|A|F|1995-04-16|1995-04-03|1995-05-14|DELIVER IN PERSON|AIR|nusual requests. bli|
+3814|132|8|7|12|12385.56|0.10|0.01|R|F|1995-03-18|1995-04-16|1995-03-20|TAKE BACK RETURN|REG AIR|ages cajole. packages haggle. final|
+3815|77|7|1|3|2931.21|0.07|0.00|N|O|1997-11-16|1997-11-15|1997-11-30|NONE|FOB|egular, express ideas. ironic, final dep|
+3815|130|5|2|11|11331.43|0.02|0.04|N|O|1997-11-01|1997-11-05|1997-11-27|COLLECT COD|TRUCK|sleep blithe|
+3840|187|8|1|45|48923.10|0.02|0.08|N|O|1998-10-31|1998-09-19|1998-11-30|DELIVER IN PERSON|TRUCK|o beans are. carefully final courts x|
+3840|46|9|2|12|11352.48|0.04|0.07|N|O|1998-10-02|1998-08-19|1998-10-20|TAKE BACK RETURN|RAIL|xpress pinto beans. accounts a|
+3840|73|4|3|45|43788.15|0.02|0.05|N|O|1998-10-12|1998-10-12|1998-10-28|TAKE BACK RETURN|FOB|onic, even packages are. pe|
+3840|148|9|4|41|42973.74|0.07|0.02|N|O|1998-07-21|1998-10-08|1998-08-01|TAKE BACK RETURN|MAIL| nag slyly? slyly pending accounts |
+3840|173|3|5|7|7512.19|0.09|0.08|N|O|1998-09-17|1998-09-20|1998-10-14|DELIVER IN PERSON|MAIL|. furiously final gifts sleep carefully pin|
+3840|107|8|6|33|33234.30|0.10|0.02|N|O|1998-07-29|1998-10-06|1998-08-04|DELIVER IN PERSON|SHIP|hely silent deposits w|
+3841|157|5|1|1|1057.15|0.06|0.03|A|F|1994-10-10|1994-11-12|1994-10-21|DELIVER IN PERSON|AIR| boost even re|
+3841|21|10|2|31|28551.62|0.09|0.03|A|F|1995-01-24|1994-11-25|1995-02-20|TAKE BACK RETURN|SHIP|n theodolites shall promise carefully. qui|
+3841|152|10|3|40|42086.00|0.06|0.02|A|F|1995-02-02|1994-11-30|1995-02-14|TAKE BACK RETURN|MAIL|its. quickly regular ideas nag carefully|
+3841|50|1|4|9|8550.45|0.10|0.07|A|F|1994-11-21|1994-12-26|1994-11-26|NONE|FOB|s according to the courts shall nag s|
+3841|176|7|5|3|3228.51|0.04|0.02|R|F|1994-10-24|1994-12-07|1994-11-09|COLLECT COD|FOB|foxes integrate |
+3841|163|8|6|48|51031.68|0.03|0.00|R|F|1994-11-23|1994-11-22|1994-12-01|DELIVER IN PERSON|FOB| according to the regular, |
+3842|162|7|1|28|29740.48|0.05|0.07|A|F|1992-06-17|1992-06-03|1992-06-24|DELIVER IN PERSON|TRUCK|s excuses thrash carefully.|
+3842|122|1|2|21|21464.52|0.07|0.05|R|F|1992-07-15|1992-06-02|1992-07-21|NONE|RAIL|r pinto be|
+3842|194|7|3|28|30637.32|0.00|0.00|A|F|1992-06-20|1992-05-22|1992-07-13|DELIVER IN PERSON|MAIL|lly alongside of the|
+3842|88|9|4|15|14821.20|0.07|0.01|A|F|1992-06-26|1992-06-23|1992-07-09|COLLECT COD|MAIL|ave packages are slyl|
+3842|68|3|5|13|12584.78|0.09|0.02|R|F|1992-04-13|1992-06-22|1992-05-11|COLLECT COD|RAIL|t blithely. busily regular accounts alon|
+3842|107|4|6|24|24170.40|0.08|0.08|R|F|1992-08-05|1992-06-29|1992-08-16|TAKE BACK RETURN|MAIL|phins are quickly|
+3843|15|6|1|7|6405.07|0.10|0.03|N|O|1997-02-13|1997-02-21|1997-02-20|TAKE BACK RETURN|SHIP|slyly even instructions. furiously eve|
+3843|1|4|2|30|27030.00|0.01|0.05|N|O|1997-02-14|1997-03-25|1997-03-13|DELIVER IN PERSON|AIR| wake. slyly even packages boost |
+3844|135|1|1|2|2070.26|0.03|0.07|R|F|1995-02-24|1995-02-03|1995-03-18|TAKE BACK RETURN|AIR|es haggle final acco|
+3844|102|7|2|5|5010.50|0.10|0.03|R|F|1995-04-29|1995-02-24|1995-05-05|TAKE BACK RETURN|RAIL| unwind quickly about the pending, i|
+3845|34|5|1|44|41097.32|0.01|0.08|A|F|1992-07-20|1992-07-15|1992-07-24|DELIVER IN PERSON|REG AIR|s haggle among the fluffily regula|
+3845|24|7|2|16|14784.32|0.09|0.05|A|F|1992-08-08|1992-06-08|1992-08-26|DELIVER IN PERSON|SHIP|ely bold ideas use. ex|
+3845|59|1|3|17|16303.85|0.08|0.01|A|F|1992-06-12|1992-07-05|1992-06-26|TAKE BACK RETURN|RAIL|counts haggle. reg|
+3845|46|9|4|1|946.04|0.04|0.05|R|F|1992-05-21|1992-06-07|1992-06-17|COLLECT COD|REG AIR| blithely ironic t|
+3845|196|7|5|27|29597.13|0.00|0.05|R|F|1992-08-20|1992-07-17|1992-09-02|COLLECT COD|REG AIR|kages. care|
+3845|105|8|6|30|30153.00|0.09|0.06|R|F|1992-08-21|1992-07-07|1992-08-25|COLLECT COD|FOB|counts do wake blithely. ironic requests |
+3846|61|10|1|15|14415.90|0.06|0.03|N|O|1998-02-17|1998-04-27|1998-02-21|NONE|REG AIR|uternes. carefully even|
+3846|171|2|2|30|32135.10|0.08|0.07|N|O|1998-05-01|1998-03-12|1998-05-20|TAKE BACK RETURN|FOB|deposits according to the fur|
+3846|15|5|3|49|44835.49|0.08|0.07|N|O|1998-02-14|1998-03-22|1998-02-17|DELIVER IN PERSON|RAIL|efully even packages against the blithe|
+3846|165|10|4|33|35150.28|0.05|0.00|N|O|1998-05-12|1998-03-14|1998-05-14|DELIVER IN PERSON|TRUCK|s instructions are. fu|
+3847|189|10|1|7|7624.26|0.08|0.00|A|F|1993-05-06|1993-06-06|1993-05-22|COLLECT COD|MAIL| about the blithely daring Tiresias. fl|
+3872|181|2|1|28|30273.04|0.10|0.04|N|O|1996-11-05|1996-11-10|1996-11-24|DELIVER IN PERSON|REG AIR|t after the carefully ironic excuses. f|
+3872|17|4|2|38|34846.38|0.04|0.05|N|O|1996-10-18|1996-12-03|1996-11-15|TAKE BACK RETURN|AIR|iously against the ironic, unusual a|
+3872|169|4|3|18|19244.88|0.07|0.07|N|O|1996-12-25|1996-10-24|1997-01-08|TAKE BACK RETURN|SHIP|s. regular, brave accounts sleep blith|
+3872|11|2|4|41|37351.41|0.07|0.03|N|O|1996-11-23|1996-11-12|1996-12-03|COLLECT COD|REG AIR|ly regular epitaphs boost|
+3872|70|7|5|42|40742.94|0.03|0.00|N|O|1997-01-03|1996-10-12|1997-01-16|COLLECT COD|MAIL|s the furio|
+3872|140|6|6|40|41605.60|0.07|0.05|N|O|1997-01-02|1996-10-29|1997-01-14|NONE|REG AIR|nts? regularly ironic ex|
+3873|68|3|1|19|18393.14|0.04|0.04|N|O|1998-05-15|1998-05-10|1998-05-17|NONE|FOB|y final ac|
+3873|145|8|2|44|45986.16|0.05|0.05|N|O|1998-07-23|1998-05-22|1998-08-14|COLLECT COD|AIR|yly even platelets wake. |
+3873|140|6|3|29|30164.06|0.01|0.04|N|O|1998-06-22|1998-05-20|1998-07-05|COLLECT COD|REG AIR|olphins af|
+3874|170|7|1|21|22473.57|0.09|0.08|R|F|1993-06-19|1993-07-20|1993-07-08|DELIVER IN PERSON|SHIP| requests cajole fluff|
+3874|19|6|2|48|44112.48|0.06|0.07|R|F|1993-06-13|1993-07-20|1993-06-20|NONE|RAIL| ideas throughout |
+3875|81|2|1|24|23545.92|0.02|0.08|N|O|1997-10-15|1997-11-27|1997-11-09|COLLECT COD|AIR|ecial packages. |
+3875|113|7|2|49|49642.39|0.04|0.04|N|O|1997-10-18|1997-10-13|1997-10-19|NONE|MAIL|sleep furiously about the deposits. quickl|
+3876|141|8|1|12|12493.68|0.06|0.07|N|O|1996-09-16|1996-10-23|1996-10-05|TAKE BACK RETURN|REG AIR|y above the pending tithes. blithely ironi|
+3876|140|6|2|37|38485.18|0.00|0.03|N|O|1996-11-30|1996-10-18|1996-12-18|DELIVER IN PERSON|AIR|t dependencies. blithely final packages u|
+3876|127|8|3|41|42111.92|0.02|0.04|N|O|1996-10-15|1996-10-17|1996-10-19|NONE|AIR| quickly blit|
+3877|50|7|1|12|11400.60|0.06|0.01|R|F|1993-05-30|1993-08-09|1993-06-24|TAKE BACK RETURN|FOB|nal requests. even requests are. pac|
+3877|145|4|2|47|49121.58|0.05|0.00|A|F|1993-08-01|1993-08-16|1993-08-04|NONE|FOB|furiously quick requests nag along the theo|
+3877|80|8|3|44|43123.52|0.09|0.00|A|F|1993-06-07|1993-07-15|1993-07-06|DELIVER IN PERSON|REG AIR|elets. quickly regular accounts caj|
+3877|148|9|4|36|37733.04|0.06|0.01|A|F|1993-07-27|1993-07-13|1993-08-11|DELIVER IN PERSON|AIR|lithely about the dogged ideas. ac|
+3877|5|6|5|41|37105.00|0.03|0.07|A|F|1993-06-30|1993-07-20|1993-07-01|DELIVER IN PERSON|FOB|integrate against the expres|
+3877|123|4|6|7|7161.84|0.04|0.08|R|F|1993-06-14|1993-07-09|1993-06-28|NONE|TRUCK|lar dolphins cajole silently |
+3878|200|1|1|6|6601.20|0.07|0.04|N|O|1997-06-21|1997-05-22|1997-07-01|COLLECT COD|FOB|s. regular instru|
+3878|88|9|2|13|12845.04|0.01|0.06|N|O|1997-06-08|1997-06-03|1997-06-25|TAKE BACK RETURN|TRUCK|leep ruthlessly about the carefu|
+3878|41|8|3|20|18820.80|0.08|0.03|N|O|1997-06-20|1997-05-24|1997-07-20|TAKE BACK RETURN|MAIL|the furiously careful ideas cajole slyly sl|
+3878|152|3|4|20|21043.00|0.01|0.07|N|O|1997-07-13|1997-05-22|1997-07-20|NONE|FOB|about the carefully ironic pa|
+3879|126|5|1|45|46175.40|0.10|0.08|N|O|1996-03-18|1996-01-03|1996-04-03|COLLECT COD|RAIL|ly according to the expr|
+3879|45|4|2|35|33076.40|0.00|0.07|N|O|1995-12-08|1996-01-23|1995-12-28|TAKE BACK RETURN|MAIL|o beans. accounts cajole furiously. re|
+3904|38|4|1|22|20636.66|0.04|0.03|N|O|1998-02-02|1998-02-09|1998-02-10|TAKE BACK RETURN|REG AIR|structions cajole carefully. carefully f|
+3904|184|5|2|19|20599.42|0.09|0.01|N|O|1998-02-10|1998-02-13|1998-02-20|TAKE BACK RETURN|AIR| excuses sleep slyly according to th|
+3905|101|8|1|43|43047.30|0.07|0.08|A|F|1994-03-30|1994-02-18|1994-04-09|DELIVER IN PERSON|REG AIR|uses are care|
+3905|116|10|2|7|7112.77|0.03|0.00|R|F|1994-03-01|1994-02-19|1994-03-11|DELIVER IN PERSON|AIR|ully furiously furious packag|
+3905|170|7|3|6|6421.02|0.07|0.02|R|F|1994-04-07|1994-03-07|1994-04-21|DELIVER IN PERSON|RAIL|ow furiously. deposits wake ironic |
+3906|153|1|1|42|44232.30|0.00|0.04|R|F|1992-09-03|1992-07-22|1992-09-04|COLLECT COD|RAIL|jole blithely after the furiously regular |
+3906|40|1|2|50|47002.00|0.01|0.07|R|F|1992-09-24|1992-08-24|1992-09-29|NONE|MAIL|ke slyly. stealt|
+3906|180|9|3|15|16202.70|0.06|0.02|R|F|1992-07-30|1992-08-26|1992-08-02|TAKE BACK RETURN|FOB|dependencies at the |
+3906|59|10|4|36|34525.80|0.08|0.08|A|F|1992-08-07|1992-08-08|1992-08-24|NONE|SHIP|y. ironic deposits haggle sl|
+3907|112|6|1|41|41496.51|0.06|0.02|A|F|1992-09-13|1992-10-23|1992-09-29|COLLECT COD|MAIL|ackages wake along the carefully regul|
+3907|145|4|2|41|42850.74|0.03|0.00|A|F|1992-10-25|1992-10-17|1992-11-01|TAKE BACK RETURN|RAIL|s above the unusual ideas sleep furiousl|
+3907|52|4|3|45|42842.25|0.02|0.07|R|F|1992-09-21|1992-09-19|1992-10-18|COLLECT COD|REG AIR| about the regular pac|
+3907|176|5|4|48|51656.16|0.05|0.07|A|F|1992-09-24|1992-10-16|1992-10-06|DELIVER IN PERSON|TRUCK|nt asymptotes lose across th|
+3907|62|3|5|22|21165.32|0.09|0.01|R|F|1992-09-20|1992-10-30|1992-09-29|TAKE BACK RETURN|TRUCK|ly. furiously unusual deposits use afte|
+3907|126|9|6|34|34888.08|0.02|0.02|R|F|1992-09-06|1992-10-08|1992-09-12|COLLECT COD|FOB| requests according to the slyly pending |
+3907|110|5|7|8|8080.88|0.10|0.01|A|F|1992-09-18|1992-10-29|1992-09-27|NONE|REG AIR|furiously final packages.|
+3908|92|4|1|50|49604.50|0.05|0.04|R|F|1993-06-19|1993-04-27|1993-07-05|DELIVER IN PERSON|MAIL| even accounts wake |
+3908|148|9|2|8|8385.12|0.06|0.03|A|F|1993-03-12|1993-04-13|1993-03-22|DELIVER IN PERSON|SHIP|r instructions was requests. ironically |
+3909|178|6|1|30|32345.10|0.03|0.07|N|O|1998-10-17|1998-10-14|1998-10-28|COLLECT COD|TRUCK|ly even deposits across the ironic notorni|
+3909|191|4|2|46|50194.74|0.03|0.01|N|O|1998-10-08|1998-10-15|1998-10-24|NONE|FOB|the blithely unusual ideas|
+3910|139|10|1|10|10391.30|0.00|0.08|N|O|1996-10-18|1996-10-31|1996-11-14|DELIVER IN PERSON|FOB|tions boost furiously unusual e|
+3910|71|10|2|31|30103.17|0.05|0.03|N|O|1996-12-22|1996-11-14|1997-01-01|TAKE BACK RETURN|SHIP|ess instructions. |
+3910|20|7|3|6|5520.12|0.04|0.04|N|O|1996-12-08|1996-10-30|1996-12-31|DELIVER IN PERSON|MAIL|ly sly platelets are fluffily slyly si|
+3910|153|1|4|1|1053.15|0.03|0.06|N|O|1996-09-12|1996-10-21|1996-09-19|DELIVER IN PERSON|FOB|s sleep neve|
+3911|113|7|1|10|10131.10|0.07|0.06|N|O|1995-06-22|1995-05-30|1995-06-28|COLLECT COD|FOB|ss theodolites are blithely along t|
+3911|119|9|2|14|14267.54|0.08|0.05|R|F|1995-04-28|1995-05-03|1995-05-22|NONE|RAIL|e blithely brave depo|
+3911|92|5|3|12|11905.08|0.10|0.05|R|F|1995-04-04|1995-04-16|1995-04-10|COLLECT COD|FOB|uctions. blithely regula|
+3936|137|8|1|25|25928.25|0.06|0.03|N|O|1996-12-03|1996-12-27|1997-01-01|DELIVER IN PERSON|RAIL|gular requests nag quic|
+3936|188|9|2|24|26116.32|0.10|0.07|N|O|1996-11-22|1997-01-01|1996-12-08|NONE|AIR|ns. accounts mold fl|
+3936|83|4|3|42|41289.36|0.00|0.07|N|O|1997-01-03|1997-01-29|1997-01-14|COLLECT COD|AIR|elets wake amo|
+3936|62|7|4|12|11544.72|0.06|0.05|N|O|1996-11-25|1997-01-09|1996-12-06|DELIVER IN PERSON|SHIP|ithely across the carefully brave req|
+3936|84|5|5|35|34442.80|0.02|0.08|N|O|1996-12-04|1997-01-06|1996-12-22|NONE|SHIP|lly ironic requ|
+3936|103|6|6|26|26080.60|0.01|0.02|N|O|1997-02-27|1997-01-16|1997-03-22|NONE|RAIL|quickly pen|
+3937|70|7|1|48|46563.36|0.10|0.02|N|O|1998-03-15|1998-02-22|1998-03-30|DELIVER IN PERSON|FOB|gainst the thinl|
+3937|48|1|2|30|28441.20|0.01|0.07|N|O|1998-01-17|1998-01-03|1998-02-08|COLLECT COD|TRUCK|al packages slee|
+3937|115|5|3|27|27407.97|0.03|0.00|N|O|1998-02-06|1998-01-12|1998-02-20|NONE|MAIL|ven ideas. slyly expr|
+3937|154|2|4|50|52707.50|0.01|0.02|N|O|1998-01-15|1998-01-09|1998-02-04|DELIVER IN PERSON|AIR|ong the carefully exp|
+3937|3|10|5|29|26187.00|0.03|0.07|N|O|1998-03-06|1998-02-22|1998-03-14|NONE|TRUCK|nt pinto beans above the pending instr|
+3937|193|6|6|6|6559.14|0.00|0.00|N|O|1998-01-24|1998-02-13|1998-01-27|DELIVER IN PERSON|FOB|into beans. slyly silent orbits alongside o|
+3937|164|9|7|1|1064.16|0.02|0.05|N|O|1998-03-29|1998-01-08|1998-04-27|TAKE BACK RETURN|TRUCK|refully agains|
+3938|159|4|1|46|48720.90|0.10|0.07|R|F|1993-05-20|1993-05-04|1993-06-12|DELIVER IN PERSON|FOB|ly even foxes are slyly fu|
+3939|160|8|1|8|8481.28|0.03|0.06|N|O|1996-01-29|1996-04-05|1996-02-26|COLLECT COD|REG AIR|e packages. express, pen|
+3940|178|7|1|33|35579.61|0.10|0.07|N|O|1996-05-19|1996-04-19|1996-05-23|TAKE BACK RETURN|RAIL|ly ironic packages about the pending accou|
+3940|69|4|2|40|38762.40|0.08|0.02|N|O|1996-02-29|1996-03-22|1996-03-04|NONE|MAIL|ts. regular fox|
+3940|89|10|3|8|7912.64|0.07|0.08|N|O|1996-04-04|1996-04-12|1996-04-18|DELIVER IN PERSON|RAIL|ions cajole furiously regular pinto beans. |
+3940|137|3|4|11|11408.43|0.09|0.05|N|O|1996-03-09|1996-05-13|1996-03-17|COLLECT COD|REG AIR|e of the special packages. furiously|
+3940|1|6|5|41|36941.00|0.00|0.07|N|O|1996-05-08|1996-05-03|1996-06-03|COLLECT COD|MAIL|thily. deposits cajole.|
+3941|41|2|1|47|44228.88|0.05|0.07|N|O|1996-11-24|1996-10-09|1996-12-22|DELIVER IN PERSON|RAIL| carefully pending|
+3941|123|6|2|19|19439.28|0.05|0.00|N|O|1996-11-10|1996-10-26|1996-12-05|COLLECT COD|RAIL|eposits haggle furiously even|
+3941|10|3|3|2|1820.02|0.01|0.03|N|O|1996-12-04|1996-10-01|1996-12-25|NONE|REG AIR|es wake after the|
+3941|110|7|4|29|29293.19|0.00|0.03|N|O|1996-09-14|1996-10-04|1996-09-19|NONE|MAIL|g the blithely|
+3942|183|4|1|6|6499.08|0.05|0.05|A|F|1993-07-01|1993-09-14|1993-07-23|DELIVER IN PERSON|SHIP|ep ruthlessly carefully final accounts: s|
+3942|194|7|2|5|5470.95|0.06|0.02|R|F|1993-09-27|1993-09-24|1993-10-07|DELIVER IN PERSON|MAIL|. fluffily pending deposits above the flu|
+3942|156|4|3|25|26403.75|0.04|0.06|R|F|1993-09-13|1993-08-01|1993-09-29|COLLECT COD|RAIL|d the quick packages|
+3943|198|2|1|15|16472.85|0.03|0.01|N|O|1997-01-13|1996-12-17|1997-02-02|COLLECT COD|REG AIR| grow fluffily according to the |
+3943|96|7|2|9|8964.81|0.00|0.06|N|O|1996-11-27|1997-01-03|1996-12-17|COLLECT COD|RAIL|refully ironic |
+3943|17|4|3|32|29344.32|0.00|0.02|N|O|1996-10-22|1996-12-17|1996-11-04|TAKE BACK RETURN|TRUCK| unusual ideas into the furiously even pack|
+3943|50|1|4|5|4750.25|0.04|0.04|N|O|1997-01-09|1996-11-10|1997-02-06|COLLECT COD|RAIL|arefully regular deposits accord|
+3968|54|2|1|27|25759.35|0.04|0.05|N|O|1997-04-25|1997-04-17|1997-05-11|TAKE BACK RETURN|MAIL|t silently.|
+3968|26|9|2|45|41670.90|0.00|0.07|N|O|1997-06-18|1997-04-24|1997-06-25|DELIVER IN PERSON|FOB|ully slyly fi|
+3968|156|7|3|43|45414.45|0.07|0.06|N|O|1997-04-30|1997-05-14|1997-05-18|TAKE BACK RETURN|SHIP|ly regular accounts|
+3968|61|8|4|7|6727.42|0.07|0.02|N|O|1997-03-30|1997-05-01|1997-04-12|DELIVER IN PERSON|SHIP|efully bold instructions. express|
+3969|52|4|1|39|37129.95|0.04|0.04|N|O|1997-06-12|1997-06-13|1997-07-05|NONE|MAIL|ly bold ideas s|
+3969|197|1|2|26|28526.94|0.05|0.03|N|O|1997-07-08|1997-07-30|1997-07-10|TAKE BACK RETURN|AIR|fluffily; braids detect.|
+3969|79|8|3|46|45037.22|0.04|0.02|N|O|1997-05-29|1997-06-15|1997-06-10|TAKE BACK RETURN|SHIP|fully final requests sleep stealthily. care|
+3969|151|9|4|21|22074.15|0.07|0.04|N|O|1997-08-31|1997-07-16|1997-09-02|TAKE BACK RETURN|MAIL|unts doze quickly final reque|
+3969|72|3|5|40|38882.80|0.09|0.00|N|O|1997-05-19|1997-08-02|1997-06-05|COLLECT COD|TRUCK|lar requests cajole furiously blithely regu|
+3969|105|8|6|4|4020.40|0.02|0.01|N|O|1997-06-04|1997-07-31|1997-06-13|COLLECT COD|REG AIR|dencies wake blithely? quickly even theodo|
+3970|88|9|1|2|1976.16|0.01|0.07|R|F|1992-04-24|1992-06-03|1992-05-16|TAKE BACK RETURN|RAIL|carefully pending foxes wake blithely |
+3970|109|6|2|18|18163.80|0.03|0.08|A|F|1992-06-06|1992-06-18|1992-07-05|DELIVER IN PERSON|TRUCK| maintain slyly. ir|
+3970|154|6|3|10|10541.50|0.10|0.04|A|F|1992-07-01|1992-05-31|1992-07-02|NONE|AIR| special packages wake after the final br|
+3970|22|5|4|34|31348.68|0.05|0.00|A|F|1992-06-25|1992-05-23|1992-07-12|COLLECT COD|SHIP|y final gifts are. carefully pe|
+3970|30|3|5|23|21390.69|0.05|0.04|A|F|1992-06-04|1992-06-14|1992-06-13|COLLECT COD|TRUCK| above the final braids. regular|
+3970|9|6|6|46|41814.00|0.07|0.04|R|F|1992-04-29|1992-05-14|1992-05-24|NONE|FOB|yly ironic|
+3970|5|8|7|46|41630.00|0.08|0.08|R|F|1992-05-02|1992-05-12|1992-05-10|COLLECT COD|MAIL|ix slyly. quickly silen|
+3971|96|8|1|47|46816.23|0.06|0.04|N|O|1996-07-07|1996-08-08|1996-08-01|TAKE BACK RETURN|RAIL|e slyly final dependencies x-ray |
+3971|191|5|2|2|2182.38|0.04|0.03|N|O|1996-07-15|1996-08-12|1996-07-26|NONE|SHIP|haggle abou|
+3972|51|3|1|2|1902.10|0.05|0.03|A|F|1994-07-24|1994-06-30|1994-08-13|TAKE BACK RETURN|SHIP|y final theodolite|
+3973|30|9|1|21|19530.63|0.02|0.06|R|F|1992-06-18|1992-06-03|1992-07-02|COLLECT COD|REG AIR|equests. furiously|
+3973|115|2|2|37|37559.07|0.07|0.00|A|F|1992-05-29|1992-05-04|1992-06-23|TAKE BACK RETURN|SHIP|inos wake fluffily. pending requests nag |
+3973|40|6|3|40|37601.60|0.08|0.05|R|F|1992-05-03|1992-06-09|1992-05-21|COLLECT COD|RAIL|g the carefully blithe f|
+3974|22|1|1|47|43334.94|0.10|0.03|N|O|1996-06-03|1996-05-08|1996-06-28|NONE|TRUCK|dencies above the re|
+3974|61|8|2|17|16338.02|0.05|0.07|N|O|1996-04-05|1996-05-21|1996-04-28|COLLECT COD|TRUCK|ions eat slyly after the blithely |
+3975|57|9|1|38|36367.90|0.01|0.05|N|O|1995-08-02|1995-06-18|1995-08-19|COLLECT COD|TRUCK|es are furiously: furi|
+4000|196|7|1|41|44943.79|0.06|0.01|A|F|1992-03-02|1992-03-14|1992-03-27|COLLECT COD|FOB|ve the even, fi|
+4000|75|5|2|44|42903.08|0.09|0.06|A|F|1992-03-27|1992-02-18|1992-03-31|COLLECT COD|AIR|equests use blithely blithely bold d|
+4001|106|1|1|26|26158.60|0.00|0.01|N|O|1997-07-26|1997-06-18|1997-08-08|DELIVER IN PERSON|RAIL|tegrate blithely|
+4001|41|10|2|19|17879.76|0.03|0.02|N|O|1997-08-23|1997-06-15|1997-09-18|COLLECT COD|SHIP|ackages. carefully ironi|
+4001|94|5|3|18|17893.62|0.07|0.00|N|O|1997-06-04|1997-06-22|1997-06-13|DELIVER IN PERSON|MAIL|lithely ironic d|
+4001|2|9|4|39|35178.00|0.00|0.00|N|O|1997-06-13|1997-06-17|1997-06-25|NONE|SHIP| dogged excuses. blithe|
+4002|111|5|1|35|35388.85|0.01|0.08|N|O|1997-05-16|1997-06-15|1997-06-02|DELIVER IN PERSON|TRUCK|eep. quickly|
+4002|198|9|2|20|21963.80|0.00|0.03|N|O|1997-06-15|1997-05-20|1997-07-11|NONE|MAIL|lly even ins|
+4002|40|1|3|6|5640.24|0.08|0.07|N|O|1997-05-02|1997-07-07|1997-05-16|TAKE BACK RETURN|RAIL| furiously furiously special theodoli|
+4002|199|3|4|6|6595.14|0.06|0.06|N|O|1997-07-01|1997-05-15|1997-07-31|NONE|MAIL|he slyly iro|
+4002|99|1|5|4|3996.36|0.08|0.07|N|O|1997-05-06|1997-06-15|1997-05-24|NONE|REG AIR|ccording to the careful|
+4003|52|4|1|18|17136.90|0.04|0.07|R|F|1993-02-02|1993-04-15|1993-02-28|TAKE BACK RETURN|AIR|ar grouches s|
+4004|121|2|1|23|23485.76|0.07|0.02|A|F|1993-08-12|1993-07-13|1993-08-16|TAKE BACK RETURN|TRUCK| bold theodolites? special packages accordi|
+4004|64|5|2|47|45310.82|0.07|0.04|R|F|1993-06-25|1993-08-03|1993-07-12|NONE|SHIP|thely instead of the even, unu|
+4004|114|5|3|39|39550.29|0.10|0.05|R|F|1993-07-12|1993-07-27|1993-07-18|NONE|MAIL|ccounts sleep furious|
+4004|74|4|4|46|44807.22|0.10|0.04|R|F|1993-09-04|1993-07-13|1993-09-28|COLLECT COD|FOB|ncies. slyly pending dolphins sleep furio|
+4004|155|3|5|9|9496.35|0.04|0.06|A|F|1993-08-25|1993-06-10|1993-09-24|COLLECT COD|MAIL|ly ironic requests. quickly pending ide|
+4004|161|10|6|44|46691.04|0.07|0.05|R|F|1993-07-25|1993-07-23|1993-08-16|TAKE BACK RETURN|REG AIR|ut the sauternes. bold, ironi|
+4004|126|9|7|20|20522.40|0.07|0.05|A|F|1993-06-19|1993-06-14|1993-07-04|COLLECT COD|REG AIR|. ironic deposits cajole blithely?|
+4005|4|1|1|26|23504.00|0.09|0.05|N|O|1996-12-01|1997-02-03|1996-12-15|NONE|REG AIR| to the quic|
+4005|17|8|2|28|25676.28|0.02|0.06|N|O|1996-12-11|1997-01-24|1996-12-17|DELIVER IN PERSON|REG AIR|ly carefully ironic deposits. slyly|
+4005|72|10|3|28|27217.96|0.03|0.01|N|O|1996-12-08|1997-01-14|1996-12-30|TAKE BACK RETURN|MAIL|y pending dependenc|
+4005|15|9|4|49|44835.49|0.09|0.00|N|O|1997-01-31|1996-12-24|1997-03-02|NONE|RAIL|tions sleep across the silent d|
+4005|6|7|5|14|12684.00|0.09|0.08|N|O|1996-11-27|1997-01-09|1996-12-25|NONE|TRUCK|ld requests. slyly final instructi|
+4006|55|7|1|11|10505.55|0.05|0.08|A|F|1995-04-29|1995-02-21|1995-05-20|TAKE BACK RETURN|RAIL|ress foxes cajole quick|
+4006|159|4|2|18|19064.70|0.05|0.03|A|F|1995-01-29|1995-03-08|1995-02-02|TAKE BACK RETURN|MAIL|gouts! slyly iron|
+4006|24|5|3|15|13860.30|0.01|0.02|R|F|1995-02-23|1995-04-02|1995-02-25|TAKE BACK RETURN|RAIL|n deposits cajole slyl|
+4006|114|5|4|25|25352.75|0.00|0.07|A|F|1995-02-23|1995-02-09|1995-02-24|DELIVER IN PERSON|SHIP| requests use depos|
+4007|57|2|1|32|30625.60|0.00|0.03|R|F|1993-09-30|1993-08-16|1993-10-03|DELIVER IN PERSON|RAIL|nal accounts across t|
+4007|116|10|2|41|41660.51|0.04|0.06|A|F|1993-10-11|1993-08-30|1993-11-04|DELIVER IN PERSON|TRUCK|eposits. regular epitaphs boost blithely.|
+4007|102|9|3|5|5010.50|0.09|0.06|A|F|1993-09-17|1993-08-29|1993-10-12|TAKE BACK RETURN|FOB|y unusual packa|
+4007|138|4|4|15|15571.95|0.05|0.02|A|F|1993-09-01|1993-07-19|1993-09-03|DELIVER IN PERSON|FOB|le furiously quickly |
+4007|26|7|5|23|21298.46|0.02|0.07|A|F|1993-10-08|1993-09-09|1993-10-23|COLLECT COD|MAIL|ter the accounts. expr|
+4032|102|3|1|8|8016.80|0.06|0.00|N|O|1998-06-04|1998-05-17|1998-07-03|TAKE BACK RETURN|RAIL|ometimes even cou|
+4032|2|9|2|27|24354.00|0.09|0.00|N|O|1998-05-31|1998-04-19|1998-06-24|COLLECT COD|REG AIR|le furiously according to|
+4032|154|2|3|23|24245.45|0.09|0.06|N|O|1998-06-12|1998-05-11|1998-06-24|COLLECT COD|MAIL|ording to the |
+4032|85|6|4|10|9850.80|0.09|0.05|N|O|1998-03-31|1998-04-22|1998-04-07|NONE|REG AIR| carefully bol|
+4033|110|1|1|27|27272.97|0.01|0.04|R|F|1993-08-08|1993-08-14|1993-08-09|NONE|AIR|pinto beans|
+4033|38|4|2|34|31893.02|0.07|0.00|R|F|1993-07-19|1993-08-05|1993-07-26|NONE|RAIL|t the blithely dogg|
+4034|190|1|1|48|52329.12|0.03|0.03|A|F|1994-03-01|1994-01-16|1994-03-16|NONE|RAIL| blithely regular requests play carefull|
+4034|57|5|2|47|44981.35|0.07|0.05|A|F|1994-01-27|1993-12-26|1994-02-04|NONE|TRUCK|eodolites was slyly ironic ideas. de|
+4034|54|5|3|43|41024.15|0.10|0.03|A|F|1993-11-29|1994-01-08|1993-12-10|DELIVER IN PERSON|FOB|posits wake carefully af|
+4034|28|9|4|46|42688.92|0.06|0.00|A|F|1994-02-22|1994-01-09|1994-03-04|DELIVER IN PERSON|AIR|uests. furiously unusual instructions wake|
+4034|196|10|5|7|7673.33|0.07|0.06|R|F|1994-03-04|1994-01-22|1994-04-01|NONE|AIR|y even theodolites. slyly regular instru|
+4034|50|9|6|5|4750.25|0.01|0.06|A|F|1994-02-12|1994-01-24|1994-02-13|COLLECT COD|AIR|fully around the furiously ironic re|
+4035|97|8|1|4|3988.36|0.08|0.03|R|F|1992-04-21|1992-04-23|1992-04-25|COLLECT COD|AIR|ilent, even pear|
+4035|136|7|2|4|4144.52|0.07|0.00|A|F|1992-05-21|1992-04-24|1992-05-24|DELIVER IN PERSON|FOB|en instructions sleep blith|
+4035|118|8|3|1|1018.11|0.03|0.01|R|F|1992-06-18|1992-05-19|1992-07-02|COLLECT COD|FOB| requests. quickly |
+4035|182|3|4|13|14068.34|0.00|0.01|R|F|1992-06-10|1992-05-16|1992-07-10|NONE|SHIP|s. furiously even courts wake slyly|
+4036|6|1|1|46|41676.00|0.09|0.00|N|O|1997-06-21|1997-05-29|1997-07-18|NONE|REG AIR|usly across the even th|
+4036|53|1|2|21|20014.05|0.09|0.07|N|O|1997-08-08|1997-06-28|1997-08-09|COLLECT COD|MAIL|e carefully. qui|
+4036|142|3|3|6|6252.84|0.07|0.01|N|O|1997-06-19|1997-06-16|1997-07-01|DELIVER IN PERSON|SHIP|equests wake about the bold id|
+4036|127|10|4|20|20542.40|0.08|0.02|N|O|1997-08-11|1997-07-11|1997-09-03|NONE|TRUCK|slyly bold deposits cajole pending, blithe|
+4037|64|9|1|32|30849.92|0.00|0.06|A|F|1993-05-06|1993-06-08|1993-05-31|DELIVER IN PERSON|AIR|e of the pending, iron|
+4037|47|8|2|4|3788.16|0.09|0.07|A|F|1993-07-05|1993-06-12|1993-08-03|DELIVER IN PERSON|RAIL|s around the blithely ironic ac|
+4038|196|10|1|40|43847.60|0.05|0.01|N|O|1996-01-15|1996-03-13|1996-01-25|COLLECT COD|TRUCK|t. slyly silent pinto beans amo|
+4038|12|9|2|37|33744.37|0.04|0.03|N|O|1996-03-17|1996-03-19|1996-04-07|DELIVER IN PERSON|REG AIR| packages |
+4038|32|3|3|24|22368.72|0.10|0.04|N|O|1996-04-06|1996-02-15|1996-04-18|TAKE BACK RETURN|RAIL|the furiously regu|
+4038|150|1|4|29|30454.35|0.07|0.06|N|O|1996-01-07|1996-03-08|1996-01-13|NONE|FOB|ffix. quietly ironic packages a|
+4038|79|7|5|24|23497.68|0.07|0.06|N|O|1996-04-01|1996-04-05|1996-04-28|DELIVER IN PERSON|TRUCK|ake quickly after the final, ironic ac|
+4038|36|2|6|6|5616.18|0.07|0.05|N|O|1996-02-09|1996-03-05|1996-03-10|COLLECT COD|SHIP| special instructions. packa|
+4039|94|5|1|38|37775.42|0.03|0.06|N|O|1998-03-09|1997-12-31|1998-03-21|DELIVER IN PERSON|REG AIR|sual asymptotes. ironic deposits nag aft|
+4039|122|5|2|17|17376.04|0.10|0.04|N|O|1998-01-15|1998-01-20|1998-01-28|TAKE BACK RETURN|MAIL| regular foxes haggle carefully bo|
+4039|64|1|3|9|8676.54|0.10|0.01|N|O|1998-03-08|1998-02-05|1998-04-05|TAKE BACK RETURN|FOB|t? pinto beans cajole across the thinly r|
+4039|28|3|4|43|39904.86|0.01|0.02|N|O|1998-01-02|1997-12-22|1998-01-15|NONE|FOB|beans believe bene|
+4039|134|5|5|43|44467.59|0.09|0.00|N|O|1998-01-20|1998-01-11|1998-02-05|COLLECT COD|SHIP|sts along the regular in|
+4064|199|1|1|3|3297.57|0.10|0.04|N|O|1997-01-04|1997-01-01|1997-01-23|NONE|SHIP|its! quickly sp|
+4064|40|6|2|15|14100.60|0.02|0.02|N|O|1996-11-09|1996-12-04|1996-11-18|DELIVER IN PERSON|MAIL|braids affix across the regular sheave|
+4064|197|10|3|32|35110.08|0.04|0.07|N|O|1997-01-14|1997-01-01|1997-01-21|COLLECT COD|REG AIR|es boost. careful|
+4064|163|8|4|24|25515.84|0.02|0.02|N|O|1997-01-01|1996-12-31|1997-01-23|DELIVER IN PERSON|SHIP|ly regular ideas.|
+4064|21|2|5|12|11052.24|0.08|0.08|N|O|1997-02-08|1996-12-18|1997-03-06|TAKE BACK RETURN|RAIL|ding to the requests|
+4064|184|5|6|46|49872.28|0.03|0.00|N|O|1996-10-13|1997-01-05|1996-11-06|DELIVER IN PERSON|REG AIR|alongside of the f|
+4064|200|2|7|9|9901.80|0.01|0.06|N|O|1996-12-17|1996-12-13|1997-01-12|NONE|AIR|furiously f|
+4065|138|9|1|14|14533.82|0.04|0.02|A|F|1994-08-22|1994-07-29|1994-09-19|DELIVER IN PERSON|TRUCK|e furiously outside |
+4065|15|6|2|46|42090.46|0.03|0.05|A|F|1994-06-29|1994-08-01|1994-07-03|TAKE BACK RETURN|SHIP|, regular requests may mold above the |
+4065|97|10|3|33|32903.97|0.00|0.03|A|F|1994-09-03|1994-08-16|1994-09-13|DELIVER IN PERSON|AIR|ain blithely |
+4065|107|2|4|8|8056.80|0.00|0.01|R|F|1994-10-04|1994-08-05|1994-10-25|TAKE BACK RETURN|SHIP|ages haggle carefully|
+4065|123|4|5|29|29670.48|0.02|0.07|A|F|1994-06-29|1994-08-19|1994-07-17|NONE|RAIL|equests. packages sleep slyl|
+4065|110|5|6|16|16161.76|0.05|0.00|R|F|1994-08-25|1994-08-06|1994-09-09|COLLECT COD|TRUCK|ncies use furiously. quickly un|
+4065|144|7|7|11|11485.54|0.10|0.04|A|F|1994-07-25|1994-08-02|1994-07-30|NONE|RAIL|hang silently about |
+4066|139|5|1|9|9352.17|0.01|0.05|N|O|1997-05-06|1997-03-25|1997-05-27|COLLECT COD|FOB|nal, ironic accounts. blithel|
+4066|93|5|2|19|18868.71|0.05|0.00|N|O|1997-05-13|1997-04-17|1997-06-08|NONE|TRUCK|quests. slyly regu|
+4066|76|5|3|8|7808.56|0.03|0.03|N|O|1997-04-24|1997-03-11|1997-05-20|NONE|REG AIR|accounts. special pinto beans|
+4066|179|9|4|49|52879.33|0.01|0.01|N|O|1997-02-17|1997-03-24|1997-02-19|NONE|TRUCK|ial braids. furiously final deposits sl|
+4066|171|2|5|43|46060.31|0.05|0.02|N|O|1997-02-16|1997-04-14|1997-02-18|DELIVER IN PERSON|MAIL|r instructions. slyly special |
+4066|109|2|6|44|44400.40|0.01|0.00|N|O|1997-03-01|1997-04-27|1997-03-29|DELIVER IN PERSON|MAIL|express accounts nag bli|
+4067|180|1|1|18|19443.24|0.03|0.08|A|F|1993-01-24|1992-12-23|1993-02-20|TAKE BACK RETURN|FOB|e the slyly final packages d|
+4067|96|10|2|14|13945.26|0.00|0.00|R|F|1993-02-03|1992-12-02|1993-02-07|TAKE BACK RETURN|TRUCK|ructions. quickly ironic accounts detect |
+4067|141|10|3|17|17699.38|0.03|0.05|A|F|1993-01-26|1992-11-23|1993-01-27|NONE|REG AIR|ts haggle slyly unusual, final|
+4067|90|1|4|40|39603.60|0.07|0.08|R|F|1993-01-09|1992-11-21|1993-01-16|DELIVER IN PERSON|TRUCK|lar theodolites nag blithely above the|
+4067|85|6|5|17|16746.36|0.08|0.03|A|F|1993-01-20|1992-12-29|1993-02-03|DELIVER IN PERSON|REG AIR|r accounts. slyly special pa|
+4067|96|8|6|12|11953.08|0.04|0.03|A|F|1992-12-12|1992-11-28|1992-12-15|DELIVER IN PERSON|AIR|lly slyly even theodol|
+4067|83|4|7|17|16712.36|0.01|0.01|R|F|1992-12-12|1992-12-23|1992-12-30|NONE|AIR|ts affix. regular, regular requests s|
+4068|110|1|1|43|43434.73|0.05|0.06|N|O|1996-11-28|1996-11-16|1996-12-22|NONE|AIR|ructions. regular, special packag|
+4068|57|5|2|31|29668.55|0.08|0.03|N|O|1996-12-11|1996-12-07|1996-12-30|NONE|SHIP|ds wake carefully amon|
+4069|129|2|1|39|40135.68|0.09|0.02|R|F|1992-09-06|1992-07-22|1992-09-25|COLLECT COD|SHIP|ven theodolites nag quickly. fluffi|
+4069|43|4|2|32|30177.28|0.10|0.08|A|F|1992-06-18|1992-07-20|1992-07-07|TAKE BACK RETURN|TRUCK|unts. deposit|
+4069|186|7|3|3|3258.54|0.06|0.01|R|F|1992-07-26|1992-07-07|1992-08-04|COLLECT COD|FOB|l packages. even, |
+4069|79|8|4|22|21539.54|0.10|0.05|A|F|1992-08-05|1992-08-04|1992-08-25|COLLECT COD|SHIP|ts. slyly special instruction|
+4069|157|5|5|50|52857.50|0.09|0.06|A|F|1992-07-26|1992-06-30|1992-08-01|TAKE BACK RETURN|REG AIR|even foxes among the express wate|
+4069|125|8|6|3|3075.36|0.02|0.01|A|F|1992-05-24|1992-06-18|1992-06-12|COLLECT COD|MAIL|y final deposits wake furiously! slyl|
+4069|184|5|7|50|54209.00|0.00|0.01|R|F|1992-09-03|1992-06-14|1992-10-01|NONE|REG AIR|ages. carefully regular |
+4070|183|4|1|2|2166.36|0.09|0.08|N|O|1995-08-03|1995-09-10|1995-08-17|TAKE BACK RETURN|REG AIR|ptotes affix|
+4070|155|3|2|40|42206.00|0.07|0.07|N|O|1995-07-13|1995-07-23|1995-08-06|COLLECT COD|MAIL|about the sentiments. quick|
+4070|62|3|3|11|10582.66|0.00|0.08|N|O|1995-08-23|1995-08-15|1995-08-31|TAKE BACK RETURN|MAIL| carefully final pack|
+4070|29|4|4|46|42734.92|0.02|0.02|N|O|1995-06-22|1995-07-14|1995-07-11|DELIVER IN PERSON|REG AIR|nticing ideas. boldly|
+4071|112|2|1|22|22266.42|0.02|0.07|N|O|1996-10-31|1996-12-14|1996-11-05|NONE|REG AIR|sits cajole carefully final instructio|
+4071|18|8|2|47|43146.47|0.00|0.03|N|O|1996-11-04|1996-12-09|1996-11-16|NONE|TRUCK|ts cajole furiously along the|
+4096|27|10|1|31|28737.62|0.10|0.02|A|F|1992-07-14|1992-09-03|1992-07-31|COLLECT COD|TRUCK|y final, even platelets. boldly|
+4096|57|9|2|17|16269.85|0.07|0.03|R|F|1992-09-30|1992-08-11|1992-10-11|TAKE BACK RETURN|REG AIR|platelets alongside of the |
+4096|9|10|3|21|19089.00|0.08|0.00|A|F|1992-08-24|1992-09-04|1992-09-11|DELIVER IN PERSON|MAIL|tes mold flu|
+4096|128|3|4|20|20562.40|0.02|0.07|R|F|1992-08-24|1992-09-13|1992-08-28|DELIVER IN PERSON|TRUCK|sual requests. furiously bold packages wake|
+4097|74|5|1|50|48703.50|0.04|0.04|N|O|1996-08-31|1996-08-14|1996-09-27|DELIVER IN PERSON|MAIL|egular deposits. blithely pending|
+4097|74|4|2|46|44807.22|0.10|0.01|N|O|1996-07-29|1996-08-19|1996-08-25|COLLECT COD|AIR| even depend|
+4097|174|2|3|42|45115.14|0.06|0.06|N|O|1996-08-11|1996-07-30|1996-08-15|NONE|FOB|carefully silent foxes are against the |
+4098|200|1|1|46|50609.20|0.07|0.03|N|O|1997-01-26|1997-01-27|1997-02-13|TAKE BACK RETURN|SHIP|e slyly blithely silent deposits. fluff|
+4099|4|7|1|29|26216.00|0.09|0.07|R|F|1992-11-21|1992-11-04|1992-11-30|NONE|FOB| slowly final warthogs sleep blithely. q|
+4099|137|3|2|3|3111.39|0.04|0.06|A|F|1992-09-12|1992-10-18|1992-10-01|NONE|RAIL|. special packages sleep|
+4099|51|3|3|36|34237.80|0.06|0.06|R|F|1992-11-06|1992-09-28|1992-12-02|NONE|FOB|beans cajole slyly quickly ironic |
+4099|139|5|4|7|7273.91|0.05|0.02|A|F|1992-09-12|1992-11-13|1992-09-14|TAKE BACK RETURN|AIR|onic foxes. quickly final fox|
+4099|163|10|5|48|51031.68|0.00|0.02|R|F|1992-10-18|1992-10-14|1992-11-01|NONE|REG AIR|ts haggle according to the slyly f|
+4099|59|10|6|39|37402.95|0.07|0.02|R|F|1992-12-13|1992-11-13|1992-12-26|DELIVER IN PERSON|REG AIR|fluffy accounts impress pending, iro|
+4099|180|8|7|46|49688.28|0.06|0.07|R|F|1992-10-29|1992-11-03|1992-11-10|DELIVER IN PERSON|REG AIR|ages nag requests.|
+4100|74|5|1|4|3896.28|0.03|0.03|N|O|1996-06-20|1996-04-29|1996-06-21|TAKE BACK RETURN|FOB|lyly regular, bold requ|
+4101|115|2|1|22|22332.42|0.05|0.02|R|F|1994-02-02|1994-02-19|1994-02-12|COLLECT COD|AIR|ly express instructions. careful|
+4102|10|3|1|17|15470.17|0.02|0.02|N|O|1996-06-03|1996-05-06|1996-07-02|COLLECT COD|AIR|ly silent theodolites sleep unusual exc|
+4102|69|8|2|5|4845.30|0.08|0.02|N|O|1996-05-11|1996-05-11|1996-05-16|COLLECT COD|AIR| the furiously even|
+4102|67|4|3|39|37715.34|0.08|0.01|N|O|1996-04-14|1996-05-18|1996-04-20|DELIVER IN PERSON|AIR|ffix blithely slyly special |
+4102|140|6|4|39|40565.46|0.02|0.00|N|O|1996-06-15|1996-06-06|1996-06-30|DELIVER IN PERSON|SHIP|y among the furiously special|
+4102|1|6|5|32|28832.00|0.08|0.01|N|O|1996-05-14|1996-04-29|1996-05-29|NONE|RAIL| the even requests; regular pinto|
+4102|137|8|6|7|7259.91|0.02|0.01|N|O|1996-06-19|1996-05-21|1996-07-15|NONE|REG AIR|bove the carefully pending the|
+4103|75|4|1|40|39002.80|0.05|0.03|R|F|1992-09-19|1992-08-14|1992-09-21|COLLECT COD|RAIL|usly across the slyly busy accounts! fin|
+4128|196|8|1|5|5480.95|0.04|0.04|N|O|1995-10-18|1995-11-28|1995-10-28|TAKE BACK RETURN|FOB|ake permanently |
+4129|56|8|1|32|30593.60|0.03|0.04|A|F|1993-09-16|1993-08-25|1993-09-25|TAKE BACK RETURN|MAIL|ckages haggl|
+4129|27|6|2|39|36153.78|0.06|0.07|R|F|1993-10-21|1993-08-04|1993-10-29|COLLECT COD|MAIL|y regular foxes. slyly ironic deposits |
+4130|178|6|1|44|47439.48|0.07|0.04|N|O|1996-05-14|1996-04-15|1996-05-15|COLLECT COD|TRUCK|eaves haggle qui|
+4130|63|10|2|2|1926.12|0.05|0.06|N|O|1996-05-19|1996-04-24|1996-06-17|TAKE BACK RETURN|RAIL|uriously regular instructions around th|
+4131|50|7|1|6|5700.30|0.05|0.01|N|O|1998-04-27|1998-04-18|1998-04-29|TAKE BACK RETURN|MAIL|ns cajole slyly. even, iro|
+4131|178|8|2|32|34501.44|0.08|0.01|N|O|1998-03-02|1998-03-21|1998-03-07|TAKE BACK RETURN|TRUCK| furiously regular asymptotes nod sly|
+4131|26|9|3|25|23150.50|0.02|0.07|N|O|1998-02-24|1998-03-01|1998-02-27|TAKE BACK RETURN|FOB|uickly exp|
+4131|36|7|4|8|7488.24|0.04|0.01|N|O|1998-03-03|1998-03-15|1998-03-26|COLLECT COD|FOB| after the furiously ironic d|
+4131|125|6|5|30|30753.60|0.01|0.01|N|O|1998-04-01|1998-04-13|1998-04-08|TAKE BACK RETURN|FOB|he fluffily express depen|
+4131|102|7|6|47|47098.70|0.02|0.00|N|O|1998-03-09|1998-04-05|1998-03-13|TAKE BACK RETURN|RAIL|ges. ironic pinto be|
+4132|138|4|1|28|29067.64|0.07|0.03|N|O|1995-08-16|1995-08-01|1995-08-29|TAKE BACK RETURN|SHIP|pths wake against the stealthily special pi|
+4132|15|5|2|23|21045.23|0.07|0.07|N|O|1995-06-27|1995-07-27|1995-07-13|TAKE BACK RETURN|FOB|d deposits. fluffily even requests haggle b|
+4132|87|8|3|18|17767.44|0.09|0.04|A|F|1995-06-01|1995-08-01|1995-06-02|TAKE BACK RETURN|RAIL|y final de|
+4133|24|5|1|35|32340.70|0.02|0.00|A|F|1992-11-25|1992-09-15|1992-12-25|NONE|AIR|g above the quickly bold packages. ev|
+4134|121|4|1|34|34718.08|0.02|0.05|R|F|1995-04-29|1995-03-13|1995-05-11|DELIVER IN PERSON|FOB|e furiously regular sheaves sleep|
+4134|96|10|2|34|33867.06|0.01|0.03|A|F|1995-05-06|1995-03-28|1995-05-13|DELIVER IN PERSON|SHIP|ual asymptotes wake carefully alo|
+4134|171|9|3|12|12854.04|0.05|0.04|A|F|1995-03-19|1995-03-27|1995-04-14|COLLECT COD|TRUCK|kly above the quickly regular |
+4134|100|4|4|45|45004.50|0.08|0.02|A|F|1995-04-11|1995-03-27|1995-04-17|TAKE BACK RETURN|MAIL|ironic pin|
+4135|2|3|1|23|20746.00|0.06|0.01|N|O|1997-04-09|1997-05-12|1997-04-16|TAKE BACK RETURN|FOB|posits cajole furiously carefully|
+4135|120|1|2|32|32643.84|0.07|0.00|N|O|1997-03-14|1997-04-23|1997-04-12|TAKE BACK RETURN|TRUCK| ideas. requests use. furiously|
+4135|160|5|3|33|34985.28|0.05|0.05|N|O|1997-05-01|1997-05-23|1997-05-23|DELIVER IN PERSON|AIR|he fluffil|
+4135|195|6|4|13|14237.47|0.04|0.07|N|O|1997-03-16|1997-05-19|1997-04-03|COLLECT COD|RAIL|efully special account|
+4160|113|10|1|25|25327.75|0.10|0.04|N|O|1996-09-22|1996-10-17|1996-09-24|NONE|SHIP|ar accounts sleep blithe|
+4160|122|7|2|12|12265.44|0.00|0.03|N|O|1996-11-22|1996-09-25|1996-12-10|DELIVER IN PERSON|REG AIR|y bold package|
+4160|63|4|3|48|46226.88|0.04|0.04|N|O|1996-09-19|1996-11-02|1996-09-24|COLLECT COD|FOB| unusual dolphins |
+4161|122|7|1|12|12265.44|0.08|0.02|R|F|1993-08-25|1993-10-04|1993-09-22|COLLECT COD|RAIL|onic dolphins. in|
+4161|28|3|2|47|43616.94|0.05|0.00|A|F|1993-12-20|1993-10-29|1994-01-19|TAKE BACK RETURN|RAIL|r requests about the final, even foxes hag|
+4161|138|4|3|42|43601.46|0.03|0.04|R|F|1993-11-12|1993-10-04|1993-11-27|COLLECT COD|MAIL|thely across the even attainments. express|
+4161|10|5|4|45|40950.45|0.02|0.06|A|F|1993-10-22|1993-10-17|1993-10-30|COLLECT COD|REG AIR|about the ironic packages cajole blithe|
+4161|29|10|5|46|42734.92|0.05|0.01|A|F|1993-11-09|1993-11-17|1993-11-17|TAKE BACK RETURN|TRUCK|he stealthily ironic foxes. ideas haggl|
+4161|148|9|6|19|19914.66|0.07|0.00|R|F|1993-08-22|1993-11-11|1993-09-01|TAKE BACK RETURN|REG AIR|beans breach s|
+4162|74|3|1|45|43833.15|0.10|0.07|A|F|1992-03-21|1992-05-02|1992-03-29|DELIVER IN PERSON|AIR|elets. slyly regular i|
+4162|90|1|2|29|28712.61|0.00|0.05|R|F|1992-02-25|1992-04-25|1992-03-17|NONE|REG AIR|nding pinto beans haggle blithe|
+4163|33|4|1|13|12129.39|0.08|0.03|A|F|1993-02-17|1993-03-13|1993-03-15|DELIVER IN PERSON|REG AIR|phins wake. pending requests inte|
+4164|120|7|1|9|9181.08|0.07|0.02|N|O|1998-08-25|1998-08-13|1998-09-19|DELIVER IN PERSON|SHIP|re fluffily slyly bold requests. |
+4165|41|2|1|12|11292.48|0.00|0.01|N|O|1997-09-20|1997-10-20|1997-10-12|TAKE BACK RETURN|REG AIR|nwind slow theodolites. carefully pending |
+4166|141|10|1|8|8329.12|0.00|0.08|A|F|1993-06-05|1993-04-10|1993-07-05|COLLECT COD|MAIL|uickly. blithely pending de|
+4166|93|5|2|8|7944.72|0.06|0.04|A|F|1993-06-07|1993-04-17|1993-06-16|DELIVER IN PERSON|REG AIR|es along the furiously regular acc|
+4166|7|10|3|17|15419.00|0.02|0.06|R|F|1993-06-29|1993-05-15|1993-07-24|DELIVER IN PERSON|SHIP|ackages. re|
+4166|86|7|4|36|35498.88|0.06|0.05|R|F|1993-03-01|1993-05-25|1993-03-05|COLLECT COD|MAIL|unts. furiously express accounts w|
+4166|77|6|5|5|4885.35|0.08|0.01|A|F|1993-06-19|1993-04-24|1993-06-27|NONE|REG AIR|hely unusual packages are above the f|
+4166|102|5|6|6|6012.60|0.04|0.08|R|F|1993-04-30|1993-04-17|1993-05-08|DELIVER IN PERSON|MAIL|ily ironic deposits print furiously. iron|
+4166|24|5|7|26|24024.52|0.09|0.01|R|F|1993-03-17|1993-05-09|1993-03-25|NONE|MAIL|lar dependencies. s|
+4167|61|8|1|47|45169.82|0.04|0.02|N|O|1998-08-02|1998-08-24|1998-08-28|DELIVER IN PERSON|REG AIR| carefully final asymptotes. slyly bo|
+4167|87|8|2|17|16780.36|0.06|0.07|N|O|1998-09-18|1998-09-06|1998-10-07|COLLECT COD|REG AIR|ly around the even instr|
+4167|73|3|3|1|973.07|0.03|0.06|N|O|1998-10-11|1998-08-14|1998-10-13|COLLECT COD|TRUCK|xpress platelets. blithely |
+4192|11|1|1|36|32796.36|0.06|0.08|N|O|1998-04-25|1998-05-26|1998-05-03|COLLECT COD|TRUCK|eodolites sleep|
+4192|121|6|2|15|15316.80|0.04|0.08|N|O|1998-06-26|1998-05-26|1998-07-16|COLLECT COD|AIR|e slyly special grouches. express pinto b|
+4192|135|6|3|7|7245.91|0.06|0.03|N|O|1998-05-19|1998-07-08|1998-05-31|COLLECT COD|FOB|y; excuses use. ironic, close instru|
+4192|24|3|4|32|29568.64|0.09|0.04|N|O|1998-06-23|1998-06-25|1998-07-17|NONE|FOB|ounts are fluffily slyly bold req|
+4192|48|7|5|48|45505.92|0.08|0.01|N|O|1998-08-17|1998-07-11|1998-09-03|NONE|AIR|ests. quickly bol|
+4192|150|7|6|44|46206.60|0.10|0.02|N|O|1998-08-06|1998-07-09|1998-08-20|NONE|FOB|structions mai|
+4192|170|5|7|27|28894.59|0.02|0.00|N|O|1998-07-03|1998-06-26|1998-07-13|TAKE BACK RETURN|AIR| carefully even escapades. care|
+4193|131|7|1|37|38151.81|0.09|0.06|A|F|1994-04-25|1994-02-24|1994-05-08|NONE|AIR|er the quickly regular dependencies wake|
+4193|117|7|2|3|3051.33|0.09|0.05|R|F|1994-04-29|1994-03-20|1994-05-29|TAKE BACK RETURN|REG AIR|osits above the depo|
+4193|179|10|3|10|10791.70|0.06|0.03|A|F|1994-02-10|1994-03-22|1994-03-09|COLLECT COD|RAIL|uffily spe|
+4193|51|9|4|29|27580.45|0.09|0.05|A|F|1994-02-11|1994-03-11|1994-03-13|TAKE BACK RETURN|RAIL|ly. final packages use blit|
+4193|20|7|5|50|46001.00|0.01|0.01|R|F|1994-04-28|1994-03-23|1994-05-09|NONE|FOB| beans. regular accounts cajole. de|
+4193|66|1|6|21|20287.26|0.02|0.04|R|F|1994-04-26|1994-03-22|1994-05-23|DELIVER IN PERSON|TRUCK|accounts cajole b|
+4194|197|1|1|43|47179.17|0.08|0.06|A|F|1994-11-06|1994-12-09|1994-11-16|NONE|TRUCK|olites are after the exp|
+4194|47|10|2|18|17046.72|0.07|0.07|A|F|1995-02-14|1994-12-04|1995-03-11|TAKE BACK RETURN|TRUCK|ld packages. quickly eve|
+4195|6|9|1|14|12684.00|0.09|0.04|R|F|1993-09-06|1993-07-21|1993-09-18|DELIVER IN PERSON|REG AIR|ironic packages. carefully express|
+4195|66|1|2|22|21253.32|0.10|0.08|R|F|1993-07-01|1993-07-23|1993-07-28|COLLECT COD|RAIL|lly express pinto bea|
+4195|194|8|3|19|20789.61|0.01|0.06|R|F|1993-09-06|1993-08-13|1993-09-15|TAKE BACK RETURN|REG AIR|telets sleep even requests. final, even i|
+4196|156|4|1|30|31684.50|0.02|0.06|N|O|1998-08-09|1998-06-30|1998-09-05|COLLECT COD|SHIP|egular foxes us|
+4196|9|6|2|31|28179.00|0.09|0.08|N|O|1998-06-12|1998-07-28|1998-07-11|NONE|MAIL|ut the blithely ironic inst|
+4196|178|9|3|46|49595.82|0.05|0.00|N|O|1998-09-05|1998-06-28|1998-09-10|TAKE BACK RETURN|MAIL|according to t|
+4196|114|8|4|42|42592.62|0.04|0.06|N|O|1998-08-13|1998-07-18|1998-09-07|TAKE BACK RETURN|AIR| instructions. courts cajole slyly ev|
+4196|72|2|5|3|2916.21|0.01|0.03|N|O|1998-05-17|1998-07-21|1998-05-18|DELIVER IN PERSON|TRUCK| accounts. fu|
+4196|87|8|6|43|42444.44|0.01|0.06|N|O|1998-08-12|1998-07-12|1998-08-22|DELIVER IN PERSON|FOB|es. slyly even |
+4196|4|1|7|3|2712.00|0.00|0.06|N|O|1998-08-05|1998-07-28|1998-08-15|DELIVER IN PERSON|REG AIR|y regular packages haggle furiously alongs|
+4197|129|8|1|50|51456.00|0.06|0.03|N|O|1996-11-15|1996-11-01|1996-11-20|NONE|FOB|. carefully bold asymptotes nag blithe|
+4197|70|9|2|39|37832.73|0.02|0.08|N|O|1996-10-07|1996-10-11|1996-10-18|DELIVER IN PERSON|RAIL|ronic requests. quickly bold packages in|
+4197|32|8|3|28|26096.84|0.06|0.02|N|O|1996-10-05|1996-10-24|1996-10-22|TAKE BACK RETURN|AIR|regular pin|
+4197|96|7|4|23|22910.07|0.00|0.03|N|O|1996-09-10|1996-10-10|1996-09-25|NONE|AIR|l instructions print slyly past the reg|
+4197|121|6|5|37|37781.44|0.03|0.04|N|O|1996-10-20|1996-10-10|1996-11-10|COLLECT COD|TRUCK|carefully enticing decoys boo|
+4197|31|7|6|48|44689.44|0.08|0.00|N|O|1996-10-07|1996-10-25|1996-10-23|COLLECT COD|REG AIR| final instructions. blithe, spe|
+4198|146|9|1|48|50214.72|0.09|0.05|N|O|1997-09-03|1997-07-18|1997-09-11|NONE|REG AIR|cajole carefully final, ironic ide|
+4198|143|6|2|46|47984.44|0.09|0.01|N|O|1997-08-17|1997-09-08|1997-09-11|COLLECT COD|TRUCK|posits among th|
+4198|145|4|3|13|13586.82|0.03|0.04|N|O|1997-07-18|1997-07-24|1997-08-10|NONE|REG AIR| furious excuses. bli|
+4199|70|5|1|16|15521.12|0.10|0.00|A|F|1992-06-11|1992-04-10|1992-07-10|COLLECT COD|TRUCK|ncies. furiously special accounts|
+4199|9|10|2|18|16362.00|0.00|0.01|A|F|1992-06-01|1992-03-30|1992-06-28|DELIVER IN PERSON|RAIL|pending, regular accounts. carefully|
+4224|199|10|1|27|29678.13|0.05|0.03|N|O|1997-09-05|1997-08-19|1997-09-30|NONE|SHIP|ly special deposits sleep qui|
+4224|37|3|2|20|18740.60|0.07|0.05|N|O|1997-11-09|1997-08-23|1997-11-14|NONE|FOB|unts promise across the requests. blith|
+4224|24|7|3|4|3696.08|0.08|0.05|N|O|1997-09-07|1997-09-05|1997-09-25|TAKE BACK RETURN|FOB| even dinos. carefull|
+4224|160|2|4|50|53008.00|0.10|0.06|N|O|1997-07-30|1997-09-10|1997-08-19|COLLECT COD|RAIL|side of the carefully silent dep|
+4224|85|6|5|48|47283.84|0.00|0.04|N|O|1997-10-03|1997-08-31|1997-10-10|NONE|RAIL| final, regular asymptotes use alway|
+4225|49|8|1|25|23726.00|0.08|0.04|N|O|1997-07-10|1997-08-08|1997-07-31|TAKE BACK RETURN|TRUCK|se fluffily. busily ironic requests are;|
+4225|96|8|2|23|22910.07|0.02|0.04|N|O|1997-09-18|1997-08-31|1997-10-11|TAKE BACK RETURN|RAIL|. quickly b|
+4225|98|10|3|28|27946.52|0.08|0.03|N|O|1997-07-11|1997-09-01|1997-08-03|DELIVER IN PERSON|FOB|ts are requests. even, bold depos|
+4226|188|9|1|27|29380.86|0.06|0.08|A|F|1993-05-03|1993-04-12|1993-05-16|COLLECT COD|AIR|sly alongside of the slyly ironic pac|
+4227|158|6|1|19|20104.85|0.01|0.08|A|F|1995-05-05|1995-05-03|1995-05-22|COLLECT COD|REG AIR|ns sleep along the blithely even theodolit|
+4227|33|4|2|8|7464.24|0.09|0.00|N|F|1995-06-11|1995-04-30|1995-06-28|COLLECT COD|REG AIR| packages since the bold, u|
+4227|75|6|3|11|10725.77|0.10|0.04|A|F|1995-03-30|1995-05-02|1995-04-26|DELIVER IN PERSON|SHIP|l requests-- bold requests cajole dogg|
+4227|200|4|4|2|2200.40|0.02|0.05|R|F|1995-04-24|1995-05-09|1995-05-21|DELIVER IN PERSON|AIR|ep. specia|
+4227|147|6|5|49|51309.86|0.05|0.06|R|F|1995-05-19|1995-04-12|1995-06-12|TAKE BACK RETURN|REG AIR|ts sleep blithely carefully unusual ideas.|
+4228|141|10|1|20|20822.80|0.00|0.06|N|O|1997-04-24|1997-05-29|1997-05-17|NONE|RAIL|f the slyly fluffy pinto beans are|
+4229|96|9|1|44|43827.96|0.02|0.05|N|O|1998-05-29|1998-05-12|1998-06-16|DELIVER IN PERSON|AIR|s. carefully e|
+4229|5|8|2|34|30770.00|0.07|0.05|N|O|1998-05-26|1998-04-13|1998-06-08|DELIVER IN PERSON|MAIL|thely final accounts use even packa|
+4230|46|5|1|38|35949.52|0.10|0.03|A|F|1992-04-28|1992-04-21|1992-05-28|TAKE BACK RETURN|FOB|ly regular packages. regular ideas boost|
+4230|199|3|2|43|47265.17|0.02|0.08|R|F|1992-03-14|1992-05-13|1992-03-28|NONE|FOB|ses lose blithely slyly final e|
+4230|196|9|3|10|10961.90|0.06|0.02|A|F|1992-06-11|1992-04-11|1992-07-02|TAKE BACK RETURN|MAIL|ar packages are |
+4230|75|6|4|28|27301.96|0.01|0.03|R|F|1992-05-12|1992-05-10|1992-06-01|TAKE BACK RETURN|MAIL|nt instruct|
+4230|125|10|5|50|51256.00|0.00|0.01|A|F|1992-03-29|1992-05-19|1992-04-20|TAKE BACK RETURN|SHIP|ts. final instructions in|
+4230|35|6|6|30|28050.90|0.05|0.07|A|F|1992-03-11|1992-04-29|1992-03-30|NONE|AIR|s. final excuses across the|
+4230|152|3|7|18|18938.70|0.10|0.04|R|F|1992-06-23|1992-05-10|1992-07-04|COLLECT COD|SHIP| the final acco|
+4231|142|3|1|47|48980.58|0.09|0.03|N|O|1997-11-27|1998-01-26|1997-12-17|NONE|REG AIR|hely along the silent at|
+4231|166|3|2|4|4264.64|0.06|0.02|N|O|1997-11-28|1998-01-26|1997-12-12|TAKE BACK RETURN|MAIL|lithely even packages. |
+4231|121|2|3|31|31654.72|0.07|0.08|N|O|1998-02-14|1997-12-27|1998-03-01|DELIVER IN PERSON|FOB|ublate. theodoli|
+4231|40|1|4|35|32901.40|0.10|0.00|N|O|1998-02-21|1998-01-24|1998-03-18|DELIVER IN PERSON|FOB|le quickly regular, unus|
+4256|151|9|1|22|23125.30|0.05|0.05|R|F|1992-07-30|1992-05-14|1992-08-14|NONE|TRUCK|, final platelets are slyly final pint|
+4257|65|10|1|3|2895.18|0.10|0.03|N|O|1995-06-18|1995-05-01|1995-07-12|DELIVER IN PERSON|MAIL|thin the theodolites use after the bl|
+4257|35|6|2|5|4675.15|0.01|0.04|R|F|1995-04-29|1995-06-05|1995-05-13|TAKE BACK RETURN|TRUCK|n deposits. furiously e|
+4257|128|9|3|33|33927.96|0.03|0.04|A|F|1995-05-23|1995-05-03|1995-05-31|COLLECT COD|AIR|uffily regular accounts ar|
+4258|166|7|1|36|38381.76|0.02|0.06|N|O|1997-02-23|1997-01-25|1997-02-27|TAKE BACK RETURN|SHIP|ns use alongs|
+4258|162|1|2|19|20181.04|0.03|0.02|N|O|1997-01-14|1996-12-12|1997-01-20|TAKE BACK RETURN|AIR|ly busily ironic foxes. f|
+4258|31|7|3|46|42827.38|0.04|0.07|N|O|1997-01-02|1996-12-26|1997-01-12|DELIVER IN PERSON|AIR| furiously pend|
+4258|35|6|4|22|20570.66|0.04|0.04|N|O|1996-12-12|1996-12-06|1996-12-20|TAKE BACK RETURN|AIR|e regular, even asym|
+4258|163|10|5|9|9568.44|0.04|0.03|N|O|1996-12-04|1996-12-08|1996-12-20|DELIVER IN PERSON|TRUCK|counts wake permanently after the bravely|
+4259|43|6|1|14|13202.56|0.05|0.03|N|O|1998-01-09|1997-11-21|1998-01-29|TAKE BACK RETURN|RAIL| furiously pending excuses. ideas hagg|
+4260|24|7|1|21|19404.42|0.08|0.04|R|F|1992-08-06|1992-06-18|1992-08-22|NONE|AIR|al, pending accounts must|
+4261|110|1|1|12|12121.32|0.05|0.01|A|F|1992-11-01|1993-01-01|1992-11-12|NONE|FOB|into beans |
+4261|82|3|2|4|3928.32|0.02|0.07|R|F|1992-12-11|1992-12-18|1992-12-24|DELIVER IN PERSON|FOB|ackages unwind furiously fluff|
+4261|175|5|3|3|3225.51|0.07|0.02|R|F|1992-11-10|1992-12-14|1992-11-17|COLLECT COD|RAIL|ly even deposits eat blithely alo|
+4261|174|3|4|36|38670.12|0.04|0.06|R|F|1992-12-02|1992-12-18|1992-12-25|NONE|REG AIR| slyly pendi|
+4261|24|7|5|28|25872.56|0.07|0.06|A|F|1992-10-08|1992-12-23|1992-10-11|TAKE BACK RETURN|MAIL|packages. fluffily i|
+4262|76|7|1|30|29282.10|0.01|0.03|N|O|1996-08-11|1996-10-11|1996-09-09|TAKE BACK RETURN|RAIL|tes after the carefully|
+4262|96|7|2|5|4980.45|0.02|0.05|N|O|1996-09-27|1996-09-05|1996-10-25|COLLECT COD|SHIP|blithely final asymptotes integrate|
+4262|162|1|3|5|5310.80|0.08|0.00|N|O|1996-10-02|1996-10-16|1996-10-05|NONE|REG AIR|ironic accounts are unusu|
+4262|74|2|4|45|43833.15|0.02|0.01|N|O|1996-11-09|1996-09-09|1996-11-12|DELIVER IN PERSON|SHIP|ackages boost. pending, even instruction|
+4262|100|3|5|28|28002.80|0.06|0.02|N|O|1996-10-22|1996-09-06|1996-11-13|DELIVER IN PERSON|FOB|ironic, regular depend|
+4262|17|7|6|26|23842.26|0.03|0.02|N|O|1996-08-29|1996-09-25|1996-08-31|NONE|RAIL|s boost slyly along the bold, iro|
+4262|160|5|7|41|43466.56|0.03|0.01|N|O|1996-08-28|1996-09-14|1996-09-20|COLLECT COD|RAIL|cuses unwind ac|
+4263|18|9|1|9|8262.09|0.08|0.07|N|O|1998-04-04|1998-04-29|1998-05-04|COLLECT COD|AIR|structions cajole quic|
+4263|196|10|2|28|30693.32|0.05|0.03|N|O|1998-06-24|1998-06-08|1998-07-14|NONE|MAIL|ideas for the carefully re|
+4263|11|1|3|38|34618.38|0.01|0.01|N|O|1998-07-10|1998-05-08|1998-07-17|NONE|TRUCK|rding to the dep|
+4263|19|3|4|20|18380.20|0.02|0.07|N|O|1998-04-09|1998-04-30|1998-05-04|NONE|RAIL|uietly regular deposits. sly deposits w|
+4263|198|2|5|14|15374.66|0.09|0.06|N|O|1998-05-06|1998-04-17|1998-05-11|DELIVER IN PERSON|TRUCK|d accounts. daringly regular accounts hagg|
+4263|113|10|6|47|47616.17|0.08|0.06|N|O|1998-06-28|1998-05-09|1998-07-02|DELIVER IN PERSON|TRUCK|y. theodolites wake idly ironic do|
+4263|29|4|7|6|5574.12|0.04|0.04|N|O|1998-05-01|1998-06-02|1998-05-14|TAKE BACK RETURN|REG AIR|g the final, regular instructions: |
+4288|74|5|1|32|31170.24|0.10|0.07|R|F|1993-03-19|1993-01-26|1993-04-18|TAKE BACK RETURN|AIR|e blithely even instructions. speci|
+4288|105|6|2|39|39198.90|0.05|0.02|R|F|1993-03-25|1993-02-06|1993-03-28|DELIVER IN PERSON|AIR|uffy theodolites run|
+4288|125|8|3|7|7175.84|0.03|0.01|A|F|1993-01-15|1993-02-05|1993-01-26|NONE|TRUCK|ngside of the special platelet|
+4289|196|7|1|19|20827.61|0.06|0.06|R|F|1993-12-31|1993-11-06|1994-01-23|DELIVER IN PERSON|TRUCK|e carefully regular ideas. sl|
+4290|137|3|1|23|23853.99|0.06|0.04|R|F|1995-04-04|1995-02-16|1995-04-07|TAKE BACK RETURN|REG AIR|uests cajole carefully.|
+4290|99|2|2|3|2997.27|0.09|0.03|A|F|1995-03-25|1995-03-07|1995-04-11|NONE|RAIL|lar platelets cajole|
+4291|192|6|1|3|3276.57|0.08|0.08|A|F|1994-03-17|1994-02-21|1994-03-27|COLLECT COD|SHIP|tes sleep slyly above the quickly sl|
+4291|125|8|2|43|44080.16|0.01|0.06|A|F|1994-02-01|1994-02-27|1994-02-06|DELIVER IN PERSON|REG AIR|s. quietly regular |
+4291|8|1|3|25|22700.00|0.09|0.08|R|F|1994-02-14|1994-02-08|1994-03-15|COLLECT COD|AIR|uctions. furiously regular ins|
+4292|44|3|1|22|20768.88|0.08|0.03|R|F|1992-02-14|1992-02-16|1992-03-01|NONE|FOB|refully expres|
+4292|40|6|2|1|940.04|0.03|0.01|A|F|1992-02-07|1992-03-16|1992-02-10|DELIVER IN PERSON|FOB| the furiously ev|
+4292|120|10|3|35|35704.20|0.03|0.06|A|F|1992-03-23|1992-04-04|1992-04-02|COLLECT COD|TRUCK|dugouts use. furiously bold packag|
+4292|163|10|4|40|42526.40|0.05|0.04|A|F|1992-04-27|1992-03-07|1992-05-04|COLLECT COD|REG AIR|ounts according to the furiously |
+4292|131|7|5|6|6186.78|0.07|0.08|R|F|1992-03-03|1992-02-24|1992-03-25|COLLECT COD|FOB|bove the silently regula|
+4292|4|1|6|47|42488.00|0.05|0.00|R|F|1992-05-02|1992-03-21|1992-05-27|TAKE BACK RETURN|FOB|y packages; even ideas boost|
+4293|1|6|1|34|30634.00|0.03|0.08|N|O|1996-11-05|1996-10-12|1996-12-04|NONE|FOB|ions sleep blithely on|
+4293|77|5|2|50|48853.50|0.01|0.05|N|O|1996-11-27|1996-10-30|1996-12-22|COLLECT COD|MAIL| special deposits. furiousl|
+4293|199|1|3|47|51661.93|0.08|0.02|N|O|1996-09-07|1996-10-24|1996-09-15|NONE|RAIL|ithely pending deposits af|
+4293|88|9|4|25|24702.00|0.04|0.04|N|O|1996-09-11|1996-11-14|1996-09-22|DELIVER IN PERSON|FOB|inal asympt|
+4293|181|2|5|1|1081.18|0.06|0.05|N|O|1996-11-15|1996-10-09|1996-11-26|COLLECT COD|AIR|eposits should boost along the |
+4293|79|7|6|45|44058.15|0.10|0.04|N|O|1996-11-04|1996-11-06|1996-11-23|NONE|MAIL|lar ideas use carefully|
+4294|105|8|1|19|19096.90|0.03|0.04|A|F|1992-10-16|1992-11-13|1992-10-26|DELIVER IN PERSON|AIR|nt dependencies. furiously regular ideas d|
+4294|27|2|2|16|14832.32|0.01|0.02|R|F|1992-08-17|1992-09-24|1992-09-04|TAKE BACK RETURN|REG AIR|lithely pint|
+4294|198|1|3|30|32945.70|0.01|0.00|A|F|1992-09-12|1992-11-06|1992-09-25|NONE|MAIL|olites. bold foxes affix ironic theodolite|
+4294|105|2|4|34|34173.40|0.02|0.01|R|F|1992-09-09|1992-11-06|1992-10-04|TAKE BACK RETURN|REG AIR|pendencies!|
+4294|119|3|5|37|37707.07|0.05|0.01|R|F|1992-09-07|1992-10-13|1992-09-08|NONE|REG AIR|cial packages nag f|
+4294|87|8|6|42|41457.36|0.02|0.03|A|F|1992-09-30|1992-11-13|1992-10-15|DELIVER IN PERSON|FOB| carefully; furiously ex|
+4294|175|3|7|47|50532.99|0.02|0.08|R|F|1992-11-09|1992-11-03|1992-12-05|TAKE BACK RETURN|SHIP|es. blithely r|
+4295|29|2|1|49|45521.98|0.09|0.01|N|O|1996-05-25|1996-03-17|1996-06-19|TAKE BACK RETURN|REG AIR|refully silent requests. f|
+4295|71|9|2|4|3884.28|0.09|0.07|N|O|1996-06-05|1996-04-26|1996-06-13|DELIVER IN PERSON|TRUCK|arefully according to the pending ac|
+4295|193|4|3|3|3279.57|0.04|0.00|N|O|1996-06-04|1996-04-24|1996-06-24|DELIVER IN PERSON|AIR|telets cajole bravely|
+4295|80|9|4|30|29402.40|0.07|0.06|N|O|1996-03-22|1996-04-23|1996-04-20|NONE|SHIP|yly ironic frets. pending foxes after |
+4320|46|5|1|28|26489.12|0.02|0.06|N|O|1997-01-28|1997-02-07|1997-02-07|COLLECT COD|FOB|nts. even, ironic excuses hagg|
+4320|140|6|2|6|6240.84|0.08|0.08|N|O|1997-01-11|1997-01-26|1997-01-22|DELIVER IN PERSON|SHIP|against the carefully careful asym|
+4320|188|9|3|33|35909.94|0.09|0.02|N|O|1996-12-11|1997-02-27|1997-01-08|TAKE BACK RETURN|SHIP|ess asymptotes so|
+4321|147|6|1|33|34555.62|0.09|0.02|A|F|1994-09-01|1994-08-17|1994-09-05|DELIVER IN PERSON|TRUCK|yly special excuses. fluffily |
+4321|54|2|2|45|42932.25|0.00|0.08|R|F|1994-11-13|1994-09-15|1994-11-18|DELIVER IN PERSON|SHIP| haggle ironically bold theodolites. quick|
+4321|186|7|3|23|24982.14|0.01|0.05|A|F|1994-11-03|1994-10-08|1994-11-06|DELIVER IN PERSON|SHIP|ly even orbits slee|
+4321|91|2|4|4|3964.36|0.02|0.00|R|F|1994-09-10|1994-10-06|1994-09-11|NONE|FOB|ironic deposi|
+4321|172|2|5|10|10721.70|0.04|0.03|A|F|1994-09-07|1994-08-23|1994-09-17|TAKE BACK RETURN|SHIP|wake carefully alongside of |
+4322|69|4|1|39|37793.34|0.04|0.02|N|O|1998-04-27|1998-06-03|1998-05-04|TAKE BACK RETURN|MAIL|its integrate fluffily |
+4322|140|1|2|9|9361.26|0.05|0.08|N|O|1998-05-18|1998-04-27|1998-05-28|COLLECT COD|AIR|ual instructio|
+4322|8|9|3|12|10896.00|0.09|0.05|N|O|1998-03-29|1998-06-05|1998-04-16|DELIVER IN PERSON|TRUCK|e blithely against the slyly unusu|
+4322|46|7|4|17|16082.68|0.09|0.08|N|O|1998-05-31|1998-05-31|1998-06-10|TAKE BACK RETURN|FOB|ructions boost |
+4322|102|7|5|10|10021.00|0.00|0.05|N|O|1998-05-31|1998-04-27|1998-06-25|TAKE BACK RETURN|REG AIR| regular ideas engage carefully quick|
+4322|60|8|6|39|37442.34|0.09|0.08|N|O|1998-03-16|1998-05-21|1998-04-11|COLLECT COD|AIR|ccounts. dogged pin|
+4322|14|4|7|34|31076.34|0.05|0.00|N|O|1998-05-27|1998-04-12|1998-06-16|NONE|REG AIR|ounts haggle fluffily ideas. pend|
+4323|1|2|1|33|29733.00|0.09|0.02|A|F|1994-05-04|1994-03-06|1994-05-23|COLLECT COD|TRUCK|the slyly bold deposits slee|
+4324|51|2|1|44|41846.20|0.05|0.04|N|O|1995-10-15|1995-09-07|1995-11-07|DELIVER IN PERSON|AIR|ainst the u|
+4324|48|7|2|12|11376.48|0.04|0.02|N|O|1995-10-05|1995-09-07|1995-10-18|NONE|REG AIR|c packages. furiously express sauternes|
+4324|82|3|3|14|13749.12|0.07|0.06|N|O|1995-11-12|1995-08-26|1995-11-21|COLLECT COD|AIR| packages nag express excuses. qui|
+4324|50|7|4|14|13300.70|0.02|0.04|N|O|1995-09-20|1995-10-08|1995-10-06|COLLECT COD|RAIL| express ideas. blithely blit|
+4324|84|5|5|22|21649.76|0.07|0.03|N|O|1995-09-13|1995-10-04|1995-09-23|DELIVER IN PERSON|SHIP|ke express, special ideas.|
+4324|43|2|6|31|29234.24|0.08|0.04|N|O|1995-10-23|1995-09-14|1995-11-09|COLLECT COD|RAIL|efully flu|
+4324|154|6|7|46|48490.90|0.00|0.03|N|O|1995-11-03|1995-09-28|1995-11-22|NONE|SHIP|ular, final theodo|
+4325|160|2|1|18|19082.88|0.01|0.07|N|O|1996-10-07|1996-09-28|1996-10-31|DELIVER IN PERSON|RAIL|. blithely|
+4326|163|4|1|11|11694.76|0.01|0.01|N|O|1997-02-02|1996-12-10|1997-02-20|DELIVER IN PERSON|TRUCK|press reque|
+4326|167|6|2|27|28813.32|0.06|0.01|N|O|1996-11-29|1997-01-20|1996-12-23|COLLECT COD|AIR|inal packages. final asymptotes about t|
+4327|95|8|1|18|17911.62|0.08|0.00|N|F|1995-06-16|1995-04-20|1995-07-12|COLLECT COD|RAIL|y final excuses. ironic, special requests a|
+4327|106|9|2|40|40244.00|0.07|0.01|N|F|1995-05-26|1995-04-17|1995-06-18|NONE|AIR|quests. packages are after th|
+4327|145|2|3|11|11496.54|0.10|0.07|R|F|1995-04-24|1995-05-27|1995-05-24|TAKE BACK RETURN|FOB| ironic dolphins|
+4327|21|10|4|8|7368.16|0.04|0.08|N|F|1995-05-26|1995-05-28|1995-06-19|DELIVER IN PERSON|AIR|eodolites cajole; unusual Tiresias|
+4327|190|1|5|39|42517.41|0.01|0.00|N|O|1995-06-23|1995-04-18|1995-07-13|TAKE BACK RETURN|FOB|kages against the blit|
+4327|152|4|6|10|10521.50|0.00|0.06|A|F|1995-04-28|1995-06-11|1995-05-07|TAKE BACK RETURN|TRUCK|arefully sile|
+4352|106|9|1|18|18109.80|0.00|0.03|N|O|1998-02-27|1998-02-02|1998-03-01|DELIVER IN PERSON|RAIL|ding to th|
+4353|94|8|1|22|21869.98|0.05|0.05|N|O|1998-01-19|1998-01-23|1998-02-10|COLLECT COD|FOB|ent packages. accounts are slyly. |
+4354|15|9|1|30|27450.30|0.08|0.07|R|F|1995-01-27|1994-11-24|1995-02-25|TAKE BACK RETURN|REG AIR|around the ir|
+4354|153|8|2|23|24222.45|0.01|0.08|R|F|1994-11-20|1994-12-23|1994-11-27|TAKE BACK RETURN|AIR|kly along the ironic, ent|
+4354|51|6|3|2|1902.10|0.10|0.04|A|F|1995-01-09|1994-12-15|1995-01-24|TAKE BACK RETURN|REG AIR|s nag quickly |
+4354|86|7|4|36|35498.88|0.05|0.05|A|F|1994-11-20|1994-12-06|1994-12-06|DELIVER IN PERSON|AIR| wake slyly eve|
+4354|65|10|5|37|35707.22|0.06|0.02|R|F|1995-01-13|1994-12-29|1995-01-31|DELIVER IN PERSON|FOB|deas use blithely! special foxes print af|
+4354|108|3|6|36|36291.60|0.03|0.04|R|F|1994-12-03|1994-12-05|1995-01-02|TAKE BACK RETURN|TRUCK|efully special packages use fluffily|
+4354|139|5|7|18|18704.34|0.03|0.04|A|F|1994-12-07|1994-12-11|1994-12-11|TAKE BACK RETURN|SHIP|ross the furiously |
+4355|195|7|1|32|35046.08|0.10|0.02|N|O|1996-12-29|1997-02-08|1997-01-24|DELIVER IN PERSON|REG AIR|y silent deposits. b|
+4355|17|1|2|4|3668.04|0.05|0.02|N|O|1997-02-25|1997-01-29|1997-03-17|TAKE BACK RETURN|TRUCK|slyly blithely regular packag|
+4355|1|2|3|13|11713.00|0.07|0.05|N|O|1997-01-21|1996-12-22|1997-02-14|COLLECT COD|TRUCK| ought to mold. blithely pending ideas |
+4355|194|6|4|14|15318.66|0.04|0.02|N|O|1997-03-08|1997-01-22|1997-03-26|NONE|RAIL|he furiously ironic accounts. quickly iro|
+4355|31|7|5|50|46551.50|0.10|0.00|N|O|1996-11-25|1997-01-01|1996-12-06|DELIVER IN PERSON|REG AIR| regular accounts boost along the |
+4355|122|7|6|35|35774.20|0.00|0.08|N|O|1997-01-28|1997-01-28|1997-02-20|NONE|FOB|ess accounts affix ironic|
+4355|101|4|7|47|47051.70|0.09|0.02|N|O|1996-12-28|1996-12-29|1997-01-09|NONE|RAIL|e. realms integrate |
+4356|194|5|1|35|38296.65|0.00|0.04|R|F|1994-05-30|1994-06-14|1994-06-08|COLLECT COD|MAIL|arefully ironic |
+4357|84|5|1|50|49204.00|0.04|0.07|N|O|1997-11-25|1997-12-03|1997-12-17|DELIVER IN PERSON|RAIL|s. final, e|
+4357|108|9|2|17|17137.70|0.10|0.07|N|O|1998-02-01|1997-12-08|1998-02-09|DELIVER IN PERSON|MAIL|e carefully furiou|
+4358|126|5|1|47|48227.64|0.04|0.00|N|O|1997-10-15|1997-10-14|1997-11-04|DELIVER IN PERSON|SHIP|refully busy dep|
+4359|174|3|1|41|44040.97|0.03|0.07|A|F|1993-04-06|1993-05-06|1993-04-14|COLLECT COD|RAIL|s affix sly|
+4359|153|8|2|8|8425.20|0.03|0.08|R|F|1993-06-27|1993-05-16|1993-07-04|DELIVER IN PERSON|MAIL|packages affix. fluffily regular f|
+4359|193|6|3|32|34982.08|0.10|0.03|R|F|1993-06-18|1993-04-04|1993-07-18|COLLECT COD|MAIL|olites nag quietly caref|
+4359|78|8|4|1|978.07|0.05|0.03|R|F|1993-04-27|1993-05-09|1993-05-08|NONE|MAIL| fluffily ironic, bold pac|
+4359|33|4|5|22|20526.66|0.04|0.01|A|F|1993-03-28|1993-06-01|1993-04-13|NONE|REG AIR|accounts wake ironic deposits. ironic|
+4384|136|7|1|5|5180.65|0.09|0.01|A|F|1992-08-22|1992-08-24|1992-09-20|DELIVER IN PERSON|MAIL|instructions sleep. blithely express pa|
+4384|89|10|2|38|37585.04|0.07|0.06|A|F|1992-10-18|1992-09-24|1992-11-04|NONE|FOB|ly final requests. regu|
+4384|89|10|3|11|10879.88|0.05|0.04|R|F|1992-08-31|1992-10-04|1992-09-28|TAKE BACK RETURN|FOB|deposits promise carefully even, regular e|
+4385|111|8|1|38|38422.18|0.00|0.02|N|O|1996-11-22|1996-10-30|1996-12-21|DELIVER IN PERSON|TRUCK|inal frays. final, bold exc|
+4386|130|3|1|10|10301.30|0.05|0.07|N|O|1998-06-03|1998-04-16|1998-06-28|TAKE BACK RETURN|MAIL|gainst the quickly expre|
+4386|118|2|2|28|28507.08|0.03|0.06|N|O|1998-03-19|1998-05-01|1998-03-27|NONE|FOB|. quick packages play slyly |
+4386|140|1|3|4|4160.56|0.07|0.05|N|O|1998-04-07|1998-03-25|1998-04-19|COLLECT COD|FOB|ns wake carefully carefully iron|
+4386|121|2|4|21|21443.52|0.09|0.00|N|O|1998-05-05|1998-03-19|1998-05-13|NONE|RAIL|e pending, sp|
+4386|130|3|5|39|40175.07|0.09|0.06|N|O|1998-03-05|1998-03-15|1998-03-16|NONE|RAIL|structions cajole quickly express|
+4386|90|1|6|18|17821.62|0.02|0.05|N|O|1998-04-12|1998-04-09|1998-05-12|TAKE BACK RETURN|SHIP| deposits use according to the pending, |
+4386|20|4|7|16|14720.32|0.07|0.02|N|O|1998-05-05|1998-03-17|1998-06-03|COLLECT COD|AIR|e furiously final pint|
+4387|122|5|1|3|3066.36|0.02|0.01|N|O|1996-01-17|1996-01-14|1996-01-28|COLLECT COD|AIR| boost slyly ironic instructions. furiou|
+4387|177|5|2|48|51704.16|0.06|0.05|N|O|1995-10-29|1995-12-11|1995-11-01|NONE|REG AIR|sleep slyly. blithely sl|
+4387|2|5|3|15|13530.00|0.00|0.03|N|O|1996-01-11|1996-01-14|1996-01-30|TAKE BACK RETURN|REG AIR|s hinder quietly across the pla|
+4387|47|8|4|9|8523.36|0.00|0.03|N|O|1996-01-04|1995-12-26|1996-01-12|DELIVER IN PERSON|REG AIR|c ideas. slyly regular packages sol|
+4387|82|3|5|3|2946.24|0.05|0.08|N|O|1995-11-17|1995-12-28|1995-11-25|COLLECT COD|SHIP| pinto beans |
+4387|6|3|6|40|36240.00|0.02|0.04|N|O|1995-11-29|1995-12-10|1995-12-20|NONE|REG AIR|deas according to the blithely regular fox|
+4388|65|10|1|30|28951.80|0.02|0.07|N|O|1996-06-07|1996-05-07|1996-06-22|DELIVER IN PERSON|FOB|s cajole fluffil|
+4388|84|5|2|28|27554.24|0.05|0.04|N|O|1996-05-08|1996-06-20|1996-05-12|TAKE BACK RETURN|RAIL|ove the ide|
+4388|52|4|3|13|12376.65|0.07|0.05|N|O|1996-06-28|1996-05-23|1996-07-04|DELIVER IN PERSON|REG AIR|ly even, expre|
+4389|157|2|1|20|21143.00|0.08|0.00|A|F|1994-06-06|1994-06-17|1994-06-17|DELIVER IN PERSON|SHIP|ng the carefully express d|
+4389|153|5|2|13|13690.95|0.00|0.00|A|F|1994-08-18|1994-06-06|1994-08-20|NONE|RAIL|nal, regula|
+4389|79|9|3|39|38183.73|0.04|0.07|A|F|1994-06-08|1994-06-04|1994-06-10|TAKE BACK RETURN|TRUCK| unusual, final excuses cajole carefully |
+4389|160|2|4|5|5300.80|0.09|0.00|A|F|1994-09-03|1994-06-23|1994-09-16|NONE|FOB| ironic request|
+4389|11|5|5|22|20042.22|0.08|0.00|R|F|1994-07-05|1994-06-12|1994-07-12|NONE|TRUCK|lly silent de|
+4389|2|3|6|22|19844.00|0.01|0.04|R|F|1994-06-07|1994-06-29|1994-06-19|COLLECT COD|TRUCK|at the final excuses hinder carefully a|
+4389|185|6|7|4|4340.72|0.09|0.08|R|F|1994-06-14|1994-06-30|1994-07-06|NONE|REG AIR| blithely even d|
+4390|152|10|1|35|36825.25|0.07|0.04|R|F|1995-05-30|1995-07-02|1995-06-15|DELIVER IN PERSON|TRUCK|ongside of the slyly regular ideas|
+4390|196|8|2|28|30693.32|0.03|0.00|N|O|1995-09-07|1995-06-22|1995-10-05|COLLECT COD|SHIP|ld braids haggle atop the for|
+4390|101|8|3|42|42046.20|0.05|0.08|A|F|1995-06-12|1995-07-16|1995-06-17|NONE|AIR|arefully even accoun|
+4390|98|2|4|32|31938.88|0.07|0.08|N|O|1995-09-15|1995-08-12|1995-10-05|TAKE BACK RETURN|TRUCK|ctions across|
+4391|161|10|1|1|1061.16|0.09|0.00|R|F|1992-06-18|1992-04-27|1992-06-20|COLLECT COD|TRUCK|ong the silent deposits|
+4391|187|8|2|45|48923.10|0.07|0.04|R|F|1992-04-01|1992-05-01|1992-04-13|TAKE BACK RETURN|TRUCK|ep quickly after |
+4416|94|7|1|37|36781.33|0.08|0.03|A|F|1992-10-23|1992-08-23|1992-11-16|COLLECT COD|RAIL|fluffily ironic |
+4416|89|10|2|3|2967.24|0.06|0.03|R|F|1992-10-22|1992-08-06|1992-11-13|DELIVER IN PERSON|SHIP| requests sleep along the |
+4416|9|6|3|45|40905.00|0.09|0.03|A|F|1992-10-16|1992-09-09|1992-10-28|COLLECT COD|AIR|the final pinto beans. special frets |
+4417|75|5|1|28|27301.96|0.08|0.02|N|O|1998-09-04|1998-10-04|1998-09-19|TAKE BACK RETURN|REG AIR|ies across the furious|
+4417|181|2|2|1|1081.18|0.06|0.08|N|O|1998-10-23|1998-08-22|1998-10-24|NONE|REG AIR|press deposits promise stealthily amo|
+4417|98|2|3|35|34933.15|0.06|0.04|N|O|1998-08-08|1998-09-23|1998-09-02|DELIVER IN PERSON|FOB|slyly regular, silent courts. even packag|
+4418|35|1|1|32|29920.96|0.02|0.06|A|F|1993-05-28|1993-06-02|1993-05-30|TAKE BACK RETURN|RAIL|ly. bold pinto b|
+4418|22|5|2|14|12908.28|0.03|0.04|A|F|1993-05-20|1993-06-18|1993-06-05|TAKE BACK RETURN|SHIP| blithely regular requests. blith|
+4418|79|7|3|3|2937.21|0.00|0.02|R|F|1993-04-08|1993-06-04|1993-05-02|NONE|SHIP|luffily across the unusual ideas. reque|
+4419|108|9|1|45|45364.50|0.01|0.05|N|O|1996-07-20|1996-09-07|1996-08-18|DELIVER IN PERSON|TRUCK|s doze sometimes fluffily regular a|
+4419|32|8|2|42|39145.26|0.00|0.03|N|O|1996-09-18|1996-07-25|1996-09-21|COLLECT COD|RAIL|sts. furious|
+4419|132|3|3|6|6192.78|0.02|0.08|N|O|1996-06-25|1996-09-04|1996-07-20|DELIVER IN PERSON|AIR|ts wake slyly final dugou|
+4420|8|5|1|7|6356.00|0.07|0.03|R|F|1994-08-30|1994-09-03|1994-09-25|NONE|FOB| regular instructions sleep around|
+4421|98|2|1|37|36929.33|0.09|0.08|N|O|1997-07-22|1997-06-27|1997-07-25|DELIVER IN PERSON|SHIP|l accounts. ironic request|
+4421|56|1|2|46|43978.30|0.04|0.04|N|O|1997-04-21|1997-05-13|1997-05-15|DELIVER IN PERSON|FOB|reful packages. bold, |
+4421|167|6|3|46|49089.36|0.00|0.06|N|O|1997-05-25|1997-05-21|1997-06-23|COLLECT COD|TRUCK|g dependenci|
+4421|191|4|4|32|34918.08|0.06|0.04|N|O|1997-07-09|1997-06-03|1997-07-25|NONE|SHIP|ar ideas eat among the furiousl|
+4421|190|1|5|32|34886.08|0.06|0.04|N|O|1997-07-28|1997-06-14|1997-08-13|NONE|REG AIR|uickly final pinto beans impress. bold |
+4421|47|6|6|44|41669.76|0.09|0.06|N|O|1997-06-17|1997-06-20|1997-06-29|NONE|TRUCK|le carefully. bl|
+4421|116|3|7|18|18289.98|0.01|0.00|N|O|1997-06-07|1997-05-13|1997-06-10|DELIVER IN PERSON|FOB|. regular, s|
+4422|135|1|1|5|5175.65|0.09|0.07|N|O|1995-07-17|1995-08-13|1995-07-25|NONE|SHIP|e furiously about t|
+4422|48|5|2|41|38869.64|0.08|0.05|N|F|1995-06-12|1995-07-09|1995-06-20|COLLECT COD|TRUCK| theodolites shal|
+4422|103|10|3|39|39120.90|0.00|0.05|N|O|1995-09-02|1995-06-24|1995-09-14|NONE|TRUCK|en hockey players engage|
+4422|153|4|4|4|4212.60|0.02|0.05|N|O|1995-09-18|1995-08-12|1995-10-18|COLLECT COD|FOB|cies along the bo|
+4422|80|9|5|20|19601.60|0.07|0.05|N|O|1995-08-17|1995-07-16|1995-09-13|DELIVER IN PERSON|RAIL|ructions wake slyly al|
+4423|150|9|1|3|3150.45|0.03|0.00|A|F|1995-03-22|1995-04-06|1995-04-19|NONE|TRUCK| final theodolites nag after the bli|
+4423|60|5|2|2|1920.12|0.07|0.04|A|F|1995-03-04|1995-04-04|1995-03-08|TAKE BACK RETURN|REG AIR|old sheaves sleep|
+4448|52|7|1|24|22849.20|0.10|0.07|N|O|1998-09-09|1998-07-06|1998-09-27|DELIVER IN PERSON|SHIP|nal packages along the ironic instructi|
+4448|189|10|2|13|14159.34|0.00|0.01|N|O|1998-07-26|1998-07-03|1998-08-14|COLLECT COD|MAIL|fluffily express accounts integrate furiou|
+4448|41|4|3|35|32936.40|0.10|0.06|N|O|1998-09-18|1998-07-27|1998-10-08|NONE|REG AIR|aggle carefully alongside of the q|
+4448|141|2|4|3|3123.42|0.01|0.01|N|O|1998-07-20|1998-07-10|1998-08-07|DELIVER IN PERSON|TRUCK|ronic theod|
+4448|91|2|5|41|40634.69|0.00|0.08|N|O|1998-07-30|1998-08-09|1998-08-03|NONE|AIR|pon the permanently even excuses nag |
+4448|172|3|6|12|12866.04|0.06|0.03|N|O|1998-08-21|1998-06-30|1998-09-09|COLLECT COD|RAIL|sits about the ironic, bu|
+4449|32|3|1|42|39145.26|0.10|0.07|N|O|1998-03-22|1998-05-09|1998-04-03|NONE|FOB| packages. blithely final |
+4449|141|8|2|10|10411.40|0.02|0.03|N|O|1998-05-09|1998-05-04|1998-05-15|NONE|SHIP|ccounts alongside of the platelets integr|
+4450|174|5|1|44|47263.48|0.10|0.00|N|O|1997-10-12|1997-10-13|1997-10-29|DELIVER IN PERSON|RAIL| the slyly eve|
+4450|15|6|2|9|8235.09|0.03|0.03|N|O|1997-08-13|1997-08-16|1997-08-15|NONE|FOB|gular requests cajole carefully. regular c|
+4450|96|8|3|45|44824.05|0.08|0.01|N|O|1997-09-01|1997-10-06|1997-09-19|NONE|TRUCK|express ideas are furiously regular|
+4450|62|9|4|13|12506.78|0.00|0.00|N|O|1997-08-26|1997-09-18|1997-09-20|COLLECT COD|MAIL| brave foxes. slyly unusual|
+4450|56|7|5|6|5736.30|0.09|0.01|N|O|1997-09-02|1997-09-30|1997-09-09|NONE|FOB|eposits. foxes cajole unusual fox|
+4451|164|5|1|40|42566.40|0.03|0.03|A|F|1994-11-18|1994-12-25|1994-11-26|DELIVER IN PERSON|RAIL|y. slyly special deposits are sly|
+4451|63|4|2|34|32744.04|0.10|0.02|A|F|1994-11-30|1994-12-04|1994-12-13|COLLECT COD|SHIP| regular ideas.|
+4451|159|10|3|19|20123.85|0.05|0.06|R|F|1994-10-09|1994-11-26|1994-10-23|COLLECT COD|FOB|ly after the fluffi|
+4452|114|8|1|21|21296.31|0.07|0.03|R|F|1994-10-06|1994-08-23|1994-10-15|COLLECT COD|TRUCK|multipliers x-ray carefully in place of |
+4452|1|8|2|47|42347.00|0.01|0.06|A|F|1994-10-08|1994-08-09|1994-10-09|TAKE BACK RETURN|TRUCK|ts. slyly regular cour|
+4453|147|10|1|41|42932.74|0.00|0.08|N|O|1997-07-17|1997-05-15|1997-07-31|NONE|REG AIR|anent theodolites are slyly except t|
+4453|133|4|2|16|16530.08|0.03|0.00|N|O|1997-07-22|1997-05-05|1997-08-03|COLLECT COD|FOB|ar excuses nag quickly even accounts. b|
+4453|62|7|3|48|46178.88|0.02|0.07|N|O|1997-05-29|1997-06-24|1997-06-03|NONE|SHIP|eep. fluffily express accounts at the furi|
+4453|102|5|4|26|26054.60|0.06|0.07|N|O|1997-05-07|1997-06-07|1997-05-22|NONE|TRUCK|express packages are|
+4454|151|9|1|20|21023.00|0.10|0.08|R|F|1994-05-06|1994-03-17|1994-05-20|COLLECT COD|SHIP|lar theodolites. even instructio|
+4454|152|10|2|22|23147.30|0.06|0.02|A|F|1994-02-06|1994-04-11|1994-03-06|DELIVER IN PERSON|RAIL|ully. carefully final accounts accordi|
+4454|192|3|3|45|49148.55|0.07|0.04|A|F|1994-03-29|1994-03-26|1994-04-04|TAKE BACK RETURN|RAIL|ests promise. packages print fur|
+4454|2|3|4|1|902.00|0.09|0.05|A|F|1994-02-05|1994-04-19|1994-02-12|COLLECT COD|RAIL|equests run.|
+4454|52|4|5|48|45698.40|0.00|0.07|R|F|1994-04-23|1994-04-03|1994-04-26|COLLECT COD|FOB|to beans wake across th|
+4454|160|8|6|20|21203.20|0.10|0.03|A|F|1994-04-08|1994-03-06|1994-04-26|DELIVER IN PERSON|TRUCK|quickly regular requests. furiously|
+4455|70|5|1|20|19401.40|0.01|0.05|A|F|1994-01-31|1993-11-21|1994-03-02|DELIVER IN PERSON|MAIL| express packages. packages boost quickly|
+4455|153|4|2|47|49498.05|0.09|0.01|R|F|1994-01-01|1993-12-25|1994-01-05|COLLECT COD|FOB| requests. even, even accou|
+4455|123|2|3|34|34786.08|0.00|0.06|A|F|1993-10-24|1993-11-27|1993-11-04|TAKE BACK RETURN|AIR| slyly ironic requests. quickly even d|
+4480|108|5|1|30|30243.00|0.08|0.03|R|F|1994-07-29|1994-06-22|1994-08-01|NONE|FOB|ven braids us|
+4481|24|9|1|50|46201.00|0.02|0.06|N|O|1996-07-22|1996-05-13|1996-08-14|DELIVER IN PERSON|RAIL|ar packages. regula|
+4481|190|1|2|27|29435.13|0.02|0.03|N|O|1996-04-06|1996-05-17|1996-04-12|TAKE BACK RETURN|AIR|ackages haggle even, |
+4482|71|2|1|32|31074.24|0.06|0.03|A|F|1995-05-16|1995-07-22|1995-06-07|NONE|RAIL| quickly pendin|
+4482|96|9|2|32|31874.88|0.01|0.06|N|O|1995-08-16|1995-06-26|1995-09-10|DELIVER IN PERSON|AIR|eans wake according |
+4483|6|7|1|32|28992.00|0.07|0.07|R|F|1992-04-05|1992-05-25|1992-04-08|DELIVER IN PERSON|MAIL|ests haggle. slyl|
+4483|62|1|2|50|48103.00|0.01|0.06|A|F|1992-06-19|1992-05-12|1992-07-08|DELIVER IN PERSON|TRUCK|ag blithely even|
+4483|9|4|3|50|45450.00|0.00|0.04|R|F|1992-06-10|1992-04-18|1992-06-17|DELIVER IN PERSON|MAIL|ackages. furiously ironi|
+4484|95|9|1|4|3980.36|0.06|0.03|N|O|1997-04-09|1997-02-11|1997-04-12|TAKE BACK RETURN|TRUCK|packages de|
+4484|137|8|2|39|40448.07|0.05|0.02|N|O|1997-04-01|1997-01-26|1997-04-21|NONE|RAIL|onic accounts wake blithel|
+4484|190|1|3|38|41427.22|0.06|0.07|N|O|1997-03-07|1997-01-31|1997-04-01|COLLECT COD|REG AIR|. even requests un|
+4484|122|5|4|41|41906.92|0.06|0.03|N|O|1997-01-25|1997-02-15|1997-01-29|TAKE BACK RETURN|REG AIR|ress accounts. ironic deposits unwind fur|
+4484|3|4|5|42|37926.00|0.03|0.07|N|O|1997-03-25|1997-02-21|1997-04-05|DELIVER IN PERSON|REG AIR|ding, pending requests wake. fluffily |
+4484|36|7|6|29|27144.87|0.09|0.06|N|O|1996-12-27|1997-03-10|1997-01-13|NONE|FOB| wake blithely ironic|
+4484|103|8|7|50|50155.00|0.07|0.01|N|O|1997-03-17|1997-03-16|1997-03-21|COLLECT COD|FOB|the ironic, final theodo|
+4485|191|5|1|1|1091.19|0.03|0.05|R|F|1994-12-04|1995-02-07|1994-12-09|NONE|AIR|play according to the ironic, ironic|
+4485|141|10|2|46|47892.44|0.04|0.06|R|F|1995-03-09|1994-12-14|1995-03-23|DELIVER IN PERSON|AIR|. ironic foxes haggle. regular war|
+4485|175|6|3|43|46232.31|0.01|0.05|R|F|1995-01-17|1995-02-11|1995-02-07|DELIVER IN PERSON|TRUCK|al accounts according to the slyly r|
+4485|144|5|4|43|44898.02|0.08|0.06|R|F|1995-01-28|1995-01-26|1995-02-07|DELIVER IN PERSON|AIR|. blithely|
+4485|6|7|5|47|42582.00|0.08|0.04|R|F|1995-03-11|1995-01-11|1995-03-21|TAKE BACK RETURN|RAIL|luffily pending acc|
+4486|135|1|1|46|47615.98|0.08|0.00|N|O|1998-05-02|1998-04-05|1998-05-08|COLLECT COD|MAIL|ackages. specia|
+4486|49|2|2|19|18031.76|0.10|0.01|N|O|1998-06-07|1998-05-28|1998-07-02|NONE|MAIL|pending foxes after|
+4486|96|7|3|47|46816.23|0.02|0.07|N|O|1998-04-09|1998-05-24|1998-05-07|DELIVER IN PERSON|MAIL|ts around the quiet packages ar|
+4486|91|4|4|28|27750.52|0.07|0.02|N|O|1998-04-21|1998-04-19|1998-04-26|TAKE BACK RETURN|AIR|to the furious, regular foxes play abov|
+4487|138|4|1|37|38410.81|0.03|0.07|R|F|1993-02-28|1993-04-18|1993-03-17|TAKE BACK RETURN|MAIL|bove the fu|
+4487|113|10|2|49|49642.39|0.10|0.00|R|F|1993-06-13|1993-05-08|1993-07-10|COLLECT COD|FOB|sual packages should ha|
+4487|190|1|3|1|1090.19|0.02|0.07|A|F|1993-05-11|1993-05-23|1993-05-17|TAKE BACK RETURN|FOB|ithely final asym|
+4487|93|4|4|25|24827.25|0.07|0.03|A|F|1993-03-09|1993-04-27|1993-03-30|COLLECT COD|RAIL|g the final instructions. slyly c|
+4512|162|1|1|30|31864.80|0.07|0.07|N|O|1996-01-28|1995-12-22|1996-02-22|TAKE BACK RETURN|TRUCK|ly unusual package|
+4512|41|4|2|24|22584.96|0.04|0.06|N|O|1995-12-16|1996-01-16|1995-12-25|NONE|SHIP|ly regular pinto beans. carefully bold depo|
+4512|145|8|3|21|21947.94|0.00|0.00|N|O|1995-10-31|1995-12-30|1995-11-15|NONE|REG AIR|lly unusual pinto b|
+4512|141|2|4|32|33316.48|0.10|0.01|N|O|1995-11-25|1995-12-28|1995-12-06|NONE|FOB|counts are against the quickly regular |
+4512|133|4|5|43|44424.59|0.06|0.00|N|O|1995-12-20|1995-11-28|1996-01-14|NONE|AIR|are carefully. theodolites wake|
+4513|170|1|1|29|31034.93|0.03|0.01|N|O|1996-05-18|1996-05-23|1996-06-08|NONE|REG AIR|cajole. regular packages boost. s|
+4513|70|9|2|39|37832.73|0.01|0.04|N|O|1996-06-25|1996-05-14|1996-07-24|NONE|MAIL|slyly furiously unusual deposits. blit|
+4513|138|4|3|34|35296.42|0.00|0.03|N|O|1996-03-27|1996-06-12|1996-04-06|DELIVER IN PERSON|SHIP|sits. quickly even instructions |
+4513|192|6|4|13|14198.47|0.08|0.08|N|O|1996-04-12|1996-05-19|1996-04-25|DELIVER IN PERSON|AIR|l, final excuses detect furi|
+4514|164|9|1|27|28732.32|0.06|0.06|R|F|1994-07-01|1994-07-13|1994-07-26|COLLECT COD|AIR| even, silent foxes be|
+4514|46|3|2|15|14190.60|0.10|0.04|R|F|1994-08-24|1994-07-11|1994-09-14|DELIVER IN PERSON|RAIL|! unusual, special deposits afte|
+4514|78|8|3|10|9780.70|0.09|0.05|A|F|1994-06-19|1994-06-25|1994-07-01|COLLECT COD|SHIP|ake furiously. carefully regular requests|
+4514|81|2|4|9|8829.72|0.10|0.03|A|F|1994-08-04|1994-07-01|1994-09-01|DELIVER IN PERSON|REG AIR|wly. quick|
+4514|149|8|5|12|12589.68|0.02|0.03|R|F|1994-08-20|1994-06-09|1994-09-15|TAKE BACK RETURN|FOB| carefully ironic foxes nag caref|
+4514|189|10|6|38|41388.84|0.03|0.05|A|F|1994-07-28|1994-07-06|1994-08-25|NONE|AIR|ending excuses. sl|
+4514|177|8|7|27|29083.59|0.04|0.06|A|F|1994-06-24|1994-07-14|1994-06-30|TAKE BACK RETURN|TRUCK|. slyly sile|
+4515|39|10|1|15|14085.45|0.06|0.01|R|F|1992-05-26|1992-05-25|1992-06-03|NONE|SHIP|posits wake|
+4515|103|10|2|50|50155.00|0.06|0.03|A|F|1992-03-28|1992-05-16|1992-04-20|NONE|AIR|ding instructions again|
+4515|154|6|3|27|28462.05|0.09|0.01|A|F|1992-06-06|1992-06-08|1992-06-07|DELIVER IN PERSON|REG AIR| against the even re|
+4515|54|5|4|32|30529.60|0.06|0.03|R|F|1992-04-07|1992-05-11|1992-04-09|COLLECT COD|MAIL|carefully express depo|
+4515|45|8|5|22|20790.88|0.09|0.07|A|F|1992-07-16|1992-05-07|1992-07-23|NONE|SHIP|le quickly above the even, bold ideas.|
+4515|180|8|6|23|24844.14|0.04|0.00|R|F|1992-05-23|1992-06-15|1992-06-20|TAKE BACK RETURN|FOB|ns. bold r|
+4516|170|9|1|34|36385.78|0.05|0.04|A|F|1994-05-16|1994-06-23|1994-06-12|NONE|SHIP|even pinto beans wake qui|
+4517|43|4|1|50|47152.00|0.01|0.02|N|O|1998-06-08|1998-04-18|1998-06-20|DELIVER IN PERSON|MAIL|refully pending acco|
+4518|144|7|1|9|9397.26|0.09|0.04|N|O|1997-06-26|1997-07-07|1997-07-10|NONE|RAIL| pending deposits. slyly re|
+4518|45|6|2|19|17955.76|0.10|0.05|N|O|1997-08-09|1997-06-06|1997-08-27|COLLECT COD|RAIL|ter the slyly bo|
+4519|55|3|1|30|28651.50|0.09|0.07|R|F|1993-04-11|1993-06-05|1993-04-22|DELIVER IN PERSON|REG AIR|totes. slyly bold somas after the |
+4519|191|3|2|37|40374.03|0.06|0.08|R|F|1993-07-22|1993-06-16|1993-08-19|COLLECT COD|AIR|ly slyly furious depth|
+4544|131|7|1|40|41245.20|0.07|0.01|N|O|1997-08-15|1997-10-16|1997-08-20|DELIVER IN PERSON|RAIL| detect slyly. evenly pending instru|
+4544|172|2|2|19|20371.23|0.08|0.01|N|O|1997-08-14|1997-09-08|1997-08-25|NONE|SHIP|regular ideas are furiously about|
+4544|71|9|3|20|19421.40|0.02|0.07|N|O|1997-10-12|1997-10-11|1997-10-13|COLLECT COD|REG AIR| waters about the|
+4544|51|6|4|39|37090.95|0.07|0.05|N|O|1997-08-20|1997-09-07|1997-08-27|COLLECT COD|REG AIR|ular packages. s|
+4544|133|4|5|31|32027.03|0.09|0.03|N|O|1997-08-09|1997-09-29|1997-08-17|COLLECT COD|TRUCK|dolites detect quickly reg|
+4544|27|8|6|8|7416.16|0.10|0.03|N|O|1997-10-13|1997-10-06|1997-10-25|COLLECT COD|AIR|olites. fi|
+4545|173|1|1|38|40780.46|0.06|0.06|R|F|1993-01-27|1993-03-01|1993-02-04|NONE|TRUCK|nts serve according to th|
+4545|63|4|2|27|26002.62|0.01|0.06|R|F|1993-02-07|1993-02-18|1993-02-18|NONE|FOB|ously bold asymptotes! blithely pen|
+4545|87|8|3|9|8883.72|0.10|0.06|R|F|1993-03-20|1993-02-23|1993-04-11|TAKE BACK RETURN|AIR|xpress accounts|
+4545|64|9|4|2|1928.12|0.10|0.00|R|F|1993-04-16|1993-04-17|1993-05-03|NONE|REG AIR|ages use. slyly even i|
+4545|117|1|5|27|27461.97|0.08|0.05|A|F|1993-03-18|1993-02-22|1993-03-23|NONE|RAIL|ccounts haggle carefully. deposits |
+4545|109|2|6|8|8072.80|0.03|0.02|A|F|1993-05-01|1993-03-12|1993-05-15|NONE|FOB| boost slyly. slyly|
+4545|9|2|7|36|32724.00|0.10|0.04|R|F|1993-01-28|1993-03-30|1993-02-04|DELIVER IN PERSON|SHIP|sublate slyly. furiously ironic accounts b|
+4546|133|4|1|10|10331.30|0.09|0.02|N|O|1995-09-23|1995-10-10|1995-10-23|COLLECT COD|TRUCK|osits alongside of the|
+4546|171|10|2|15|16067.55|0.04|0.07|N|O|1995-07-31|1995-10-17|1995-08-06|NONE|REG AIR|ught to cajole furiously. qu|
+4546|77|8|3|4|3908.28|0.06|0.08|N|O|1995-08-14|1995-10-07|1995-08-16|COLLECT COD|MAIL|kly pending dependencies along the furio|
+4546|149|6|4|10|10491.40|0.08|0.02|N|O|1995-09-02|1995-09-16|1995-09-10|DELIVER IN PERSON|FOB|above the enticingly ironic dependencies|
+4547|188|9|1|15|16322.70|0.10|0.04|A|F|1993-12-08|1993-11-15|1993-12-22|NONE|REG AIR|ets haggle. regular dinos affix fu|
+4547|116|10|2|7|7112.77|0.10|0.02|A|F|1993-09-04|1993-09-29|1993-09-20|COLLECT COD|RAIL|slyly express a|
+4547|45|2|3|15|14175.60|0.00|0.00|R|F|1993-11-18|1993-10-06|1993-12-13|NONE|TRUCK|e carefully across the unus|
+4547|148|7|4|15|15722.10|0.05|0.08|R|F|1993-11-29|1993-10-12|1993-12-29|COLLECT COD|REG AIR|ironic gifts integrate |
+4548|14|8|1|21|19194.21|0.10|0.05|N|O|1996-07-11|1996-09-04|1996-07-30|COLLECT COD|REG AIR|pecial theodoli|
+4548|47|10|2|17|16099.68|0.00|0.08|N|O|1996-07-23|1996-09-21|1996-07-26|DELIVER IN PERSON|REG AIR|y ironic requests above the fluffily d|
+4548|123|2|3|47|48086.64|0.05|0.04|N|O|1996-07-24|1996-09-12|1996-08-08|NONE|MAIL|ts. excuses use slyly spec|
+4548|177|6|4|22|23697.74|0.07|0.01|N|O|1996-07-06|1996-08-23|1996-07-15|DELIVER IN PERSON|RAIL|s. furiously ironic theodolites c|
+4548|45|4|5|36|34021.44|0.04|0.06|N|O|1996-08-19|1996-09-12|1996-09-08|COLLECT COD|FOB|tions integrat|
+4549|159|1|1|44|46602.60|0.08|0.00|N|O|1998-03-13|1998-04-15|1998-03-27|TAKE BACK RETURN|TRUCK|ding to the regular, silent requests|
+4549|89|10|2|1|989.08|0.05|0.08|N|O|1998-05-04|1998-04-11|1998-05-14|TAKE BACK RETURN|AIR| requests wake. furiously even |
+4550|150|7|1|9|9451.35|0.05|0.06|R|F|1995-04-19|1995-02-07|1995-04-24|COLLECT COD|SHIP|l dependencies boost slyly after th|
+4550|66|5|2|19|18355.14|0.06|0.04|A|F|1995-01-01|1995-02-13|1995-01-20|NONE|AIR|quests. express |
+4551|11|1|1|6|5466.06|0.08|0.08|N|O|1996-05-18|1996-04-23|1996-06-13|DELIVER IN PERSON|TRUCK|fily silent fo|
+4551|179|8|2|26|28058.42|0.02|0.04|N|O|1996-04-14|1996-04-26|1996-04-17|TAKE BACK RETURN|RAIL|le. carefully dogged accounts use furiousl|
+4551|22|1|3|22|20284.44|0.08|0.01|N|O|1996-05-12|1996-03-17|1996-05-29|TAKE BACK RETURN|REG AIR|ly ironic reques|
+4551|198|10|4|27|29651.13|0.00|0.01|N|O|1996-04-28|1996-03-22|1996-05-22|TAKE BACK RETURN|RAIL|y along the slyly even |
+4576|90|1|1|5|4950.45|0.09|0.03|N|O|1996-08-23|1996-11-08|1996-09-20|TAKE BACK RETURN|AIR|ly express, special asymptote|
+4576|58|9|2|43|41196.15|0.08|0.06|N|O|1996-10-24|1996-09-23|1996-11-10|NONE|SHIP|ly final deposits. never|
+4576|42|1|3|14|13188.56|0.09|0.01|N|O|1996-09-12|1996-09-30|1996-09-24|COLLECT COD|MAIL|detect slyly.|
+4577|185|6|1|43|46662.74|0.01|0.03|N|O|1998-06-16|1998-07-09|1998-06-17|TAKE BACK RETURN|AIR|packages. |
+4577|177|6|2|43|46318.31|0.05|0.03|N|O|1998-08-24|1998-06-02|1998-09-14|TAKE BACK RETURN|RAIL|ly accounts. carefully |
+4577|69|6|3|12|11628.72|0.07|0.05|N|O|1998-07-29|1998-06-17|1998-08-04|DELIVER IN PERSON|TRUCK|equests alongsi|
+4578|74|2|1|10|9740.70|0.09|0.06|R|F|1993-01-01|1992-11-19|1993-01-28|TAKE BACK RETURN|REG AIR|uests. blithely unus|
+4578|169|10|2|42|44904.72|0.06|0.00|R|F|1993-01-05|1992-11-06|1993-01-13|DELIVER IN PERSON|FOB|s are caref|
+4578|179|8|3|15|16187.55|0.01|0.01|R|F|1992-10-23|1992-11-22|1992-11-09|DELIVER IN PERSON|REG AIR|gular theodo|
+4578|139|10|4|7|7273.91|0.09|0.08|A|F|1992-12-07|1992-11-27|1993-01-05|TAKE BACK RETURN|SHIP|odolites. carefully unusual ideas accor|
+4578|163|2|5|20|21263.20|0.04|0.02|A|F|1993-01-11|1992-11-09|1993-01-23|TAKE BACK RETURN|RAIL|iously pending theodolites--|
+4579|175|4|1|14|15052.38|0.02|0.02|N|O|1996-02-01|1996-01-08|1996-02-08|TAKE BACK RETURN|MAIL|nding theodolites. fluffil|
+4579|42|3|2|28|26377.12|0.02|0.05|N|O|1996-01-22|1996-02-13|1996-02-03|DELIVER IN PERSON|RAIL|slyly across the |
+4579|178|9|3|34|36657.78|0.05|0.02|N|O|1996-02-26|1996-02-22|1996-03-16|COLLECT COD|MAIL|hely. carefully blithe dependen|
+4579|120|1|4|8|8160.96|0.05|0.06|N|O|1995-12-16|1996-01-15|1995-12-18|TAKE BACK RETURN|AIR|posits. carefully perman|
+4580|92|5|1|22|21825.98|0.01|0.05|A|F|1994-01-16|1994-01-26|1994-02-05|COLLECT COD|AIR|nticingly final packag|
+4580|32|3|2|10|9320.30|0.05|0.04|R|F|1993-12-20|1993-12-30|1994-01-17|COLLECT COD|RAIL|gular, pending deposits. fina|
+4580|1|8|3|41|36941.00|0.00|0.07|R|F|1993-12-13|1994-01-31|1994-01-06|NONE|SHIP|requests. quickly silent asymptotes sle|
+4580|178|8|4|5|5390.85|0.07|0.00|A|F|1994-01-28|1993-12-17|1994-02-22|NONE|TRUCK|o beans. f|
+4580|189|10|5|39|42478.02|0.03|0.02|R|F|1993-12-28|1993-12-26|1994-01-23|NONE|RAIL|. fluffily final dolphins use furiously al|
+4581|165|4|1|37|39410.92|0.01|0.04|A|F|1992-10-17|1992-11-05|1992-11-04|DELIVER IN PERSON|MAIL|e the blithely bold pearls ha|
+4581|50|3|2|7|6650.35|0.01|0.02|A|F|1992-10-09|1992-10-20|1992-10-21|TAKE BACK RETURN|MAIL|express accounts d|
+4581|21|10|3|46|42366.92|0.04|0.04|A|F|1992-09-09|1992-11-27|1992-09-26|NONE|REG AIR|nag toward the carefully final accounts. |
+4582|192|5|1|17|18567.23|0.09|0.08|N|O|1996-08-17|1996-08-26|1996-08-20|COLLECT COD|REG AIR|ng packages. depo|
+4583|141|2|1|17|17699.38|0.01|0.05|A|F|1994-11-08|1994-11-03|1994-11-29|COLLECT COD|MAIL|romise. reques|
+4583|187|8|2|43|46748.74|0.04|0.04|A|F|1994-10-30|1994-12-17|1994-11-16|COLLECT COD|RAIL|fully after the speci|
+4583|196|10|3|28|30693.32|0.00|0.07|A|F|1994-10-29|1994-11-21|1994-11-28|NONE|SHIP|to beans haggle sly|
+4583|173|4|4|27|28975.59|0.08|0.03|R|F|1995-01-11|1994-12-24|1995-02-10|DELIVER IN PERSON|TRUCK| detect silent requests. furiously speci|
+4583|184|5|5|36|39030.48|0.09|0.06|A|F|1995-01-06|1994-11-25|1995-01-29|DELIVER IN PERSON|RAIL|ar requests haggle after the furiously |
+4583|122|7|6|14|14309.68|0.09|0.01|R|F|1994-11-17|1994-11-08|1994-11-21|DELIVER IN PERSON|AIR|detect. doggedly regular pi|
+4583|87|8|7|32|31586.56|0.04|0.00|A|F|1995-01-13|1994-10-29|1995-02-08|TAKE BACK RETURN|RAIL|across the pinto beans-- quickly|
+4608|173|1|1|30|32195.10|0.08|0.05|R|F|1994-10-08|1994-07-18|1994-10-25|DELIVER IN PERSON|SHIP|s cajole. slyly |
+4608|47|8|2|50|47352.00|0.06|0.01|A|F|1994-07-25|1994-09-01|1994-08-10|NONE|FOB| theodolites|
+4608|79|9|3|50|48953.50|0.03|0.01|A|F|1994-08-04|1994-09-10|1994-08-13|COLLECT COD|TRUCK| wake closely. even decoys haggle above|
+4608|31|2|4|36|33517.08|0.05|0.06|R|F|1994-10-04|1994-08-02|1994-10-21|COLLECT COD|FOB|ages wake quickly slyly iron|
+4609|47|6|1|28|26517.12|0.10|0.05|N|O|1997-02-02|1997-02-17|1997-03-02|DELIVER IN PERSON|REG AIR|ously. quickly final requests cajole fl|
+4609|185|6|2|3|3255.54|0.09|0.03|N|O|1996-12-28|1997-02-06|1997-01-20|NONE|FOB|nstructions. furious instructions |
+4609|23|4|3|46|42458.92|0.05|0.05|N|O|1997-02-11|1997-01-16|1997-03-07|NONE|FOB|r foxes. fluffily ironic ideas ha|
+4610|87|8|1|21|20728.68|0.07|0.07|R|F|1993-08-10|1993-08-05|1993-08-27|NONE|REG AIR|ly special theodolites. even,|
+4610|175|5|2|14|15052.38|0.00|0.07|R|F|1993-07-28|1993-07-25|1993-07-31|TAKE BACK RETURN|SHIP| ironic frays. dependencies detect blithel|
+4610|159|1|3|44|46602.60|0.05|0.03|A|F|1993-08-05|1993-07-20|1993-08-19|COLLECT COD|TRUCK| final theodolites |
+4610|75|3|4|26|25351.82|0.06|0.03|R|F|1993-07-01|1993-07-19|1993-07-19|NONE|MAIL| to the fluffily ironic requests h|
+4610|147|8|5|29|30367.06|0.08|0.04|R|F|1993-08-09|1993-07-27|1993-08-16|DELIVER IN PERSON|AIR| foxes. special, express package|
+4611|52|7|1|47|44746.35|0.09|0.06|A|F|1993-03-05|1993-03-01|1993-03-17|COLLECT COD|TRUCK|iously. furiously regular|
+4611|35|6|2|31|28985.93|0.04|0.02|A|F|1993-01-28|1993-02-14|1993-01-29|TAKE BACK RETURN|AIR| final pinto beans. permanent, sp|
+4611|82|3|3|50|49104.00|0.08|0.01|R|F|1993-01-22|1993-03-30|1993-02-16|TAKE BACK RETURN|AIR|l platelets. |
+4611|71|9|4|48|46611.36|0.02|0.08|R|F|1993-02-28|1993-02-12|1993-03-01|COLLECT COD|AIR|ular accounts |
+4612|6|9|1|20|18120.00|0.02|0.03|R|F|1993-09-24|1993-12-18|1993-10-22|NONE|AIR|beans sleep blithely iro|
+4612|50|7|2|17|16150.85|0.10|0.06|A|F|1994-01-09|1993-11-08|1994-02-06|TAKE BACK RETURN|REG AIR|equests haggle carefully silent excus|
+4612|137|8|3|40|41485.20|0.08|0.01|R|F|1993-10-08|1993-11-23|1993-10-24|DELIVER IN PERSON|RAIL|special platelets.|
+4612|185|6|4|10|10851.80|0.10|0.06|A|F|1993-11-11|1993-11-19|1993-11-13|TAKE BACK RETURN|SHIP|unusual theodol|
+4613|38|9|1|17|15946.51|0.09|0.07|N|O|1998-06-07|1998-05-11|1998-06-29|DELIVER IN PERSON|SHIP|liers cajole a|
+4613|108|1|2|25|25202.50|0.05|0.04|N|O|1998-05-22|1998-04-11|1998-05-27|TAKE BACK RETURN|SHIP|y pending platelets x-ray ironically! pend|
+4613|174|3|3|15|16112.55|0.10|0.02|N|O|1998-05-31|1998-04-16|1998-06-25|DELIVER IN PERSON|MAIL|against the quickly r|
+4613|8|1|4|36|32688.00|0.04|0.01|N|O|1998-04-22|1998-05-05|1998-05-04|DELIVER IN PERSON|AIR|gainst the furiously ironic|
+4613|111|8|5|35|35388.85|0.04|0.06|N|O|1998-06-04|1998-04-17|1998-06-20|COLLECT COD|MAIL|e blithely against the even, bold pi|
+4613|196|8|6|47|51520.93|0.04|0.04|N|O|1998-07-03|1998-05-26|1998-07-09|NONE|FOB|uriously special requests wak|
+4613|119|3|7|39|39745.29|0.09|0.05|N|O|1998-06-12|1998-06-01|1998-07-06|DELIVER IN PERSON|REG AIR|ously express|
+4614|7|2|1|19|17233.00|0.09|0.08|N|O|1996-05-17|1996-06-21|1996-06-08|TAKE BACK RETURN|AIR|ix. carefully regular |
+4614|65|6|2|3|2895.18|0.08|0.01|N|O|1996-07-22|1996-07-21|1996-08-07|NONE|MAIL|ions engage final, ironic |
+4614|8|1|3|36|32688.00|0.10|0.04|N|O|1996-07-05|1996-06-26|1996-07-07|NONE|REG AIR|onic foxes affix furi|
+4614|126|9|4|6|6156.72|0.09|0.01|N|O|1996-06-11|1996-05-30|1996-07-03|COLLECT COD|REG AIR|ake quickly quickly regular epitap|
+4614|73|3|5|24|23353.68|0.07|0.06|N|O|1996-07-01|1996-06-24|1996-07-08|COLLECT COD|REG AIR|regular, even|
+4614|34|5|6|32|29888.96|0.10|0.05|N|O|1996-08-21|1996-05-28|1996-09-16|NONE|REG AIR|ickly furio|
+4614|128|1|7|41|42152.92|0.01|0.07|N|O|1996-07-31|1996-07-12|1996-08-16|COLLECT COD|REG AIR|ackages haggle carefully about the even, b|
+4615|92|4|1|10|9920.90|0.02|0.08|A|F|1993-11-20|1993-10-05|1993-12-08|DELIVER IN PERSON|AIR|sits. slyly express deposits are|
+4640|88|9|1|5|4940.40|0.03|0.08|N|O|1996-02-05|1996-02-14|1996-02-15|TAKE BACK RETURN|RAIL| warthogs against the regular|
+4640|88|9|2|9|8892.72|0.03|0.05|N|O|1996-02-12|1996-02-14|1996-02-29|DELIVER IN PERSON|AIR| accounts. unu|
+4640|27|10|3|18|16686.36|0.02|0.07|N|O|1996-02-28|1996-03-06|1996-03-28|DELIVER IN PERSON|RAIL|boost furiously accord|
+4640|23|2|4|36|33228.72|0.06|0.08|N|O|1996-01-03|1996-03-09|1996-01-11|DELIVER IN PERSON|RAIL|iously furious accounts boost. carefully|
+4640|156|1|5|15|15842.25|0.03|0.02|N|O|1996-03-19|1996-02-09|1996-04-11|TAKE BACK RETURN|FOB|y regular instructions doze furiously. reg|
+4641|190|1|1|45|49058.55|0.07|0.03|R|F|1993-05-11|1993-04-19|1993-05-21|DELIVER IN PERSON|MAIL| about the close |
+4641|95|7|2|39|38808.51|0.06|0.00|R|F|1993-02-10|1993-03-06|1993-02-15|TAKE BACK RETURN|REG AIR| the bold reque|
+4641|36|7|3|15|14040.45|0.01|0.08|R|F|1993-01-25|1993-04-09|1993-02-05|TAKE BACK RETURN|AIR|s. carefully even exc|
+4642|194|7|1|11|12036.09|0.04|0.07|A|F|1995-05-23|1995-04-26|1995-06-04|COLLECT COD|TRUCK|lithely express asympt|
+4642|180|10|2|34|36726.12|0.04|0.07|R|F|1995-04-01|1995-05-11|1995-04-23|COLLECT COD|SHIP|theodolites detect among the ironically sp|
+4642|21|2|3|10|9210.20|0.04|0.02|R|F|1995-04-16|1995-04-28|1995-04-24|COLLECT COD|RAIL|urts. even deposits nag beneath |
+4642|94|7|4|18|17893.62|0.00|0.04|N|F|1995-06-16|1995-04-16|1995-06-21|NONE|TRUCK|ily pending accounts hag|
+4642|179|10|5|41|44245.97|0.10|0.00|R|F|1995-04-08|1995-04-13|1995-05-01|DELIVER IN PERSON|MAIL|s are blithely. requests wake above the fur|
+4643|185|6|1|50|54259.00|0.08|0.05|N|O|1995-09-11|1995-08-13|1995-09-30|DELIVER IN PERSON|SHIP|. ironic deposits cajo|
+4644|177|7|1|4|4308.68|0.06|0.03|N|O|1998-05-06|1998-03-19|1998-05-28|NONE|MAIL|gular requests? pendi|
+4644|97|8|2|16|15953.44|0.03|0.04|N|O|1998-03-13|1998-02-21|1998-04-03|COLLECT COD|SHIP|lar excuses across the |
+4644|115|9|3|10|10151.10|0.02|0.02|N|O|1998-02-21|1998-02-28|1998-03-19|COLLECT COD|REG AIR|osits according to the|
+4644|154|2|4|45|47436.75|0.10|0.07|N|O|1998-02-02|1998-04-08|1998-02-15|COLLECT COD|SHIP| carefully a|
+4644|87|8|5|10|9870.80|0.08|0.08|N|O|1998-03-12|1998-03-11|1998-03-19|TAKE BACK RETURN|REG AIR| the slow, final fo|
+4645|50|7|1|45|42752.25|0.09|0.05|A|F|1994-12-27|1994-11-02|1994-12-31|DELIVER IN PERSON|AIR|ular ideas. slyly|
+4645|66|7|2|32|30913.92|0.10|0.08|A|F|1994-11-17|1994-10-30|1994-11-18|COLLECT COD|REG AIR| final accounts alongside|
+4645|54|5|3|25|23851.25|0.03|0.00|R|F|1994-10-25|1994-12-11|1994-11-14|NONE|REG AIR|braids. ironic dependencies main|
+4645|37|8|4|42|39355.26|0.10|0.02|R|F|1994-12-02|1994-12-18|1994-12-16|COLLECT COD|TRUCK|regular pinto beans amon|
+4645|161|10|5|35|37140.60|0.03|0.07|A|F|1994-12-08|1994-11-25|1994-12-09|TAKE BACK RETURN|FOB|sias believe bl|
+4645|42|9|6|27|25435.08|0.09|0.08|R|F|1994-11-26|1994-10-25|1994-12-04|NONE|SHIP|ously express pinto beans. ironic depos|
+4645|31|2|7|42|39103.26|0.10|0.06|A|F|1994-12-31|1994-10-22|1995-01-28|DELIVER IN PERSON|AIR|e slyly regular pinto beans. thin|
+4646|191|3|1|24|26188.56|0.02|0.05|N|O|1996-09-18|1996-08-09|1996-09-21|TAKE BACK RETURN|RAIL|ic platelets lose carefully. blithely unu|
+4646|178|6|2|26|28032.42|0.07|0.00|N|O|1996-10-02|1996-08-25|1996-10-27|DELIVER IN PERSON|MAIL|ix according to the slyly spe|
+4646|34|10|3|18|16812.54|0.01|0.00|N|O|1996-06-30|1996-08-10|1996-07-12|TAKE BACK RETURN|TRUCK|beans sleep car|
+4646|40|1|4|38|35721.52|0.08|0.01|N|O|1996-09-01|1996-08-23|1996-09-27|COLLECT COD|SHIP|al platelets cajole. slyly final dol|
+4646|26|1|5|22|20372.44|0.01|0.08|N|O|1996-07-14|1996-08-06|1996-07-29|DELIVER IN PERSON|MAIL|cies are blithely after the slyly reg|
+4647|93|6|1|16|15889.44|0.09|0.07|R|F|1994-09-07|1994-07-15|1994-10-06|COLLECT COD|RAIL|o beans about the fluffily special the|
+4647|129|2|2|34|34990.08|0.01|0.02|R|F|1994-05-20|1994-06-20|1994-05-29|COLLECT COD|TRUCK|ly sly accounts|
+4647|147|8|3|27|28272.78|0.03|0.08|R|F|1994-05-20|1994-06-26|1994-05-30|NONE|FOB|ully even ti|
+4647|139|10|4|2|2078.26|0.04|0.07|R|F|1994-07-03|1994-07-22|1994-07-22|TAKE BACK RETURN|RAIL|dolites wake furiously special pinto be|
+4647|187|8|5|2|2174.36|0.07|0.06|A|F|1994-05-27|1994-08-05|1994-06-10|TAKE BACK RETURN|FOB| pinto beans believe furiously slyly silent|
+4647|29|4|6|28|26012.56|0.02|0.03|A|F|1994-08-25|1994-08-06|1994-09-18|DELIVER IN PERSON|FOB| are above the fluffily fin|
+4672|59|7|1|22|21099.10|0.01|0.07|N|O|1995-12-03|1995-12-08|1995-12-17|COLLECT COD|AIR|l instructions. blithely ironic packages |
+4672|61|10|2|41|39403.46|0.00|0.00|N|O|1995-12-01|1995-12-15|1995-12-12|COLLECT COD|RAIL| slyly quie|
+4672|163|10|3|24|25515.84|0.04|0.03|N|O|1995-11-11|1995-12-28|1995-12-04|NONE|REG AIR|y fluffily stealt|
+4672|57|2|4|13|12441.65|0.10|0.03|N|O|1996-02-02|1995-12-13|1996-03-02|DELIVER IN PERSON|RAIL|ar requests? pending accounts against|
+4672|55|10|5|45|42977.25|0.08|0.07|N|O|1996-02-07|1996-01-16|1996-02-14|DELIVER IN PERSON|MAIL| platelets use amon|
+4672|141|8|6|20|20822.80|0.02|0.07|N|O|1995-12-08|1996-01-25|1995-12-19|COLLECT COD|REG AIR|s boost at the ca|
+4672|72|10|7|38|36938.66|0.01|0.01|N|O|1995-11-28|1995-12-08|1995-12-13|COLLECT COD|SHIP|ests. idle, regular ex|
+4673|17|8|1|8|7336.08|0.08|0.01|N|O|1996-10-12|1996-10-05|1996-11-04|TAKE BACK RETURN|FOB|lithely final re|
+4673|101|2|2|44|44048.40|0.06|0.01|N|O|1996-12-11|1996-10-31|1997-01-08|DELIVER IN PERSON|RAIL| gifts cajole dari|
+4673|123|2|3|9|9208.08|0.04|0.07|N|O|1996-10-15|1996-09-30|1996-10-30|DELIVER IN PERSON|MAIL|ages nag across |
+4674|150|7|1|50|52507.50|0.07|0.08|A|F|1994-05-13|1994-06-15|1994-06-05|COLLECT COD|RAIL|haggle about the blithel|
+4674|189|10|2|35|38121.30|0.02|0.05|A|F|1994-08-02|1994-06-04|1994-08-21|COLLECT COD|FOB|le quickly after the express sent|
+4674|111|5|3|3|3033.33|0.01|0.05|A|F|1994-07-19|1994-05-28|1994-07-23|TAKE BACK RETURN|RAIL| regular requests na|
+4674|13|7|4|21|19173.21|0.02|0.08|R|F|1994-05-08|1994-07-02|1994-06-04|COLLECT COD|RAIL|ent accounts sublate deposits. instruc|
+4675|171|2|1|6|6427.02|0.00|0.05|R|F|1994-01-22|1994-01-06|1994-02-12|TAKE BACK RETURN|TRUCK| unusual ideas thrash bl|
+4675|144|7|2|12|12529.68|0.00|0.04|A|F|1993-12-22|1994-01-12|1993-12-23|TAKE BACK RETURN|AIR|posits affix carefully|
+4675|181|2|3|5|5405.90|0.05|0.05|A|F|1994-01-16|1994-01-05|1994-01-18|DELIVER IN PERSON|RAIL|lent pinto beans|
+4675|34|10|4|26|24284.78|0.03|0.01|A|F|1993-12-16|1993-12-29|1993-12-23|DELIVER IN PERSON|SHIP|nts. express requests are quickly |
+4675|81|2|5|18|17659.44|0.01|0.08|R|F|1994-02-23|1994-01-18|1994-03-05|TAKE BACK RETURN|FOB|cajole unusual dep|
+4675|119|10|6|1|1019.11|0.10|0.06|R|F|1994-03-18|1994-02-14|1994-04-17|NONE|SHIP|unts. caref|
+4676|165|2|1|47|50062.52|0.03|0.06|N|O|1995-12-20|1995-10-04|1996-01-09|NONE|AIR|lithely about the carefully special requ|
+4676|6|1|2|33|29898.00|0.08|0.05|N|O|1995-12-29|1995-10-01|1996-01-18|TAKE BACK RETURN|FOB|yly express |
+4676|146|3|3|4|4184.56|0.10|0.06|N|O|1995-12-12|1995-10-22|1995-12-13|TAKE BACK RETURN|TRUCK|detect above the ironic platelets. fluffily|
+4676|111|2|4|50|50555.50|0.07|0.01|N|O|1995-09-20|1995-11-20|1995-10-18|TAKE BACK RETURN|AIR|r deposits boost boldly quickly quick asymp|
+4676|122|7|5|29|29641.48|0.01|0.02|N|O|1995-12-29|1995-11-12|1996-01-06|TAKE BACK RETURN|RAIL|ly regular theodolites sleep.|
+4676|46|7|6|8|7568.32|0.08|0.08|N|O|1995-12-05|1995-10-18|1996-01-02|COLLECT COD|AIR|cuses boost above|
+4676|64|1|7|13|12532.78|0.05|0.07|N|O|1995-11-18|1995-11-07|1995-12-10|TAKE BACK RETURN|TRUCK| at the slyly bold attainments. silently e|
+4677|128|3|1|25|25703.00|0.04|0.04|N|O|1998-04-11|1998-05-11|1998-04-18|TAKE BACK RETURN|REG AIR|unts doubt furiousl|
+4678|58|6|1|35|33531.75|0.04|0.08|N|O|1998-11-27|1998-10-02|1998-12-17|TAKE BACK RETURN|AIR|he accounts. fluffily bold sheaves b|
+4678|117|1|2|18|18307.98|0.03|0.06|N|O|1998-10-30|1998-09-22|1998-11-25|TAKE BACK RETURN|SHIP|usly ironic |
+4678|96|9|3|13|12949.17|0.10|0.07|N|O|1998-11-03|1998-10-17|1998-11-06|TAKE BACK RETURN|SHIP|its. carefully final fr|
+4678|22|1|4|23|21206.46|0.06|0.05|N|O|1998-09-03|1998-09-20|1998-09-04|DELIVER IN PERSON|SHIP|ily sly deposi|
+4678|178|9|5|40|43126.80|0.03|0.07|N|O|1998-11-11|1998-10-27|1998-11-24|TAKE BACK RETURN|AIR|. final, unusual requests sleep thinl|
+4679|190|1|1|7|7631.33|0.10|0.05|R|F|1993-05-11|1993-04-11|1993-05-16|NONE|TRUCK|kages. bold, regular packa|
+4704|78|6|1|14|13692.98|0.04|0.04|N|O|1996-10-27|1996-11-02|1996-11-07|DELIVER IN PERSON|TRUCK| above the slyly final requests. quickly |
+4704|28|3|2|7|6496.14|0.03|0.04|N|O|1996-12-04|1996-10-30|1996-12-23|DELIVER IN PERSON|SHIP|ers wake car|
+4704|64|5|3|44|42418.64|0.02|0.05|N|O|1996-09-02|1996-10-07|1996-09-17|DELIVER IN PERSON|REG AIR|out the care|
+4705|111|8|1|22|22244.42|0.04|0.04|R|F|1992-07-05|1992-05-11|1992-07-29|DELIVER IN PERSON|SHIP| fluffily pending accounts ca|
+4705|31|7|2|14|13034.42|0.00|0.08|R|F|1992-07-14|1992-05-23|1992-07-25|DELIVER IN PERSON|TRUCK|ain carefully amon|
+4705|56|1|3|16|15296.80|0.07|0.08|R|F|1992-07-02|1992-06-06|1992-07-06|DELIVER IN PERSON|RAIL|special ideas nag sl|
+4705|130|3|4|31|31934.03|0.03|0.03|R|F|1992-04-03|1992-05-30|1992-04-05|COLLECT COD|TRUCK|furiously final accou|
+4705|163|10|5|28|29768.48|0.10|0.01|A|F|1992-06-03|1992-06-07|1992-06-22|DELIVER IN PERSON|MAIL|tes wake according to the unusual plate|
+4705|184|5|6|23|24936.14|0.06|0.03|R|F|1992-06-22|1992-06-11|1992-07-18|DELIVER IN PERSON|MAIL| above the furiously ev|
+4705|89|10|7|40|39563.20|0.08|0.06|A|F|1992-04-19|1992-04-28|1992-05-07|COLLECT COD|TRUCK|blithely. sly|
+4706|182|3|1|37|40040.66|0.02|0.06|A|F|1993-02-20|1993-03-05|1993-03-03|DELIVER IN PERSON|TRUCK|kly final deposits c|
+4706|122|3|2|23|23508.76|0.03|0.01|A|F|1993-04-01|1993-03-13|1993-05-01|COLLECT COD|FOB|deas across t|
+4706|68|5|3|6|5808.36|0.01|0.04|R|F|1993-01-20|1993-03-18|1993-01-26|NONE|MAIL|efully eve|
+4706|116|10|4|5|5080.55|0.06|0.06|R|F|1993-02-14|1993-01-31|1993-02-26|NONE|REG AIR|ptotes haggle ca|
+4706|50|7|5|27|25651.35|0.06|0.08|A|F|1993-04-04|1993-03-11|1993-04-09|COLLECT COD|REG AIR|into beans. finally special instruct|
+4707|34|5|1|7|6538.21|0.02|0.05|R|F|1995-05-14|1995-04-06|1995-06-06|COLLECT COD|SHIP|ecial sheaves boost blithely accor|
+4707|136|7|2|49|50770.37|0.00|0.07|N|F|1995-06-17|1995-05-16|1995-06-25|COLLECT COD|FOB| alongside of the slyly ironic instructio|
+4708|191|4|1|18|19641.42|0.02|0.04|A|F|1994-11-11|1994-11-15|1994-11-26|NONE|REG AIR|special, eve|
+4708|75|3|2|5|4875.35|0.05|0.05|A|F|1994-10-15|1994-12-02|1994-11-12|COLLECT COD|MAIL|ely. carefully sp|
+4708|77|7|3|32|31266.24|0.04|0.07|A|F|1994-11-12|1994-11-14|1994-11-23|TAKE BACK RETURN|MAIL|the accounts. e|
+4709|25|6|1|25|23125.50|0.03|0.05|N|O|1996-02-21|1996-02-11|1996-03-17|DELIVER IN PERSON|AIR|deposits grow. fluffily unusual accounts |
+4709|177|5|2|25|26929.25|0.05|0.03|N|O|1996-01-22|1996-03-03|1996-02-21|DELIVER IN PERSON|REG AIR|inst the ironic, regul|
+4710|183|4|1|40|43327.20|0.10|0.08|A|F|1995-03-09|1995-02-25|1995-03-29|TAKE BACK RETURN|AIR|cross the blithely bold packages. silen|
+4710|128|3|2|47|48321.64|0.04|0.01|R|F|1995-02-22|1995-01-12|1995-02-28|NONE|RAIL|blithely express packages. even, ironic re|
+4711|133|4|1|7|7231.91|0.03|0.01|N|O|1998-05-12|1998-06-24|1998-05-24|COLLECT COD|MAIL|ly. bold accounts use fluff|
+4711|145|6|2|15|15677.10|0.08|0.07|N|O|1998-06-09|1998-07-30|1998-06-18|COLLECT COD|SHIP| beans wake. deposits could bo|
+4711|150|1|3|22|23103.30|0.02|0.03|N|O|1998-06-21|1998-06-18|1998-07-19|TAKE BACK RETURN|REG AIR|along the quickly careful packages. bli|
+4711|65|10|4|8|7720.48|0.07|0.00|N|O|1998-06-17|1998-06-13|1998-06-27|TAKE BACK RETURN|SHIP|g to the carefully ironic deposits. specia|
+4711|49|2|5|15|14235.60|0.05|0.01|N|O|1998-09-03|1998-07-15|1998-09-13|TAKE BACK RETURN|SHIP|ld requests: furiously final inst|
+4711|116|7|6|45|45724.95|0.05|0.06|N|O|1998-05-19|1998-07-14|1998-05-21|COLLECT COD|SHIP| ironic theodolites |
+4711|46|5|7|18|17028.72|0.05|0.04|N|O|1998-07-03|1998-07-31|1998-07-23|DELIVER IN PERSON|RAIL| blithely. bold asymptote|
+4736|196|10|1|26|28500.94|0.03|0.03|N|O|1996-02-02|1996-01-18|1996-02-09|DELIVER IN PERSON|AIR|efully speci|
+4736|4|1|2|43|38872.00|0.06|0.07|N|O|1996-02-05|1995-12-21|1996-02-06|COLLECT COD|MAIL|quests. carefully |
+4737|191|5|1|37|40374.03|0.03|0.04|R|F|1993-05-17|1993-04-10|1993-05-30|DELIVER IN PERSON|TRUCK|s. fluffily regular |
+4737|69|8|2|22|21319.32|0.04|0.04|A|F|1993-03-29|1993-05-22|1993-04-16|TAKE BACK RETURN|RAIL| hang fluffily around t|
+4738|187|8|1|9|9784.62|0.04|0.04|A|F|1992-06-01|1992-06-26|1992-06-02|COLLECT COD|TRUCK|posits serve slyly. unusual pint|
+4738|173|3|2|16|17170.72|0.07|0.08|A|F|1992-06-17|1992-06-20|1992-06-21|NONE|MAIL|nic deposits are slyly! carefu|
+4738|100|2|3|50|50005.00|0.04|0.02|A|F|1992-06-18|1992-07-04|1992-07-07|TAKE BACK RETURN|TRUCK|the blithely ironic braids sleep slyly|
+4738|29|4|4|22|20438.44|0.02|0.08|A|F|1992-05-25|1992-05-19|1992-06-12|COLLECT COD|SHIP|ld, even packages. furio|
+4738|187|8|5|13|14133.34|0.04|0.05|R|F|1992-05-30|1992-06-11|1992-06-26|COLLECT COD|AIR| wake. unusual platelets for the|
+4738|159|1|6|10|10591.50|0.10|0.01|R|F|1992-07-10|1992-06-16|1992-07-25|TAKE BACK RETURN|SHIP|hins above the|
+4738|83|4|7|28|27526.24|0.05|0.07|A|F|1992-06-09|1992-07-05|1992-06-25|NONE|AIR|e furiously ironic excuses. care|
+4739|168|9|1|8|8545.28|0.07|0.07|R|F|1993-06-22|1993-05-10|1993-07-11|TAKE BACK RETURN|SHIP|cording to the |
+4739|185|6|2|31|33640.58|0.09|0.06|R|F|1993-06-20|1993-05-18|1993-06-26|COLLECT COD|SHIP|blithely special pin|
+4739|100|4|3|30|30003.00|0.09|0.00|A|F|1993-05-29|1993-04-12|1993-06-18|NONE|TRUCK|ly even packages use across th|
+4740|3|4|1|22|19866.00|0.06|0.01|N|O|1996-10-04|1996-08-17|1996-10-05|TAKE BACK RETURN|RAIL|final dependencies nag |
+4740|153|5|2|24|25275.60|0.08|0.02|N|O|1996-09-10|1996-09-27|1996-10-07|TAKE BACK RETURN|TRUCK|hely regular deposits|
+4741|73|2|1|24|23353.68|0.00|0.01|A|F|1992-09-16|1992-09-19|1992-09-20|DELIVER IN PERSON|RAIL|deas boost furiously slyly regular id|
+4741|113|4|2|16|16209.76|0.01|0.07|R|F|1992-08-25|1992-08-10|1992-08-29|TAKE BACK RETURN|FOB|final foxes haggle r|
+4741|156|8|3|24|25347.60|0.05|0.08|A|F|1992-11-04|1992-08-14|1992-11-06|TAKE BACK RETURN|MAIL|even requests.|
+4741|51|3|4|39|37090.95|0.09|0.06|R|F|1992-10-28|1992-10-03|1992-11-11|COLLECT COD|SHIP|t, regular requests|
+4741|179|10|5|40|43166.80|0.09|0.03|R|F|1992-09-20|1992-09-23|1992-10-09|TAKE BACK RETURN|REG AIR| fluffily slow deposits. fluffily regu|
+4741|157|5|6|34|35943.10|0.02|0.07|R|F|1992-08-25|1992-08-18|1992-09-20|DELIVER IN PERSON|RAIL|sly special packages after the furiously|
+4742|156|4|1|32|33796.80|0.10|0.08|R|F|1995-04-04|1995-06-12|1995-04-19|COLLECT COD|RAIL|eposits boost blithely. carefully regular a|
+4742|155|7|2|29|30599.35|0.02|0.03|N|F|1995-06-15|1995-05-05|1995-06-24|COLLECT COD|REG AIR|integrate closely among t|
+4742|72|10|3|15|14581.05|0.06|0.04|N|O|1995-07-20|1995-05-26|1995-08-11|NONE|SHIP|terns are sl|
+4742|188|9|4|31|33733.58|0.05|0.08|N|F|1995-06-13|1995-05-08|1995-06-24|COLLECT COD|REG AIR|ke slyly among the furiousl|
+4742|100|1|5|45|45004.50|0.05|0.00|R|F|1995-05-12|1995-05-14|1995-06-07|TAKE BACK RETURN|RAIL|ke carefully. do|
+4743|60|5|1|19|18241.14|0.04|0.07|A|F|1993-06-23|1993-05-03|1993-07-20|COLLECT COD|AIR|hely even accounts|
+4743|159|4|2|3|3177.45|0.01|0.03|R|F|1993-04-14|1993-06-08|1993-05-09|NONE|TRUCK|al requests. express idea|
+4743|73|2|3|21|20434.47|0.08|0.03|A|F|1993-07-02|1993-06-15|1993-07-26|DELIVER IN PERSON|RAIL|ake blithely against the packages. reg|
+4743|34|5|4|27|25218.81|0.08|0.05|R|F|1993-07-26|1993-05-27|1993-08-24|DELIVER IN PERSON|AIR|aids use. express deposits|
+4768|36|7|1|5|4680.15|0.00|0.03|R|F|1993-12-27|1994-02-09|1994-01-11|NONE|MAIL|egular accounts. bravely final fra|
+4769|35|1|1|16|14960.48|0.08|0.05|N|O|1995-07-16|1995-07-05|1995-07-22|TAKE BACK RETURN|FOB| deposits. slyly even asymptote|
+4769|63|8|2|34|32744.04|0.06|0.07|N|O|1995-07-26|1995-05-18|1995-08-03|COLLECT COD|REG AIR|ven instructions. ca|
+4769|47|10|3|36|34093.44|0.10|0.03|N|O|1995-07-22|1995-06-16|1995-08-11|NONE|RAIL|. slyly even deposit|
+4769|69|10|4|45|43607.70|0.08|0.06|R|F|1995-06-01|1995-07-13|1995-06-04|TAKE BACK RETURN|RAIL|accounts are. even accounts sleep|
+4769|112|6|5|15|15181.65|0.07|0.08|N|F|1995-06-12|1995-07-07|1995-07-04|NONE|SHIP|egular platelets can cajole across the |
+4770|32|8|1|41|38213.23|0.00|0.08|N|O|1995-09-04|1995-08-08|1995-09-10|COLLECT COD|FOB|ithely even packages sleep caref|
+4770|157|5|2|30|31714.50|0.09|0.07|N|O|1995-08-25|1995-08-27|1995-09-07|COLLECT COD|SHIP|ffily carefully ironic ideas. ironic d|
+4771|49|10|1|9|8541.36|0.01|0.00|R|F|1993-02-28|1993-02-19|1993-03-25|NONE|FOB|riously after the packages. fina|
+4771|16|7|2|21|19236.21|0.09|0.01|R|F|1993-01-19|1993-02-10|1993-02-01|NONE|FOB|fluffily pendi|
+4771|12|3|3|5|4560.05|0.06|0.08|R|F|1993-01-07|1993-01-19|1993-01-26|NONE|RAIL|ar, quiet accounts nag furiously express id|
+4771|9|4|4|21|19089.00|0.05|0.04|A|F|1992-12-20|1993-01-22|1992-12-26|TAKE BACK RETURN|SHIP| carefully re|
+4772|87|8|1|1|987.08|0.10|0.00|R|F|1994-11-13|1994-10-25|1994-11-15|DELIVER IN PERSON|AIR|ans. slyly even acc|
+4772|146|9|2|16|16738.24|0.07|0.06|R|F|1994-10-27|1994-12-07|1994-10-29|TAKE BACK RETURN|MAIL|egular accounts wake s|
+4772|95|6|3|31|30847.79|0.02|0.04|A|F|1994-10-02|1994-10-21|1994-10-13|TAKE BACK RETURN|FOB|ests are thinly. furiously unusua|
+4772|71|10|4|15|14566.05|0.02|0.07|R|F|1994-09-19|1994-10-22|1994-09-26|COLLECT COD|TRUCK| requests. express, regular th|
+4773|144|5|1|23|24015.22|0.00|0.08|N|O|1996-01-01|1996-03-19|1996-01-04|NONE|FOB|ly express grouches wak|
+4773|197|9|2|36|39498.84|0.09|0.04|N|O|1996-04-08|1996-03-03|1996-05-01|COLLECT COD|REG AIR| dependencies. quickly|
+4773|167|8|3|49|52290.84|0.05|0.02|N|O|1996-01-26|1996-02-29|1996-01-27|TAKE BACK RETURN|FOB|y final reque|
+4773|20|10|4|49|45080.98|0.09|0.04|N|O|1996-01-12|1996-02-17|1996-02-05|TAKE BACK RETURN|TRUCK|ly pending theodolites cajole caref|
+4773|150|3|5|20|21003.00|0.02|0.07|N|O|1995-12-28|1996-02-17|1996-01-15|COLLECT COD|TRUCK| blithely final deposits nag after t|
+4773|190|1|6|11|11992.09|0.10|0.06|N|O|1996-01-02|1996-01-29|1996-01-24|DELIVER IN PERSON|REG AIR|en accounts. slyly b|
+4773|158|3|7|6|6348.90|0.07|0.01|N|O|1996-03-09|1996-03-18|1996-03-27|NONE|AIR|latelets haggle s|
+4774|84|5|1|45|44283.60|0.10|0.00|R|F|1993-07-07|1993-06-08|1993-07-31|COLLECT COD|TRUCK| haggle busily afte|
+4774|39|5|2|4|3756.12|0.02|0.03|A|F|1993-08-03|1993-05-30|1993-08-19|COLLECT COD|FOB|xes according to the foxes wake above the f|
+4774|173|4|3|47|50438.99|0.10|0.08|R|F|1993-06-13|1993-07-04|1993-07-09|TAKE BACK RETURN|FOB|regular dolphins above the furi|
+4774|130|3|4|30|30903.90|0.05|0.08|A|F|1993-08-18|1993-06-08|1993-08-21|DELIVER IN PERSON|REG AIR|tions against the blithely final theodolit|
+4775|74|4|1|1|974.07|0.10|0.02|N|O|1995-09-06|1995-09-28|1995-09-29|DELIVER IN PERSON|MAIL|furiously ironic theodolite|
+4775|153|1|2|37|38966.55|0.02|0.01|N|O|1995-09-06|1995-09-28|1995-09-28|COLLECT COD|TRUCK|ts. pinto beans use according to th|
+4775|153|5|3|34|35807.10|0.09|0.06|N|O|1995-09-14|1995-10-15|1995-09-21|DELIVER IN PERSON|MAIL|onic epitaphs. f|
+4775|119|9|4|39|39745.29|0.07|0.04|N|O|1995-08-30|1995-10-12|1995-09-20|NONE|AIR|eep never with the slyly regular acc|
+4800|97|10|1|11|10967.99|0.03|0.03|R|F|1992-01-27|1992-03-16|1992-02-19|TAKE BACK RETURN|RAIL|ic dependenc|
+4800|26|5|2|1|926.02|0.06|0.06|A|F|1992-02-23|1992-03-16|1992-03-20|TAKE BACK RETURN|MAIL|nal accounts are blithely deposits. bol|
+4800|11|8|3|21|19131.21|0.09|0.05|A|F|1992-02-14|1992-03-15|1992-02-26|NONE|SHIP|ithely according to |
+4800|176|7|4|38|40894.46|0.10|0.08|R|F|1992-02-01|1992-02-28|1992-02-21|NONE|TRUCK|s sleep fluffily. furiou|
+4800|53|4|5|24|22873.20|0.08|0.04|R|F|1992-01-14|1992-02-23|1992-01-25|NONE|TRUCK|ully carefully r|
+4801|184|5|1|37|40114.66|0.10|0.02|N|O|1996-03-09|1996-02-29|1996-03-25|TAKE BACK RETURN|FOB|uests hinder blithely against the instr|
+4801|26|1|2|34|31484.68|0.03|0.02|N|O|1996-02-05|1996-04-16|1996-02-23|NONE|SHIP|y final requests |
+4801|110|1|3|4|4040.44|0.04|0.04|N|O|1996-03-23|1996-04-04|1996-03-25|COLLECT COD|RAIL|pitaphs. regular, reg|
+4801|92|3|4|39|38691.51|0.07|0.01|N|O|1996-03-19|1996-03-21|1996-04-17|TAKE BACK RETURN|REG AIR|warhorses wake never for the care|
+4802|40|1|1|6|5640.24|0.00|0.06|N|O|1997-04-16|1997-03-25|1997-04-21|TAKE BACK RETURN|SHIP|unusual accounts wake blithely. b|
+4803|132|3|1|2|2064.26|0.08|0.03|N|O|1996-04-16|1996-03-20|1996-05-15|NONE|REG AIR|gular reque|
+4803|176|4|2|47|50579.99|0.10|0.00|N|O|1996-03-14|1996-03-30|1996-03-15|DELIVER IN PERSON|FOB|ly final excuses. slyly express requ|
+4803|196|8|3|42|46039.98|0.04|0.08|N|O|1996-04-27|1996-05-05|1996-05-17|NONE|TRUCK| accounts affix quickly ar|
+4803|22|1|4|24|22128.48|0.10|0.04|N|O|1996-02-24|1996-04-02|1996-02-28|NONE|MAIL|t blithely slyly special decoys. |
+4803|189|10|5|21|22872.78|0.03|0.06|N|O|1996-05-25|1996-03-15|1996-06-09|COLLECT COD|FOB| silent packages use. b|
+4803|194|5|6|19|20789.61|0.07|0.00|N|O|1996-04-20|1996-03-25|1996-04-27|TAKE BACK RETURN|RAIL|sts. enticing, even|
+4804|128|1|1|44|45237.28|0.06|0.08|A|F|1992-05-02|1992-03-24|1992-05-28|TAKE BACK RETURN|AIR|aggle quickly among the slyly fi|
+4804|35|6|2|41|38336.23|0.10|0.02|R|F|1992-04-06|1992-04-12|1992-05-03|COLLECT COD|MAIL|. deposits haggle express tithes?|
+4804|65|2|3|33|31846.98|0.09|0.05|A|F|1992-03-02|1992-04-14|1992-03-13|DELIVER IN PERSON|AIR|, thin excuses. |
+4805|150|1|1|7|7351.05|0.09|0.03|A|F|1992-05-01|1992-07-09|1992-05-09|NONE|FOB| requests. regular deposit|
+4805|189|10|2|45|49013.10|0.02|0.03|R|F|1992-06-16|1992-06-08|1992-07-03|NONE|TRUCK|the furiously sly t|
+4805|154|6|3|44|46382.60|0.01|0.02|R|F|1992-05-14|1992-06-23|1992-05-25|DELIVER IN PERSON|SHIP|eposits sleep furiously qui|
+4805|65|2|4|13|12545.78|0.04|0.04|R|F|1992-07-16|1992-06-07|1992-08-10|COLLECT COD|AIR|its serve about the accounts. slyly regu|
+4805|9|10|5|42|38178.00|0.03|0.03|R|F|1992-08-17|1992-07-03|1992-09-14|NONE|REG AIR|the regular, fina|
+4805|136|7|6|18|18650.34|0.06|0.04|A|F|1992-06-07|1992-07-10|1992-06-12|COLLECT COD|TRUCK|o use pending, unusu|
+4806|16|7|1|26|23816.26|0.10|0.05|R|F|1993-05-28|1993-06-07|1993-05-29|DELIVER IN PERSON|SHIP| bold pearls sublate blithely. quickly pe|
+4806|72|10|2|6|5832.42|0.01|0.06|A|F|1993-05-17|1993-07-19|1993-05-29|TAKE BACK RETURN|SHIP|even theodolites. packages sl|
+4806|29|4|3|8|7432.16|0.09|0.00|A|F|1993-05-08|1993-07-16|1993-05-28|NONE|TRUCK|requests boost blithely. qui|
+4807|122|1|1|9|9199.08|0.04|0.08|N|O|1997-04-23|1997-03-01|1997-05-15|TAKE BACK RETURN|TRUCK|may are blithely. carefully even pinto b|
+4807|10|1|2|41|37310.41|0.07|0.08|N|O|1997-05-02|1997-03-31|1997-05-15|TAKE BACK RETURN|AIR| fluffily re|
+4807|145|6|3|34|35534.76|0.06|0.02|N|O|1997-01-31|1997-03-13|1997-02-01|NONE|SHIP|ecial ideas. deposits according to the fin|
+4807|190|1|4|32|34886.08|0.05|0.00|N|O|1997-04-04|1997-03-21|1997-04-16|NONE|RAIL|efully even dolphins slee|
+4807|159|1|5|2|2118.30|0.02|0.05|N|O|1997-05-09|1997-04-03|1997-06-05|TAKE BACK RETURN|RAIL|deas wake bli|
+4807|160|1|6|22|23323.52|0.09|0.06|N|O|1997-03-13|1997-02-23|1997-04-01|NONE|FOB|es use final excuses. furiously final|
+4832|15|6|1|23|21045.23|0.03|0.01|N|O|1997-12-05|1998-01-05|1997-12-10|NONE|RAIL|y express depo|
+4832|152|4|2|10|10521.50|0.00|0.06|N|O|1998-01-08|1998-02-01|1998-01-11|DELIVER IN PERSON|MAIL|ly. blithely bold pinto beans should have|
+4832|149|6|3|4|4196.56|0.04|0.01|N|O|1998-01-16|1998-02-12|1998-02-08|TAKE BACK RETURN|AIR|ages. slyly express deposits cajole car|
+4832|64|5|4|6|5784.36|0.02|0.01|N|O|1997-12-08|1998-02-03|1997-12-10|COLLECT COD|TRUCK|ages cajole after the bold requests. furi|
+4832|138|4|5|43|44639.59|0.10|0.08|N|O|1997-12-31|1998-02-20|1998-01-26|COLLECT COD|RAIL|oze according to the accou|
+4833|107|10|1|31|31220.10|0.08|0.04|N|O|1996-06-24|1996-07-15|1996-07-02|NONE|SHIP|ven instructions cajole against the caref|
+4833|117|7|2|11|11188.21|0.03|0.01|N|O|1996-08-24|1996-07-26|1996-09-19|NONE|REG AIR|s nag above the busily sile|
+4833|18|9|3|26|23868.26|0.08|0.04|N|O|1996-05-13|1996-07-12|1996-05-31|NONE|SHIP|s packages. even gif|
+4833|36|7|4|19|17784.57|0.07|0.07|N|O|1996-08-21|1996-07-09|1996-09-10|TAKE BACK RETURN|AIR|y quick theodolit|
+4833|35|1|5|4|3740.12|0.10|0.02|N|O|1996-08-16|1996-06-29|1996-08-22|NONE|AIR|y pending packages sleep blithely regular r|
+4834|183|4|1|27|29245.86|0.06|0.02|N|O|1997-01-09|1996-10-27|1997-01-27|DELIVER IN PERSON|RAIL|es nag blithe|
+4834|71|1|2|26|25247.82|0.01|0.00|N|O|1996-10-04|1996-10-21|1996-10-10|DELIVER IN PERSON|TRUCK|ages dazzle carefully. slyly daring foxes|
+4834|23|2|3|34|31382.68|0.03|0.01|N|O|1996-12-09|1996-11-26|1996-12-10|NONE|MAIL|ounts haggle bo|
+4834|143|10|4|38|39639.32|0.03|0.06|N|O|1997-01-10|1996-12-06|1997-01-22|COLLECT COD|FOB|alongside of the carefully even plate|
+4835|179|10|1|18|19425.06|0.00|0.03|R|F|1995-02-17|1994-12-14|1995-03-17|DELIVER IN PERSON|MAIL|eat furiously against the slyly |
+4835|91|3|2|3|2973.27|0.09|0.06|R|F|1995-01-24|1995-01-12|1995-02-16|COLLECT COD|AIR|etimes final pac|
+4835|86|7|3|27|26624.16|0.05|0.00|A|F|1994-12-10|1994-12-13|1995-01-02|DELIVER IN PERSON|REG AIR| accounts after the car|
+4835|102|7|4|23|23048.30|0.08|0.07|A|F|1995-02-05|1995-01-04|1995-02-28|NONE|SHIP|e carefully regular foxes. deposits are sly|
+4836|162|1|1|22|23367.52|0.01|0.03|N|O|1997-03-03|1997-02-23|1997-03-04|NONE|SHIP|al pinto beans. care|
+4836|48|5|2|16|15168.64|0.07|0.08|N|O|1997-01-14|1997-03-05|1997-01-30|COLLECT COD|MAIL|gular packages against the express reque|
+4836|76|4|3|14|13664.98|0.03|0.08|N|O|1997-02-21|1997-02-06|1997-03-08|COLLECT COD|MAIL|lites. unusual, bold dolphins ar|
+4836|106|1|4|15|15091.50|0.10|0.00|N|O|1997-03-08|1997-03-14|1997-03-30|TAKE BACK RETURN|TRUCK|eep slyly. even requests cajole|
+4836|51|6|5|12|11412.60|0.01|0.04|N|O|1997-02-02|1997-02-10|1997-02-03|COLLECT COD|TRUCK|sly ironic accoun|
+4837|42|1|1|16|15072.64|0.09|0.04|N|O|1998-08-12|1998-06-06|1998-08-26|COLLECT COD|FOB|ing requests are blithely regular instructi|
+4837|193|5|2|16|17491.04|0.01|0.02|N|O|1998-08-19|1998-06-18|1998-08-26|NONE|RAIL|counts cajole slyly furiou|
+4837|68|5|3|42|40658.52|0.10|0.00|N|O|1998-06-19|1998-07-06|1998-06-23|COLLECT COD|MAIL|o the furiously final theodolites boost|
+4838|122|3|1|35|35774.20|0.01|0.00|R|F|1992-10-30|1992-10-23|1992-11-21|TAKE BACK RETURN|RAIL|ly blithely unusual foxes. even package|
+4838|148|5|2|2|2096.28|0.03|0.08|R|F|1992-08-11|1992-09-16|1992-08-26|COLLECT COD|MAIL|hely final notornis are furiously blithe|
+4838|52|3|3|26|24753.30|0.06|0.04|R|F|1992-09-03|1992-10-25|1992-09-11|TAKE BACK RETURN|FOB|ular requests boost about the packages. r|
+4839|60|2|1|5|4800.30|0.10|0.07|A|F|1994-09-07|1994-07-15|1994-10-05|DELIVER IN PERSON|FOB|ses integrate. regular deposits are about |
+4839|10|1|2|25|22750.25|0.02|0.02|R|F|1994-05-20|1994-07-08|1994-05-30|NONE|REG AIR|regular packages ab|
+4839|60|1|3|18|17281.08|0.06|0.01|R|F|1994-05-18|1994-06-13|1994-06-09|TAKE BACK RETURN|FOB|blithely ironic theodolites use along|
+4839|100|1|4|19|19001.90|0.07|0.08|R|F|1994-05-20|1994-07-14|1994-05-30|NONE|REG AIR| deposits sublate furiously ir|
+4839|71|10|5|9|8739.63|0.05|0.01|R|F|1994-06-17|1994-06-18|1994-07-10|NONE|SHIP|ounts haggle carefully above|
+4864|150|9|1|28|29404.20|0.06|0.08|A|F|1993-02-06|1992-12-15|1993-02-10|COLLECT COD|REG AIR|thely around the bli|
+4864|38|4|2|38|35645.14|0.10|0.02|R|F|1992-12-20|1993-01-07|1993-01-06|TAKE BACK RETURN|SHIP|ording to the ironic, ir|
+4864|133|4|3|45|46490.85|0.02|0.01|A|F|1992-11-17|1993-01-02|1992-11-26|COLLECT COD|SHIP|round the furiously careful pa|
+4864|31|2|4|46|42827.38|0.07|0.03|A|F|1993-02-24|1993-01-02|1993-03-17|TAKE BACK RETURN|RAIL|sts use carefully across the carefull|
+4865|162|7|1|16|16994.56|0.07|0.05|N|O|1997-10-02|1997-08-20|1997-10-04|COLLECT COD|TRUCK|osits haggle. fur|
+4865|137|8|2|4|4148.52|0.07|0.01|N|O|1997-07-24|1997-07-25|1997-08-07|TAKE BACK RETURN|FOB|sts. blithely special instruction|
+4865|68|3|3|44|42594.64|0.10|0.08|N|O|1997-07-25|1997-08-20|1997-08-22|COLLECT COD|FOB|even deposits sleep against the quickly r|
+4865|50|3|4|21|19951.05|0.04|0.02|N|O|1997-07-17|1997-08-10|1997-07-21|NONE|RAIL|eposits detect sly|
+4865|54|9|5|33|31483.65|0.00|0.05|N|O|1997-07-17|1997-08-16|1997-07-30|TAKE BACK RETURN|FOB|y pending notornis ab|
+4865|65|2|6|47|45357.82|0.00|0.05|N|O|1997-08-26|1997-08-07|1997-08-31|NONE|RAIL|y unusual packages. packages|
+4866|11|8|1|9|8199.09|0.01|0.05|N|O|1997-08-30|1997-09-18|1997-09-24|TAKE BACK RETURN|MAIL|ven dependencies x-ray. quic|
+4866|102|3|2|1|1002.10|0.06|0.00|N|O|1997-10-15|1997-10-01|1997-11-14|TAKE BACK RETURN|AIR|latelets nag. q|
+4866|131|7|3|17|17529.21|0.07|0.00|N|O|1997-11-26|1997-10-11|1997-12-12|COLLECT COD|TRUCK|ess packages doubt. even somas wake f|
+4867|82|3|1|7|6874.56|0.09|0.03|A|F|1992-07-17|1992-08-17|1992-07-22|COLLECT COD|FOB|e carefully even packages. slyly ironic i|
+4867|160|8|2|3|3180.48|0.04|0.08|R|F|1992-07-04|1992-07-15|1992-07-21|NONE|AIR|yly silent deposits|
+4868|73|3|1|47|45734.29|0.03|0.03|N|O|1997-04-29|1997-04-27|1997-05-11|DELIVER IN PERSON|SHIP|gle unusual, fluffy packages. foxes cajol|
+4868|180|1|2|8|8641.44|0.10|0.08|N|O|1997-03-26|1997-05-09|1997-04-16|NONE|RAIL|ly special th|
+4868|191|2|3|49|53468.31|0.09|0.03|N|O|1997-04-23|1997-05-07|1997-04-26|NONE|SHIP|ys engage. th|
+4868|80|1|4|34|33322.72|0.04|0.02|N|O|1997-05-19|1997-04-27|1997-06-15|NONE|RAIL|en instructions about th|
+4868|122|3|5|22|22486.64|0.07|0.06|N|O|1997-04-26|1997-05-16|1997-05-01|DELIVER IN PERSON|FOB|osits. final foxes boost regular,|
+4869|41|8|1|31|29172.24|0.10|0.01|A|F|1995-01-17|1994-11-30|1995-02-02|NONE|SHIP|ins. always unusual ideas across the ir|
+4869|58|3|2|24|22993.20|0.09|0.06|A|F|1994-11-17|1994-11-07|1994-11-27|COLLECT COD|MAIL|olites cajole after the ideas. special t|
+4869|157|8|3|25|26428.75|0.00|0.05|R|F|1994-11-25|1994-11-14|1994-12-19|DELIVER IN PERSON|AIR|e according t|
+4869|103|8|4|24|24074.40|0.10|0.07|R|F|1994-11-23|1994-11-18|1994-12-11|DELIVER IN PERSON|MAIL|se deposits above the sly, q|
+4869|173|2|5|42|45073.14|0.07|0.04|R|F|1994-10-16|1994-12-10|1994-11-07|TAKE BACK RETURN|REG AIR| slyly even instructions. |
+4869|122|5|6|30|30663.60|0.00|0.05|A|F|1995-01-09|1994-11-20|1995-02-02|COLLECT COD|RAIL|gedly even requests. s|
+4870|48|5|1|49|46453.96|0.05|0.05|R|F|1994-11-14|1994-10-24|1994-12-12|TAKE BACK RETURN|SHIP| regular packages |
+4870|127|10|2|6|6162.72|0.06|0.08|A|F|1994-09-09|1994-10-16|1994-09-21|DELIVER IN PERSON|TRUCK|ress requests. bold, silent pinto bea|
+4870|31|2|3|5|4655.15|0.05|0.00|R|F|1994-10-11|1994-10-07|1994-10-24|NONE|AIR|s haggle furiously. slyly ironic dinos|
+4870|6|9|4|4|3624.00|0.03|0.08|A|F|1994-10-23|1994-09-16|1994-11-04|COLLECT COD|RAIL|its wake quickly. slyly quick|
+4870|71|1|5|36|34958.52|0.09|0.06|A|F|1994-09-06|1994-09-17|1994-10-01|COLLECT COD|REG AIR| instructions. carefully pending pac|
+4871|177|5|1|14|15080.38|0.07|0.03|N|O|1995-09-30|1995-07-29|1995-10-18|TAKE BACK RETURN|REG AIR|inst the never ironic |
+4871|161|6|2|17|18039.72|0.07|0.03|N|O|1995-09-09|1995-09-01|1995-10-02|DELIVER IN PERSON|AIR|es. carefully ev|
+4871|63|4|3|3|2889.18|0.03|0.06|N|O|1995-10-03|1995-08-10|1995-10-06|DELIVER IN PERSON|TRUCK|y special packages wak|
+4871|149|8|4|35|36719.90|0.08|0.07|N|O|1995-08-11|1995-07-18|1995-08-29|DELIVER IN PERSON|TRUCK|ackages sle|
+4871|152|3|5|10|10521.50|0.09|0.02|N|O|1995-09-12|1995-09-02|1995-10-05|TAKE BACK RETURN|AIR|s integrate after the a|
+4871|136|2|6|36|37300.68|0.02|0.08|N|O|1995-09-18|1995-08-29|1995-10-05|TAKE BACK RETURN|AIR|ely according|
+4871|140|6|7|10|10401.40|0.10|0.02|N|O|1995-07-13|1995-08-19|1995-07-29|NONE|REG AIR|p ironic theodolites. slyly even platel|
+4896|41|2|1|19|17879.76|0.09|0.05|A|F|1992-12-13|1992-11-13|1993-01-09|NONE|AIR|nusual requ|
+4896|140|1|2|44|45766.16|0.04|0.03|A|F|1992-11-24|1992-11-15|1992-12-18|COLLECT COD|MAIL|e after the slowly f|
+4896|58|10|3|6|5748.30|0.04|0.04|A|F|1992-10-30|1992-11-12|1992-11-28|DELIVER IN PERSON|TRUCK|usly regular deposits|
+4896|23|4|4|5|4615.10|0.08|0.02|R|F|1992-12-02|1992-11-11|1992-12-19|COLLECT COD|SHIP|eposits hang carefully. sly|
+4896|86|7|5|21|20707.68|0.07|0.08|R|F|1992-11-18|1992-11-18|1992-11-29|DELIVER IN PERSON|TRUCK|ly express deposits. carefully pending depo|
+4897|55|6|1|26|24831.30|0.01|0.01|R|F|1992-12-22|1992-10-25|1992-12-27|DELIVER IN PERSON|TRUCK|. carefully ironic dep|
+4897|143|6|2|34|35466.76|0.02|0.00|R|F|1992-12-31|1992-11-11|1993-01-30|COLLECT COD|AIR|ts. special dependencies use fluffily |
+4897|55|7|3|42|40112.10|0.09|0.03|A|F|1992-09-23|1992-10-28|1992-10-02|DELIVER IN PERSON|FOB|sts. blithely regular deposits will have|
+4897|104|5|4|19|19077.90|0.03|0.00|A|F|1992-11-08|1992-12-14|1992-12-03|DELIVER IN PERSON|FOB|! ironic, pending dependencies doze furiou|
+4898|72|1|1|44|42771.08|0.07|0.02|A|F|1994-09-13|1994-08-18|1994-09-16|NONE|FOB|y regular grouches about|
+4899|34|10|1|14|13076.42|0.06|0.00|R|F|1993-11-10|1994-01-10|1993-11-20|NONE|REG AIR| foxes eat|
+4900|116|3|1|40|40644.40|0.10|0.03|A|F|1992-09-02|1992-09-25|1992-09-21|COLLECT COD|TRUCK|heodolites. request|
+4900|77|8|2|33|32243.31|0.06|0.06|R|F|1992-08-18|1992-09-20|1992-08-19|COLLECT COD|MAIL|nto beans nag slyly reg|
+4900|103|8|3|48|48148.80|0.02|0.00|R|F|1992-09-18|1992-08-14|1992-09-28|TAKE BACK RETURN|MAIL|uickly ironic ideas kindle s|
+4900|32|3|4|20|18640.60|0.05|0.00|R|F|1992-09-22|1992-09-23|1992-09-27|TAKE BACK RETURN|MAIL|yers. accounts affix somet|
+4900|105|8|5|40|40204.00|0.03|0.02|R|F|1992-07-14|1992-09-05|1992-07-20|NONE|REG AIR|luffily final dol|
+4900|103|6|6|46|46142.60|0.06|0.08|R|F|1992-07-11|1992-09-19|1992-07-16|TAKE BACK RETURN|SHIP|ly final acco|
+4901|141|10|1|37|38522.18|0.00|0.04|N|O|1998-01-26|1998-02-20|1998-01-31|DELIVER IN PERSON|TRUCK| furiously ev|
+4901|165|4|2|12|12781.92|0.00|0.04|N|O|1998-01-12|1998-02-06|1998-02-03|COLLECT COD|REG AIR|y unusual deposits prom|
+4901|120|4|3|16|16321.92|0.05|0.08|N|O|1998-04-19|1998-03-18|1998-04-21|NONE|AIR|deposits. blithely fin|
+4901|36|7|4|41|38377.23|0.03|0.00|N|O|1998-03-18|1998-02-18|1998-04-14|TAKE BACK RETURN|AIR|efully bold packages affix carefully eve|
+4901|116|7|5|40|40644.40|0.06|0.02|N|O|1998-01-08|1998-01-30|1998-01-15|DELIVER IN PERSON|MAIL|ect across the furiou|
+4902|196|10|1|22|24116.18|0.00|0.04|N|O|1998-10-17|1998-08-10|1998-10-21|COLLECT COD|RAIL|r the furiously final fox|
+4902|83|4|2|1|983.08|0.09|0.04|N|O|1998-10-12|1998-08-20|1998-11-08|NONE|RAIL|daring foxes? even, bold requests wake f|
+4903|121|2|1|1|1021.12|0.06|0.03|R|F|1992-04-23|1992-06-13|1992-05-03|NONE|SHIP|nusual requests|
+4903|165|6|2|6|6390.96|0.09|0.07|R|F|1992-04-01|1992-05-16|1992-04-11|DELIVER IN PERSON|SHIP|azzle quickly along the blithely final pla|
+4903|120|10|3|27|27543.24|0.07|0.06|A|F|1992-06-29|1992-06-09|1992-07-08|COLLECT COD|RAIL|pinto beans are; |
+4928|100|1|1|4|4000.40|0.04|0.02|R|F|1993-10-25|1993-12-24|1993-11-16|TAKE BACK RETURN|REG AIR|bout the slyly final accounts. carefull|
+4928|93|4|2|20|19861.80|0.03|0.08|A|F|1994-01-19|1993-11-29|1994-02-13|DELIVER IN PERSON|SHIP|quiet theodolites ca|
+4928|149|8|3|34|35670.76|0.06|0.05|A|F|1993-10-12|1993-12-31|1993-10-14|DELIVER IN PERSON|AIR|, regular depos|
+4929|14|1|1|20|18280.20|0.00|0.04|N|O|1996-03-12|1996-05-23|1996-03-20|COLLECT COD|REG AIR| final pinto beans detect. final,|
+4929|79|7|2|40|39162.80|0.08|0.03|N|O|1996-05-30|1996-04-13|1996-06-22|TAKE BACK RETURN|AIR|unts against |
+4929|77|7|3|32|31266.24|0.08|0.02|N|O|1996-04-28|1996-05-23|1996-04-30|COLLECT COD|TRUCK|usly at the blithely pending pl|
+4929|109|4|4|26|26236.60|0.00|0.05|N|O|1996-06-10|1996-05-29|1996-06-26|DELIVER IN PERSON|RAIL| slyly. fl|
+4929|67|8|5|24|23209.44|0.09|0.05|N|O|1996-04-15|1996-04-30|1996-05-09|NONE|MAIL| accounts boost|
+4930|187|8|1|35|38051.30|0.03|0.01|A|F|1994-07-09|1994-07-30|1994-07-15|NONE|RAIL|lose slyly regular dependencies. fur|
+4930|115|5|2|20|20302.20|0.02|0.04|A|F|1994-08-21|1994-06-17|1994-08-24|COLLECT COD|FOB|he carefully|
+4930|168|7|3|28|29908.48|0.00|0.08|R|F|1994-08-27|1994-06-27|1994-09-18|COLLECT COD|TRUCK|e ironic, unusual courts. regula|
+4930|166|7|4|42|44778.72|0.00|0.00|A|F|1994-06-18|1994-06-22|1994-07-10|COLLECT COD|AIR|ions haggle. furiously regular ideas use |
+4930|190|1|5|38|41427.22|0.02|0.03|A|F|1994-06-06|1994-06-18|1994-07-03|TAKE BACK RETURN|AIR|bold requests sleep never|
+4931|194|7|1|1|1094.19|0.08|0.06|A|F|1995-01-24|1994-12-19|1995-02-07|DELIVER IN PERSON|SHIP| furiously |
+4931|151|3|2|8|8409.20|0.06|0.02|R|F|1994-12-15|1995-01-14|1995-01-06|NONE|SHIP|ts boost. packages wake sly|
+4931|144|5|3|20|20882.80|0.09|0.00|A|F|1995-01-25|1994-12-21|1995-02-06|DELIVER IN PERSON|MAIL|the furious|
+4931|200|4|4|50|55010.00|0.04|0.01|A|F|1994-12-15|1994-12-18|1994-12-23|COLLECT COD|REG AIR|s haggle al|
+4931|150|7|5|25|26253.75|0.05|0.05|R|F|1994-12-19|1995-01-05|1994-12-21|COLLECT COD|FOB|aggle bravely according to the quic|
+4931|103|6|6|8|8024.80|0.02|0.03|A|F|1995-02-16|1994-12-30|1995-03-15|DELIVER IN PERSON|SHIP|dependencies are slyly|
+4932|51|3|1|13|12363.65|0.04|0.03|A|F|1993-09-13|1993-10-16|1993-09-20|DELIVER IN PERSON|SHIP|slyly according to the furiously fin|
+4932|103|10|2|15|15046.50|0.01|0.02|R|F|1993-11-15|1993-10-25|1993-11-29|NONE|REG AIR|yly. unusu|
+4932|87|8|3|5|4935.40|0.06|0.06|A|F|1993-10-01|1993-09-13|1993-10-04|NONE|MAIL| haggle furiously. slyly ironic packages sl|
+4932|98|1|4|11|10978.99|0.09|0.06|A|F|1993-09-21|1993-09-30|1993-09-23|COLLECT COD|SHIP|as. special depende|
+4933|32|8|1|48|44737.44|0.08|0.00|N|O|1995-10-10|1995-10-03|1995-11-04|COLLECT COD|SHIP|ideas. sly|
+4933|82|3|2|2|1964.16|0.09|0.00|N|O|1995-10-01|1995-09-29|1995-10-19|DELIVER IN PERSON|MAIL|ctions nag final instructions. accou|
+4934|97|10|1|48|47860.32|0.00|0.01|N|O|1997-05-20|1997-04-22|1997-06-02|TAKE BACK RETURN|SHIP| ideas cajol|
+4934|110|1|2|41|41414.51|0.06|0.06|N|O|1997-06-04|1997-04-11|1997-06-25|TAKE BACK RETURN|FOB|wake final, ironic f|
+4934|140|1|3|8|8321.12|0.03|0.06|N|O|1997-05-20|1997-04-30|1997-05-27|TAKE BACK RETURN|MAIL|arefully express pains cajo|
+4934|148|5|4|9|9433.26|0.06|0.08|N|O|1997-06-10|1997-04-09|1997-06-12|TAKE BACK RETURN|REG AIR| haggle alongside of the|
+4934|138|9|5|29|30105.77|0.09|0.03|N|O|1997-04-10|1997-05-05|1997-05-04|DELIVER IN PERSON|AIR|aggle furiously among the busily final re|
+4934|52|3|6|42|39986.10|0.00|0.07|N|O|1997-03-19|1997-05-05|1997-03-25|NONE|MAIL|ven, ironic ideas|
+4934|11|5|7|2|1822.02|0.10|0.06|N|O|1997-06-05|1997-03-26|1997-06-09|COLLECT COD|MAIL|ongside of the brave, regula|
+4935|161|2|1|13|13795.08|0.09|0.01|A|F|1993-06-20|1993-08-13|1993-06-27|COLLECT COD|REG AIR|ly requests. final deposits might |
+4935|40|6|2|37|34781.48|0.01|0.05|R|F|1993-08-30|1993-07-23|1993-09-07|TAKE BACK RETURN|RAIL|y even dependencies nag a|
+4935|11|8|3|24|21864.24|0.06|0.04|A|F|1993-05-29|1993-08-17|1993-06-22|NONE|RAIL|ly quickly s|
+4935|45|6|4|49|46306.96|0.06|0.01|A|F|1993-09-16|1993-08-21|1993-10-12|COLLECT COD|TRUCK|ffily after the furiou|
+4935|10|1|5|14|12740.14|0.08|0.08|A|F|1993-05-30|1993-07-25|1993-05-31|COLLECT COD|FOB|slowly. blith|
+4935|188|9|6|36|39174.48|0.10|0.00|R|F|1993-07-11|1993-07-04|1993-08-01|DELIVER IN PERSON|RAIL|requests across the quick|
+4960|18|5|1|36|33048.36|0.01|0.05|R|F|1995-03-06|1995-05-04|1995-04-05|TAKE BACK RETURN|RAIL|c, unusual accou|
+4960|45|8|2|6|5670.24|0.03|0.08|R|F|1995-03-21|1995-05-13|1995-04-14|TAKE BACK RETURN|SHIP|ual package|
+4960|149|8|3|9|9442.26|0.01|0.03|A|F|1995-03-20|1995-05-05|1995-04-17|COLLECT COD|RAIL|e blithely carefully fina|
+4960|120|7|4|14|14281.68|0.00|0.06|A|F|1995-04-03|1995-04-17|1995-04-07|NONE|RAIL|accounts. warhorses are. grouches |
+4960|98|1|5|8|7984.72|0.07|0.04|R|F|1995-03-14|1995-04-18|1995-04-09|NONE|FOB|as. busily regular packages nag. |
+4960|146|7|6|37|38707.18|0.10|0.04|R|F|1995-05-23|1995-04-12|1995-06-01|DELIVER IN PERSON|MAIL|ending theodolites w|
+4960|170|1|7|42|44947.14|0.08|0.07|A|F|1995-04-19|1995-04-11|1995-05-08|NONE|SHIP|s requests cajole. |
+4961|44|7|1|38|35873.52|0.10|0.07|N|O|1998-07-09|1998-06-03|1998-07-11|TAKE BACK RETURN|FOB|e on the blithely bold accounts. unu|
+4961|60|5|2|1|960.06|0.08|0.08|N|O|1998-07-08|1998-05-25|1998-07-12|DELIVER IN PERSON|MAIL|s affix carefully silent dependen|
+4961|162|3|3|41|43548.56|0.02|0.02|N|O|1998-07-15|1998-06-15|1998-08-05|TAKE BACK RETURN|REG AIR|ily against the n|
+4961|100|3|4|10|10001.00|0.02|0.04|N|O|1998-04-15|1998-07-03|1998-04-18|DELIVER IN PERSON|MAIL|quests. regular, ironic ideas at the ironi|
+4962|19|6|1|46|42274.46|0.01|0.07|R|F|1993-08-23|1993-09-04|1993-08-27|COLLECT COD|REG AIR| pinto beans grow about the sl|
+4963|168|5|1|38|40590.08|0.08|0.02|N|O|1996-12-25|1996-12-12|1997-01-02|COLLECT COD|AIR|tegrate daringly accou|
+4963|76|4|2|16|15617.12|0.00|0.03|N|O|1996-11-20|1997-01-13|1996-12-06|COLLECT COD|MAIL| carefully slyly u|
+4964|133|9|1|29|29960.77|0.04|0.01|N|O|1997-10-18|1997-08-30|1997-11-01|NONE|AIR|k accounts nag carefully-- ironic, fin|
+4964|148|5|2|46|48214.44|0.06|0.06|N|O|1997-10-05|1997-09-12|1997-10-11|NONE|TRUCK|althy deposits|
+4964|143|4|3|18|18776.52|0.00|0.06|N|O|1997-10-13|1997-09-01|1997-11-10|DELIVER IN PERSON|AIR| platelets. furio|
+4964|180|10|4|12|12962.16|0.08|0.01|N|O|1997-09-03|1997-10-25|1997-09-15|NONE|TRUCK|ully silent instructions ca|
+4964|41|10|5|42|39523.68|0.06|0.04|N|O|1997-09-04|1997-08-28|1997-10-02|TAKE BACK RETURN|AIR| hinder. idly even|
+4964|193|7|6|22|24050.18|0.04|0.08|N|O|1997-09-11|1997-10-06|1997-09-29|NONE|AIR|equests doubt quickly. caref|
+4964|173|4|7|28|30048.76|0.00|0.05|N|O|1997-08-30|1997-09-15|1997-09-18|COLLECT COD|RAIL|among the carefully regula|
+4965|131|2|1|28|28871.64|0.05|0.03|A|F|1994-01-02|1993-11-20|1994-01-04|TAKE BACK RETURN|REG AIR| deposits. requests sublate quickly |
+4965|13|10|2|25|22825.25|0.10|0.02|R|F|1994-02-05|1993-12-15|1994-02-24|TAKE BACK RETURN|MAIL|wake at the carefully speci|
+4965|101|8|3|27|27029.70|0.05|0.06|R|F|1993-11-06|1993-12-24|1993-11-30|TAKE BACK RETURN|SHIP|efully final foxes|
+4965|138|9|4|33|34258.29|0.04|0.04|A|F|1993-12-31|1993-11-29|1994-01-27|DELIVER IN PERSON|REG AIR|iously slyly|
+4966|76|6|1|10|9760.70|0.06|0.03|N|O|1996-09-23|1996-11-02|1996-10-07|TAKE BACK RETURN|SHIP| requests. carefully pending requests|
+4966|194|6|2|6|6565.14|0.02|0.01|N|O|1996-12-09|1996-11-29|1996-12-30|NONE|AIR|d deposits are sly excuses. slyly iro|
+4966|165|6|3|7|7456.12|0.00|0.01|N|O|1996-12-08|1996-10-09|1997-01-06|COLLECT COD|MAIL|ckly ironic tithe|
+4966|16|6|4|26|23816.26|0.08|0.03|N|O|1996-11-14|1996-11-29|1996-12-05|COLLECT COD|REG AIR|nt pearls haggle carefully slyly even |
+4966|144|1|5|12|12529.68|0.02|0.07|N|O|1996-12-07|1996-11-23|1996-12-20|DELIVER IN PERSON|RAIL|eodolites. ironic requests across the exp|
+4967|71|1|1|50|48553.50|0.07|0.01|N|O|1997-05-27|1997-05-13|1997-06-12|NONE|REG AIR|kages. final, unusual accounts c|
+4967|53|5|2|43|40981.15|0.00|0.07|N|O|1997-05-28|1997-04-10|1997-06-09|NONE|TRUCK|ons. slyly ironic requests|
+4967|50|1|3|15|14250.75|0.08|0.02|N|O|1997-04-16|1997-04-12|1997-05-08|TAKE BACK RETURN|MAIL|y. blithel|
+4967|123|2|4|1|1023.12|0.10|0.07|N|O|1997-06-04|1997-03-29|1997-06-23|NONE|FOB|osits. unusual frets thrash furiously|
+4992|184|5|1|42|45535.56|0.07|0.01|R|F|1992-07-19|1992-06-16|1992-08-17|TAKE BACK RETURN|RAIL|foxes about the quickly final platele|
+4992|147|4|2|47|49215.58|0.10|0.08|A|F|1992-09-04|1992-08-05|1992-09-21|COLLECT COD|MAIL|atterns use fluffily.|
+4992|144|7|3|17|17750.38|0.03|0.03|A|F|1992-07-05|1992-07-19|1992-07-30|TAKE BACK RETURN|FOB|s along the perma|
+4992|70|7|4|25|24251.75|0.04|0.06|R|F|1992-08-06|1992-07-11|1992-08-20|NONE|SHIP|ly about the never ironic requests. pe|
+4992|139|5|5|23|23899.99|0.01|0.08|R|F|1992-06-28|1992-07-15|1992-07-12|DELIVER IN PERSON|MAIL|uickly regul|
+4992|163|8|6|44|46779.04|0.05|0.02|A|F|1992-06-01|1992-07-22|1992-06-03|NONE|RAIL|rmanent, sly packages print slyly. regula|
+4993|38|4|1|34|31893.02|0.05|0.00|R|F|1994-09-21|1994-10-31|1994-09-24|TAKE BACK RETURN|REG AIR|ular, pending packages at the even packa|
+4993|129|4|2|39|40135.68|0.03|0.08|R|F|1994-09-10|1994-09-04|1994-09-26|COLLECT COD|SHIP|pending, regular requests solve caref|
+4993|166|1|3|42|44778.72|0.06|0.00|A|F|1994-08-27|1994-09-24|1994-09-05|NONE|MAIL| final packages at the q|
+4993|158|6|4|31|32802.65|0.10|0.06|A|F|1994-10-02|1994-10-29|1994-10-15|NONE|AIR|nwind thinly platelets. a|
+4994|156|8|1|36|38021.40|0.00|0.06|N|O|1996-09-29|1996-07-30|1996-10-03|TAKE BACK RETURN|TRUCK|ess ideas. blithely silent brai|
+4994|80|9|2|47|46063.76|0.04|0.05|N|O|1996-09-20|1996-08-04|1996-10-15|COLLECT COD|TRUCK|sts. blithely close ideas sleep quic|
+4994|183|4|3|29|31412.22|0.08|0.01|N|O|1996-08-26|1996-09-27|1996-09-25|DELIVER IN PERSON|RAIL|ptotes boost carefully|
+4994|39|10|4|40|37561.20|0.01|0.06|N|O|1996-08-25|1996-08-16|1996-09-07|TAKE BACK RETURN|REG AIR|eposits. regula|
+4994|42|9|5|24|22608.96|0.01|0.07|N|O|1996-08-19|1996-09-24|1996-08-25|TAKE BACK RETURN|FOB|s. slyly ironic deposits cajole f|
+4994|73|4|6|6|5838.42|0.01|0.02|N|O|1996-09-05|1996-08-04|1996-09-30|TAKE BACK RETURN|FOB|grate carefully around th|
+4994|130|1|7|31|31934.03|0.07|0.04|N|O|1996-10-14|1996-09-23|1996-11-08|TAKE BACK RETURN|RAIL|lar decoys cajole fluffil|
+4995|65|4|1|16|15440.96|0.02|0.05|N|O|1996-02-27|1996-04-03|1996-02-29|DELIVER IN PERSON|MAIL|egular, bold packages. accou|
+4995|81|2|2|43|42186.44|0.00|0.06|N|O|1996-02-24|1996-02-20|1996-03-07|NONE|AIR|ts. blithely silent ideas after t|
+4995|156|7|3|22|23235.30|0.03|0.06|N|O|1996-03-17|1996-03-12|1996-04-01|DELIVER IN PERSON|MAIL|s wake furious, express dependencies.|
+4995|40|1|4|9|8460.36|0.07|0.07|N|O|1996-03-07|1996-03-17|1996-03-11|DELIVER IN PERSON|FOB| ironic packages cajole across t|
+4995|148|7|5|48|50310.72|0.08|0.07|N|O|1996-03-22|1996-04-01|1996-04-07|NONE|SHIP|t blithely. requests affix blithely. |
+4995|110|5|6|48|48485.28|0.09|0.07|N|O|1996-04-14|1996-04-04|1996-05-07|DELIVER IN PERSON|RAIL|nstructions. carefully final depos|
+4996|56|1|1|35|33461.75|0.07|0.01|A|F|1992-10-30|1992-10-27|1992-11-05|TAKE BACK RETURN|SHIP|s. unusual, regular dolphins integrate care|
+4996|156|7|2|39|41189.85|0.02|0.07|A|F|1992-09-19|1992-10-19|1992-10-06|COLLECT COD|FOB|equests are carefully final|
+4996|128|7|3|12|12337.44|0.04|0.06|R|F|1993-01-09|1992-11-22|1993-02-04|DELIVER IN PERSON|SHIP|usly bold requests sleep dogge|
+4996|144|3|4|13|13573.82|0.00|0.00|A|F|1992-09-17|1992-12-02|1992-10-07|DELIVER IN PERSON|TRUCK|o beans use about the furious|
+4997|79|7|1|44|43079.08|0.02|0.05|N|O|1998-06-09|1998-06-12|1998-07-07|NONE|RAIL|r escapades ca|
+4997|17|7|2|5|4585.05|0.02|0.04|N|O|1998-05-16|1998-06-05|1998-06-07|COLLECT COD|REG AIR|cuses are furiously unusual asymptotes|
+4997|58|9|3|24|22993.20|0.04|0.06|N|O|1998-04-20|1998-04-23|1998-05-16|NONE|AIR|xpress, bo|
+4997|40|6|4|5|4700.20|0.10|0.03|N|O|1998-06-12|1998-04-24|1998-06-13|DELIVER IN PERSON|TRUCK|aggle slyly alongside of the slyly i|
+4997|22|7|5|46|42412.92|0.00|0.04|N|O|1998-04-28|1998-06-04|1998-05-08|TAKE BACK RETURN|SHIP|ecial courts are carefully|
+4997|29|2|6|2|1858.04|0.07|0.01|N|O|1998-07-09|1998-06-10|1998-07-21|TAKE BACK RETURN|REG AIR|counts. slyl|
+4998|154|2|1|12|12649.80|0.04|0.03|A|F|1992-02-20|1992-03-06|1992-03-01|TAKE BACK RETURN|RAIL| sleep slyly furiously final accounts. ins|
+4998|183|4|2|15|16247.70|0.06|0.00|R|F|1992-04-24|1992-03-21|1992-05-02|NONE|REG AIR|heodolites sleep quickly.|
+4998|59|10|3|27|25894.35|0.06|0.02|R|F|1992-03-17|1992-02-26|1992-04-05|DELIVER IN PERSON|MAIL|the blithely ironic |
+4998|63|10|4|47|45263.82|0.10|0.04|A|F|1992-02-07|1992-03-07|1992-02-19|DELIVER IN PERSON|TRUCK|mong the careful|
+4998|145|4|5|24|25083.36|0.01|0.04|R|F|1992-01-25|1992-03-16|1992-01-27|COLLECT COD|REG AIR| unwind about|
+4998|99|1|6|8|7992.72|0.03|0.07|A|F|1992-05-01|1992-03-03|1992-05-24|TAKE BACK RETURN|AIR|ions nag quickly according to the theodolit|
+4999|153|8|1|30|31594.50|0.00|0.02|A|F|1993-08-20|1993-08-15|1993-08-30|NONE|AIR|ades cajole carefully unusual ide|
+4999|10|1|2|44|40040.44|0.03|0.01|A|F|1993-08-01|1993-08-04|1993-08-17|COLLECT COD|REG AIR|ependencies. slowly regu|
+4999|86|7|3|30|29582.40|0.09|0.01|R|F|1993-07-21|1993-08-11|1993-08-20|DELIVER IN PERSON|RAIL|s cajole among the blithel|
+5024|166|3|1|17|18124.72|0.10|0.02|N|O|1996-11-24|1997-01-10|1996-12-04|NONE|AIR| to the expre|
+5024|58|6|2|41|39280.05|0.06|0.01|N|O|1996-11-09|1996-12-03|1996-12-01|COLLECT COD|REG AIR|osits hinder carefully |
+5024|112|6|3|18|18217.98|0.04|0.03|N|O|1996-12-02|1997-01-16|1996-12-05|NONE|MAIL|zle carefully sauternes. quickly|
+5024|123|8|4|42|42971.04|0.03|0.06|N|O|1996-12-02|1996-12-08|1996-12-04|DELIVER IN PERSON|RAIL|tegrate. busily spec|
+5025|30|9|1|11|10230.33|0.00|0.04|N|O|1997-02-21|1997-04-16|1997-03-14|COLLECT COD|SHIP|the carefully final esc|
+5025|78|7|2|10|9780.70|0.07|0.04|N|O|1997-06-04|1997-04-29|1997-06-28|COLLECT COD|RAIL|lly silent deposits boost busily again|
+5026|96|8|1|13|12949.17|0.02|0.04|N|O|1997-12-23|1997-11-02|1998-01-03|TAKE BACK RETURN|SHIP|endencies sleep carefully alongs|
+5027|98|2|1|6|5988.54|0.04|0.05|N|O|1997-09-28|1997-11-24|1997-10-25|NONE|FOB|ar, ironic deposi|
+5027|62|3|2|39|37520.34|0.06|0.01|N|O|1997-09-09|1997-11-13|1997-09-21|TAKE BACK RETURN|FOB|ess requests! quickly regular pac|
+5027|126|5|3|32|32835.84|0.00|0.01|N|O|1997-11-13|1997-10-29|1997-11-18|TAKE BACK RETURN|RAIL|cording to|
+5027|26|7|4|37|34262.74|0.02|0.00|N|O|1997-10-05|1997-10-30|1997-10-26|NONE|REG AIR|ost slyly fluffily|
+5027|143|4|5|3|3129.42|0.03|0.06|N|O|1997-09-30|1997-11-26|1997-10-05|DELIVER IN PERSON|AIR|t the even mu|
+5027|87|8|6|25|24677.00|0.06|0.00|N|O|1997-09-16|1997-11-25|1997-10-08|TAKE BACK RETURN|RAIL|ic ideas. requests sleep fluffily am|
+5027|81|2|7|50|49054.00|0.07|0.02|N|O|1997-09-18|1997-11-07|1997-10-05|DELIVER IN PERSON|MAIL| beans dazzle according to the fluffi|
+5028|14|1|1|15|13710.15|0.07|0.07|R|F|1992-07-17|1992-07-16|1992-08-05|COLLECT COD|REG AIR|es are quickly final pains. furiously pend|
+5028|199|10|2|15|16487.85|0.03|0.07|R|F|1992-08-02|1992-07-09|1992-08-30|NONE|REG AIR|gular, bold pinto bea|
+5029|154|5|1|17|17920.55|0.02|0.01|A|F|1993-03-12|1992-12-18|1993-04-02|DELIVER IN PERSON|FOB|! packages boost blithely. furious|
+5029|97|9|2|2|1994.18|0.00|0.04|A|F|1992-11-25|1993-01-04|1992-12-20|DELIVER IN PERSON|MAIL|packages. furiously ironi|
+5030|102|3|1|22|22046.20|0.04|0.06|N|O|1998-09-01|1998-08-15|1998-09-30|TAKE BACK RETURN|TRUCK|. quickly regular foxes believe|
+5030|80|9|2|50|49004.00|0.05|0.06|N|O|1998-08-22|1998-07-25|1998-09-18|TAKE BACK RETURN|FOB|ss excuses serve bli|
+5031|50|1|1|15|14250.75|0.02|0.05|R|F|1995-04-01|1995-02-24|1995-04-12|DELIVER IN PERSON|AIR|yly pending theodolites.|
+5031|161|6|2|40|42446.40|0.10|0.04|A|F|1994-12-04|1995-01-27|1995-01-01|NONE|TRUCK|ns hang blithely across th|
+5031|154|6|3|4|4216.60|0.01|0.07|R|F|1994-12-26|1995-02-24|1995-01-11|NONE|RAIL|after the even frays: ironic, unusual th|
+5031|181|2|4|31|33516.58|0.10|0.08|R|F|1995-01-15|1995-01-08|1995-02-09|COLLECT COD|MAIL|ts across the even requests doze furiously|
+5056|48|7|1|7|6636.28|0.09|0.01|N|O|1997-04-28|1997-04-07|1997-05-15|DELIVER IN PERSON|TRUCK|rouches after the pending instruc|
+5056|197|1|2|19|20846.61|0.04|0.00|N|O|1997-03-24|1997-05-05|1997-04-23|DELIVER IN PERSON|AIR|c theodolites. ironic a|
+5056|90|1|3|23|22772.07|0.02|0.05|N|O|1997-05-12|1997-04-28|1997-05-25|NONE|SHIP|ickly regular requests cajole. depos|
+5056|87|8|4|14|13819.12|0.08|0.00|N|O|1997-06-09|1997-04-13|1997-07-06|COLLECT COD|SHIP|sts haggle carefully along the slyl|
+5057|37|3|1|38|35607.14|0.02|0.03|N|O|1997-10-24|1997-09-07|1997-10-30|TAKE BACK RETURN|MAIL|packages. stealthily bold wa|
+5057|8|1|2|45|40860.00|0.08|0.07|N|O|1997-09-20|1997-10-02|1997-10-20|NONE|FOB| asymptotes wake slyl|
+5058|193|5|1|16|17491.04|0.09|0.07|N|O|1998-07-12|1998-06-09|1998-07-15|DELIVER IN PERSON|SHIP| the special foxes |
+5059|70|5|1|5|4850.35|0.03|0.08|R|F|1993-12-23|1994-01-12|1993-12-24|TAKE BACK RETURN|FOB|ts affix slyly accordi|
+5059|123|2|2|19|19439.28|0.06|0.04|R|F|1994-03-02|1993-12-26|1994-03-14|TAKE BACK RETURN|MAIL| special ideas poach blithely qu|
+5059|77|7|3|45|43968.15|0.02|0.00|A|F|1994-01-28|1994-01-08|1994-02-18|DELIVER IN PERSON|MAIL|enly. requests doze. express, close pa|
+5060|25|8|1|27|24975.54|0.10|0.07|R|F|1992-07-23|1992-09-05|1992-08-07|COLLECT COD|SHIP|s. ironic |
+5060|32|8|2|28|26096.84|0.04|0.04|R|F|1992-09-25|1992-08-11|1992-10-09|NONE|REG AIR|c requests|
+5060|161|2|3|15|15917.40|0.06|0.01|A|F|1992-08-28|1992-08-20|1992-09-01|DELIVER IN PERSON|AIR|ular deposits sl|
+5061|165|2|1|18|19172.88|0.03|0.00|A|F|1993-10-20|1993-10-05|1993-10-28|TAKE BACK RETURN|SHIP|atelets among the ca|
+5061|198|1|2|8|8785.52|0.01|0.02|R|F|1993-09-07|1993-10-31|1993-10-04|DELIVER IN PERSON|REG AIR|regular foxes. ir|
+5061|24|5|3|26|24024.52|0.02|0.05|A|F|1993-11-07|1993-09-13|1993-11-13|NONE|REG AIR| cajole slyly. carefully spe|
+5062|101|4|1|9|9009.90|0.08|0.00|R|F|1993-01-02|1992-12-01|1993-01-20|TAKE BACK RETURN|MAIL| silent theodolites wake. c|
+5062|75|6|2|4|3900.28|0.02|0.02|R|F|1993-02-06|1992-12-14|1993-03-03|DELIVER IN PERSON|AIR|ke furiously express theodolites. |
+5062|159|10|3|50|52957.50|0.09|0.07|A|F|1992-12-25|1992-12-13|1992-12-29|TAKE BACK RETURN|MAIL| the regular, unusual pains. specia|
+5062|161|10|4|18|19100.88|0.03|0.07|R|F|1992-11-04|1992-12-25|1992-11-05|NONE|SHIP|furiously pending requests are ruthles|
+5062|194|8|5|25|27354.75|0.08|0.02|R|F|1992-12-15|1992-11-17|1993-01-01|NONE|TRUCK|uthless excuses ag|
+5063|129|10|1|31|31902.72|0.08|0.01|N|O|1997-06-02|1997-06-20|1997-06-27|NONE|RAIL|kages. ironic, ironic courts wake. carefu|
+5063|174|2|2|43|46189.31|0.04|0.08|N|O|1997-09-14|1997-07-05|1997-10-05|TAKE BACK RETURN|TRUCK|latelets might nod blithely regular requ|
+5063|167|4|3|2|2134.32|0.02|0.03|N|O|1997-06-17|1997-07-27|1997-06-24|COLLECT COD|SHIP|kly regular i|
+5063|135|6|4|18|18632.34|0.08|0.05|N|O|1997-06-02|1997-06-18|1997-06-06|TAKE BACK RETURN|RAIL|refully quiet reques|
+5063|161|8|5|1|1061.16|0.06|0.07|N|O|1997-09-03|1997-06-26|1997-10-03|NONE|FOB|ously special |
+5088|78|6|1|23|22495.61|0.06|0.06|R|F|1993-03-03|1993-03-07|1993-03-08|NONE|FOB|cording to the fluffily expr|
+5088|51|3|2|41|38993.05|0.09|0.00|R|F|1993-01-22|1993-03-07|1993-02-09|TAKE BACK RETURN|TRUCK|ing requests. |
+5088|86|7|3|36|35498.88|0.10|0.05|A|F|1993-04-16|1993-04-03|1993-05-14|NONE|TRUCK|the furiously final deposits. furiously re|
+5088|109|6|4|10|10091.00|0.04|0.05|R|F|1993-04-07|1993-02-06|1993-04-26|NONE|FOB|beans. special requests af|
+5089|158|6|1|4|4232.60|0.05|0.06|R|F|1992-09-18|1992-09-28|1992-10-13|DELIVER IN PERSON|TRUCK|nts sleep blithely |
+5089|162|3|2|20|21243.20|0.00|0.07|R|F|1992-10-10|1992-10-07|1992-11-06|COLLECT COD|RAIL| ironic accounts|
+5089|124|7|3|46|47109.52|0.03|0.04|A|F|1992-11-09|1992-10-13|1992-11-10|TAKE BACK RETURN|RAIL|above the express accounts. exc|
+5089|34|10|4|38|35493.14|0.05|0.03|R|F|1992-11-23|1992-09-11|1992-12-22|TAKE BACK RETURN|SHIP|regular instructions are|
+5090|22|3|1|22|20284.44|0.07|0.00|N|O|1997-05-10|1997-05-25|1997-05-24|TAKE BACK RETURN|TRUCK|ets integrate ironic, regul|
+5090|129|10|2|46|47339.52|0.05|0.00|N|O|1997-04-05|1997-04-14|1997-05-01|COLLECT COD|REG AIR|lose theodolites sleep blit|
+5090|2|9|3|22|19844.00|0.09|0.05|N|O|1997-07-03|1997-04-12|1997-07-26|NONE|REG AIR|ular requests su|
+5090|114|8|4|2|2028.22|0.03|0.06|N|O|1997-04-07|1997-04-23|1997-05-01|TAKE BACK RETURN|AIR|tes. slowly iro|
+5090|48|9|5|21|19908.84|0.10|0.02|N|O|1997-03-29|1997-04-24|1997-04-25|TAKE BACK RETURN|FOB|ly express accounts. slyly even r|
+5090|80|9|6|30|29402.40|0.02|0.03|N|O|1997-05-04|1997-04-14|1997-05-30|COLLECT COD|MAIL|osits nag slyly. fluffily ex|
+5091|78|6|1|50|48903.50|0.05|0.03|N|O|1998-07-21|1998-06-22|1998-07-26|COLLECT COD|REG AIR|al dependencies. r|
+5092|164|1|1|30|31924.80|0.06|0.00|N|O|1995-12-27|1995-12-08|1996-01-09|DELIVER IN PERSON|MAIL|ss, ironic deposits. furiously stea|
+5092|45|4|2|34|32131.36|0.04|0.02|N|O|1995-12-09|1995-12-26|1995-12-21|TAKE BACK RETURN|AIR|ckages nag |
+5092|140|6|3|13|13521.82|0.06|0.01|N|O|1995-11-21|1996-01-05|1995-12-19|TAKE BACK RETURN|SHIP|es detect sly|
+5092|180|1|4|14|15122.52|0.04|0.00|N|O|1996-02-20|1995-11-30|1996-03-20|DELIVER IN PERSON|REG AIR| deposits cajole furiously against the sly|
+5092|186|7|5|42|45619.56|0.01|0.02|N|O|1995-11-06|1996-01-01|1995-12-06|DELIVER IN PERSON|AIR|s use along t|
+5092|178|6|6|11|11859.87|0.03|0.03|N|O|1995-12-02|1995-12-27|1995-12-11|COLLECT COD|MAIL|ly against the slyly silen|
+5092|159|10|7|50|52957.50|0.10|0.03|N|O|1995-11-30|1996-01-14|1995-12-19|NONE|REG AIR|r platelets maintain car|
+5093|168|9|1|40|42726.40|0.05|0.01|R|F|1993-09-16|1993-11-04|1993-10-05|TAKE BACK RETURN|REG AIR|ing pinto beans. quickly bold dependenci|
+5093|74|2|2|15|14611.05|0.01|0.04|A|F|1993-12-02|1993-11-18|1994-01-01|DELIVER IN PERSON|FOB|ly among the unusual foxe|
+5093|151|9|3|31|32585.65|0.00|0.02|R|F|1993-09-22|1993-11-14|1993-09-26|TAKE BACK RETURN|REG AIR| against the|
+5093|156|1|4|37|39077.55|0.04|0.00|A|F|1993-10-26|1993-12-02|1993-10-27|NONE|TRUCK|courts. qui|
+5093|115|2|5|30|30453.30|0.06|0.05|A|F|1993-11-22|1993-11-27|1993-12-14|DELIVER IN PERSON|TRUCK|ithely ironic sheaves use fluff|
+5093|121|6|6|31|31654.72|0.01|0.08|A|F|1993-12-17|1993-11-14|1994-01-02|NONE|SHIP|he final foxes. fluffily ironic |
+5094|143|10|1|19|19819.66|0.03|0.03|R|F|1993-03-31|1993-06-12|1993-04-04|NONE|AIR|ronic foxes. furi|
+5094|108|5|2|23|23186.30|0.05|0.07|R|F|1993-06-13|1993-05-19|1993-07-06|NONE|MAIL|st furiously above the fluffily care|
+5094|92|6|3|11|10912.99|0.04|0.08|A|F|1993-06-25|1993-06-24|1993-07-18|TAKE BACK RETURN|MAIL|s cajole quickly against the furiously ex|
+5094|79|10|4|21|20560.47|0.09|0.08|R|F|1993-07-26|1993-05-03|1993-08-16|NONE|MAIL| blithely furiously final re|
+5095|65|10|1|46|44392.76|0.07|0.01|A|F|1992-06-26|1992-06-25|1992-07-05|COLLECT COD|RAIL|egular instruction|
+5095|106|3|2|2|2012.20|0.07|0.08|A|F|1992-07-09|1992-05-25|1992-07-21|DELIVER IN PERSON|REG AIR|detect car|
+5095|123|8|3|28|28647.36|0.01|0.04|A|F|1992-06-20|1992-06-27|1992-06-22|DELIVER IN PERSON|AIR| into the final courts. ca|
+5095|178|7|4|42|45283.14|0.08|0.08|R|F|1992-05-23|1992-06-01|1992-06-18|COLLECT COD|TRUCK|ccounts. packages could have t|
+5095|166|7|5|9|9595.44|0.10|0.07|R|F|1992-08-14|1992-06-23|1992-08-16|TAKE BACK RETURN|REG AIR|bold theodolites wake about the expr|
+5095|97|8|6|15|14956.35|0.01|0.06|A|F|1992-07-11|1992-07-12|1992-08-09|COLLECT COD|AIR| to the packages wake sly|
+5095|169|10|7|40|42766.40|0.05|0.02|A|F|1992-07-11|1992-06-07|1992-07-26|DELIVER IN PERSON|MAIL|carefully unusual plat|
+5120|133|4|1|28|28927.64|0.06|0.03|N|O|1996-07-20|1996-08-31|1996-08-06|NONE|RAIL| across the silent requests. caref|
+5121|184|5|1|23|24936.14|0.06|0.01|A|F|1992-05-18|1992-06-20|1992-06-02|TAKE BACK RETURN|REG AIR|even courts are blithely ironically |
+5121|111|1|2|45|45499.95|0.08|0.04|A|F|1992-08-13|1992-07-27|1992-09-12|NONE|TRUCK|pecial accounts cajole ca|
+5121|97|10|3|27|26921.43|0.08|0.07|R|F|1992-06-17|1992-06-11|1992-06-19|NONE|MAIL|ly silent theodolit|
+5121|68|7|4|10|9680.60|0.04|0.05|R|F|1992-06-08|1992-07-10|1992-07-02|TAKE BACK RETURN|FOB|e quickly according |
+5121|89|10|5|46|45497.68|0.03|0.02|R|F|1992-05-27|1992-07-19|1992-05-28|TAKE BACK RETURN|FOB|use express foxes. slyly |
+5121|1|8|6|2|1802.00|0.04|0.07|R|F|1992-08-10|1992-06-28|1992-08-11|NONE|FOB| final, regular account|
+5122|183|4|1|28|30329.04|0.03|0.00|N|O|1996-04-20|1996-03-29|1996-04-29|DELIVER IN PERSON|FOB|g the busily ironic accounts boos|
+5122|82|3|2|43|42229.44|0.09|0.03|N|O|1996-05-31|1996-04-12|1996-06-13|NONE|MAIL|ut the carefully special foxes. idle,|
+5122|45|6|3|12|11340.48|0.07|0.03|N|O|1996-04-02|1996-04-27|1996-04-10|DELIVER IN PERSON|AIR|lar instructions |
+5123|26|7|1|13|12038.26|0.08|0.07|N|O|1998-05-17|1998-03-23|1998-06-02|COLLECT COD|MAIL|regular pearls|
+5124|55|7|1|43|41067.15|0.00|0.02|N|O|1997-07-10|1997-05-13|1997-07-31|COLLECT COD|AIR|onic package|
+5124|6|3|2|41|37146.00|0.02|0.06|N|O|1997-07-05|1997-06-29|1997-07-23|DELIVER IN PERSON|RAIL|wake across the|
+5124|125|6|3|44|45105.28|0.03|0.03|N|O|1997-07-13|1997-06-26|1997-08-01|DELIVER IN PERSON|RAIL|equests. carefully unusual d|
+5124|70|9|4|36|34922.52|0.10|0.07|N|O|1997-04-20|1997-07-03|1997-05-04|TAKE BACK RETURN|AIR|r deposits ab|
+5125|6|9|1|38|34428.00|0.09|0.05|N|O|1998-03-20|1998-04-14|1998-03-22|COLLECT COD|MAIL|ily even deposits w|
+5125|160|1|2|5|5300.80|0.08|0.06|N|O|1998-04-07|1998-04-14|1998-04-29|COLLECT COD|RAIL| thinly even pack|
+5126|24|3|1|33|30492.66|0.02|0.02|R|F|1993-02-04|1992-12-23|1993-02-14|NONE|RAIL|ipliers promise furiously whithout the |
+5126|101|6|2|43|43047.30|0.09|0.04|R|F|1993-01-07|1992-12-19|1993-01-16|COLLECT COD|MAIL|e silently. ironic, unusual accounts|
+5126|78|8|3|23|22495.61|0.08|0.01|R|F|1993-01-02|1993-01-02|1993-01-05|COLLECT COD|TRUCK|egular, blithe packages.|
+5127|19|3|1|33|30327.33|0.08|0.04|N|O|1997-03-25|1997-03-02|1997-04-04|NONE|SHIP| bold deposits use carefully a|
+5127|32|8|2|20|18640.60|0.01|0.03|N|O|1997-05-11|1997-02-26|1997-05-12|TAKE BACK RETURN|SHIP|dolites about the final platelets w|
+5152|105|2|1|9|9045.90|0.04|0.03|N|O|1997-04-11|1997-02-11|1997-04-18|COLLECT COD|AIR| cajole furiously alongside of the bo|
+5152|134|10|2|50|51706.50|0.04|0.04|N|O|1997-03-10|1997-02-04|1997-03-15|COLLECT COD|FOB| the final deposits. slyly ironic warth|
+5153|35|1|1|42|39271.26|0.03|0.01|N|O|1995-10-03|1995-11-09|1995-10-11|COLLECT COD|RAIL|re thinly. ironic|
+5153|53|5|2|14|13342.70|0.05|0.05|N|O|1995-11-29|1995-10-21|1995-12-08|TAKE BACK RETURN|TRUCK| slyly daring pinto beans lose blithely fi|
+5153|68|7|3|30|29041.80|0.09|0.01|N|O|1995-11-10|1995-11-14|1995-11-16|DELIVER IN PERSON|AIR|beans sleep bl|
+5153|173|2|4|32|34341.44|0.10|0.08|N|O|1995-12-05|1995-09-25|1996-01-03|DELIVER IN PERSON|MAIL|egular deposits. ironi|
+5153|112|2|5|36|36435.96|0.01|0.03|N|O|1995-12-15|1995-11-08|1995-12-30|COLLECT COD|TRUCK| ironic instru|
+5153|136|2|6|42|43517.46|0.00|0.03|N|O|1995-10-19|1995-11-23|1995-11-06|TAKE BACK RETURN|RAIL|ickly even deposi|
+5154|190|1|1|11|11992.09|0.02|0.05|N|O|1997-08-06|1997-06-30|1997-09-04|NONE|RAIL|luffily bold foxes. final|
+5154|144|5|2|15|15662.10|0.07|0.08|N|O|1997-06-23|1997-07-11|1997-07-11|NONE|AIR|even packages. packages use|
+5155|48|9|1|1|948.04|0.00|0.00|A|F|1994-07-03|1994-08-11|1994-07-29|COLLECT COD|FOB|oze slyly after the silent, regular idea|
+5155|188|9|2|5|5440.90|0.08|0.02|A|F|1994-06-30|1994-08-13|1994-07-15|TAKE BACK RETURN|AIR|ole blithely slyly ironic |
+5155|106|3|3|28|28170.80|0.05|0.02|R|F|1994-07-01|1994-07-19|1994-07-18|COLLECT COD|REG AIR|s cajole. accounts wake. thinly quiet pla|
+5155|79|7|4|39|38183.73|0.09|0.06|A|F|1994-08-25|1994-09-01|1994-09-18|COLLECT COD|TRUCK|l dolphins nag caref|
+5156|117|4|1|21|21359.31|0.06|0.03|N|O|1997-01-01|1997-01-30|1997-01-11|TAKE BACK RETURN|TRUCK|ts detect against the furiously reg|
+5156|148|1|2|36|37733.04|0.04|0.07|N|O|1997-02-12|1996-12-10|1997-03-13|TAKE BACK RETURN|REG AIR| slyly even orbi|
+5157|55|7|1|35|33426.75|0.06|0.08|N|O|1997-07-28|1997-09-30|1997-08-15|TAKE BACK RETURN|REG AIR|to the furiously sil|
+5157|138|9|2|18|18686.34|0.10|0.04|N|O|1997-09-06|1997-10-03|1997-09-19|COLLECT COD|MAIL|y bold deposits nag blithely. final reque|
+5157|167|8|3|15|16007.40|0.09|0.00|N|O|1997-07-27|1997-08-30|1997-08-08|DELIVER IN PERSON|RAIL|cajole. spec|
+5157|59|7|4|25|23976.25|0.00|0.03|N|O|1997-08-24|1997-09-23|1997-08-28|COLLECT COD|REG AIR| packages detect. even requests against th|
+5157|149|8|5|40|41965.60|0.09|0.06|N|O|1997-08-11|1997-08-28|1997-09-01|TAKE BACK RETURN|FOB|ial packages according to |
+5157|150|9|6|26|27303.90|0.10|0.01|N|O|1997-07-28|1997-08-22|1997-08-22|NONE|FOB|nto beans cajole car|
+5157|49|8|7|12|11388.48|0.10|0.08|N|O|1997-10-19|1997-08-07|1997-10-26|NONE|FOB|es. busily |
+5158|45|4|1|43|40636.72|0.10|0.04|N|O|1997-04-10|1997-03-06|1997-04-15|DELIVER IN PERSON|AIR|nusual platelets. slyly even foxes cajole |
+5158|85|6|2|18|17731.44|0.04|0.04|N|O|1997-04-30|1997-03-28|1997-05-12|COLLECT COD|REG AIR|hely regular pa|
+5158|142|9|3|41|42727.74|0.05|0.05|N|O|1997-02-25|1997-03-19|1997-03-03|COLLECT COD|AIR|deposits. quickly special |
+5158|131|7|4|49|50525.37|0.05|0.01|N|O|1997-04-10|1997-03-21|1997-04-30|NONE|REG AIR|r requests sleep q|
+5158|119|9|5|20|20382.20|0.01|0.04|N|O|1997-02-03|1997-02-20|1997-02-08|TAKE BACK RETURN|AIR|latelets use accordin|
+5158|88|9|6|39|38535.12|0.08|0.04|N|O|1997-05-15|1997-04-04|1997-06-02|DELIVER IN PERSON|FOB|lithely fina|
+5158|91|5|7|38|37661.42|0.10|0.05|N|O|1997-05-09|1997-03-03|1997-06-04|NONE|SHIP|uffily regular ac|
+5159|124|7|1|39|39940.68|0.06|0.07|N|O|1996-12-17|1996-12-08|1997-01-10|COLLECT COD|MAIL|re furiously after the pending dolphin|
+5159|17|1|2|46|42182.46|0.01|0.01|N|O|1996-12-15|1996-12-07|1996-12-30|DELIVER IN PERSON|SHIP|s kindle slyly carefully regular|
+5159|152|4|3|22|23147.30|0.01|0.02|N|O|1996-11-06|1996-11-04|1996-11-15|TAKE BACK RETURN|SHIP|he furiously sile|
+5159|52|3|4|5|4760.25|0.10|0.00|N|O|1996-11-25|1996-12-19|1996-12-25|TAKE BACK RETURN|FOB|nal deposits. pending, ironic ideas grow|
+5159|198|10|5|36|39534.84|0.06|0.01|N|O|1997-01-24|1996-11-07|1997-02-08|NONE|REG AIR|packages wake.|
+5184|153|8|1|33|34753.95|0.07|0.04|N|O|1998-08-17|1998-10-16|1998-08-24|TAKE BACK RETURN|AIR|posits. carefully express asympto|
+5184|16|6|2|47|43052.47|0.05|0.01|N|O|1998-11-02|1998-08-19|1998-11-07|COLLECT COD|TRUCK|se. carefully express pinto beans x|
+5184|88|9|3|39|38535.12|0.03|0.06|N|O|1998-10-27|1998-10-17|1998-11-19|DELIVER IN PERSON|FOB|es above the care|
+5184|176|7|4|26|27980.42|0.05|0.08|N|O|1998-11-11|1998-08-26|1998-12-01|TAKE BACK RETURN|TRUCK| packages are|
+5184|124|9|5|19|19458.28|0.06|0.03|N|O|1998-11-15|1998-10-12|1998-11-21|COLLECT COD|REG AIR|refully express platelets sleep carefull|
+5184|80|9|6|49|48023.92|0.02|0.00|N|O|1998-09-18|1998-08-28|1998-10-14|COLLECT COD|FOB|thlessly closely even reque|
+5185|197|1|1|37|40596.03|0.00|0.04|N|O|1997-08-08|1997-09-08|1997-08-14|COLLECT COD|SHIP|gainst the courts dazzle care|
+5185|25|8|2|32|29600.64|0.06|0.00|N|O|1997-08-17|1997-09-30|1997-08-24|TAKE BACK RETURN|REG AIR|ackages. slyly even requests|
+5185|196|9|3|41|44943.79|0.00|0.05|N|O|1997-10-15|1997-10-11|1997-11-02|COLLECT COD|REG AIR|ly blithe deposits. furi|
+5185|96|7|4|30|29882.70|0.09|0.04|N|O|1997-10-17|1997-09-16|1997-10-23|TAKE BACK RETURN|SHIP|ress packages are furiously|
+5185|128|9|5|8|8224.96|0.04|0.00|N|O|1997-08-30|1997-09-02|1997-09-22|COLLECT COD|REG AIR|sts around the slyly perma|
+5185|146|9|6|50|52307.00|0.03|0.04|N|O|1997-10-15|1997-10-19|1997-11-06|TAKE BACK RETURN|FOB|final platelets. ideas sleep careful|
+5186|55|10|1|38|36291.90|0.06|0.02|N|O|1996-11-23|1996-09-21|1996-12-11|DELIVER IN PERSON|MAIL|y ruthless foxes. fluffily |
+5186|91|2|2|31|30723.79|0.09|0.03|N|O|1996-10-19|1996-09-26|1996-10-25|TAKE BACK RETURN|REG AIR| accounts use furiously slyly spe|
+5186|89|10|3|26|25716.08|0.03|0.02|N|O|1996-08-08|1996-10-05|1996-08-21|DELIVER IN PERSON|FOB|capades. accounts sublate. pinto|
+5186|90|1|4|8|7920.72|0.10|0.05|N|O|1996-09-23|1996-09-29|1996-09-30|COLLECT COD|RAIL|y regular notornis k|
+5186|18|2|5|28|25704.28|0.09|0.03|N|O|1996-10-05|1996-10-27|1996-10-19|TAKE BACK RETURN|RAIL|al decoys. blit|
+5186|82|3|6|35|34372.80|0.00|0.05|N|O|1996-10-20|1996-10-12|1996-11-12|TAKE BACK RETURN|RAIL|sly silent pack|
+5186|198|10|7|44|48320.36|0.00|0.08|N|O|1996-09-23|1996-10-14|1996-10-01|NONE|TRUCK|old, final accounts cajole sl|
+5187|11|1|1|49|44639.49|0.04|0.06|N|O|1997-10-20|1997-10-12|1997-10-26|DELIVER IN PERSON|AIR|l, regular platelets instead of the foxes w|
+5187|83|4|2|1|983.08|0.10|0.08|N|O|1997-08-08|1997-08-24|1997-08-22|DELIVER IN PERSON|REG AIR|aggle never bold |
+5188|118|2|1|18|18325.98|0.04|0.03|N|O|1995-06-19|1995-05-19|1995-06-25|DELIVER IN PERSON|AIR|p according to the sometimes regu|
+5188|194|8|2|36|39390.84|0.04|0.02|A|F|1995-03-09|1995-05-16|1995-03-19|NONE|TRUCK|packages? blithely s|
+5188|148|1|3|9|9433.26|0.06|0.08|A|F|1995-05-09|1995-05-22|1995-05-19|TAKE BACK RETURN|REG AIR|r attainments are across the |
+5189|138|9|1|44|45677.72|0.02|0.06|A|F|1994-01-13|1994-02-07|1994-01-21|DELIVER IN PERSON|MAIL|y finally pendin|
+5189|16|3|2|38|34808.38|0.06|0.00|A|F|1994-03-26|1994-01-28|1994-04-20|DELIVER IN PERSON|REG AIR|ideas. idle, final deposits de|
+5189|110|5|3|4|4040.44|0.09|0.02|A|F|1993-12-21|1994-02-23|1994-01-09|DELIVER IN PERSON|REG AIR|. blithely exp|
+5189|94|7|4|49|48710.41|0.05|0.01|R|F|1994-01-22|1994-01-19|1994-02-04|TAKE BACK RETURN|SHIP| requests |
+5189|123|2|5|14|14323.68|0.02|0.03|A|F|1994-01-23|1994-01-05|1994-02-12|DELIVER IN PERSON|REG AIR|unusual packag|
+5189|17|8|6|41|37597.41|0.02|0.06|R|F|1993-12-12|1994-02-05|1994-01-09|DELIVER IN PERSON|RAIL|ial theodolites cajole slyly. slyly unus|
+5190|56|1|1|43|41110.15|0.09|0.06|A|F|1992-08-19|1992-06-10|1992-09-01|DELIVER IN PERSON|FOB|encies use fluffily unusual requests? hoc|
+5190|132|3|2|6|6192.78|0.10|0.08|A|F|1992-08-08|1992-07-14|1992-08-22|COLLECT COD|RAIL|furiously regular pinto beans. furiously i|
+5190|89|10|3|45|44508.60|0.04|0.03|A|F|1992-07-23|1992-06-16|1992-08-04|NONE|FOB|y carefully final ideas. f|
+5191|115|6|1|41|41619.51|0.00|0.08|A|F|1995-02-05|1995-02-27|1995-02-15|DELIVER IN PERSON|AIR|uests! ironic theodolites cajole care|
+5191|168|7|2|40|42726.40|0.02|0.01|A|F|1995-03-31|1995-02-21|1995-04-02|NONE|AIR|nes haggle sometimes. requests eng|
+5191|43|4|3|27|25462.08|0.07|0.05|A|F|1994-12-26|1995-01-24|1995-01-14|DELIVER IN PERSON|RAIL|tructions nag bravely within the re|
+5191|183|4|4|7|7582.26|0.01|0.04|A|F|1995-03-24|1995-01-30|1995-03-30|NONE|RAIL|eposits. express|
+5216|69|10|1|17|16474.02|0.04|0.06|N|O|1997-08-20|1997-11-07|1997-09-14|COLLECT COD|FOB|s according to the accounts bo|
+5217|80|1|1|50|49004.00|0.05|0.02|N|O|1995-12-26|1995-11-21|1996-01-24|DELIVER IN PERSON|MAIL|s. express, express accounts c|
+5217|16|7|2|23|21068.23|0.06|0.07|N|O|1996-01-18|1995-12-24|1996-02-10|COLLECT COD|RAIL|ven ideas. requests amo|
+5217|102|7|3|23|23048.30|0.03|0.02|N|O|1995-11-15|1995-12-17|1995-11-27|DELIVER IN PERSON|FOB|pending packages cajole ne|
+5217|81|2|4|47|46110.76|0.04|0.00|N|O|1995-11-24|1995-12-25|1995-11-25|COLLECT COD|AIR|ronic packages i|
+5218|83|4|1|43|42272.44|0.05|0.04|A|F|1992-08-04|1992-09-12|1992-08-17|DELIVER IN PERSON|SHIP|k theodolites. express, even id|
+5218|125|4|2|33|33828.96|0.06|0.01|R|F|1992-09-16|1992-09-30|1992-09-27|NONE|TRUCK|ronic instructi|
+5219|135|6|1|2|2070.26|0.08|0.00|N|O|1997-06-26|1997-04-29|1997-07-08|TAKE BACK RETURN|FOB| blithely according to the stea|
+5219|119|9|2|20|20382.20|0.05|0.00|N|O|1997-04-20|1997-05-26|1997-05-13|COLLECT COD|FOB|e along the ironic,|
+5220|83|4|1|27|26543.16|0.10|0.04|R|F|1992-09-21|1992-08-29|1992-10-16|DELIVER IN PERSON|RAIL|s cajole blithely furiously iron|
+5221|104|9|1|24|24098.40|0.07|0.03|N|O|1995-10-04|1995-08-11|1995-10-30|COLLECT COD|REG AIR|s pinto beans sleep. sly|
+5221|9|10|2|34|30906.00|0.01|0.05|N|O|1995-09-11|1995-07-17|1995-10-10|COLLECT COD|SHIP|eans. furio|
+5221|180|10|3|16|17282.88|0.04|0.01|N|O|1995-08-29|1995-09-06|1995-09-12|TAKE BACK RETURN|TRUCK|ending request|
+5222|151|3|1|1|1051.15|0.00|0.00|A|F|1994-08-19|1994-07-16|1994-09-08|TAKE BACK RETURN|FOB|idle requests. carefully pending pinto bean|
+5223|45|4|1|24|22680.96|0.00|0.00|A|F|1994-10-03|1994-09-20|1994-10-11|TAKE BACK RETURN|TRUCK|refully bold courts besides the regular,|
+5223|124|9|2|25|25603.00|0.09|0.02|R|F|1994-07-12|1994-08-13|1994-08-01|NONE|FOB|y express ideas impress|
+5223|6|3|3|19|17214.00|0.04|0.01|R|F|1994-10-28|1994-08-26|1994-10-31|COLLECT COD|REG AIR|ntly. furiously even excuses a|
+5223|130|9|4|40|41205.20|0.01|0.04|R|F|1994-10-01|1994-09-18|1994-10-28|COLLECT COD|SHIP|kly pending |
+5248|81|2|1|39|38262.12|0.05|0.03|N|O|1995-08-10|1995-07-04|1995-09-09|TAKE BACK RETURN|MAIL|yly even accounts. spe|
+5248|138|9|2|45|46715.85|0.00|0.06|A|F|1995-05-09|1995-07-12|1995-05-27|DELIVER IN PERSON|FOB|. bold, pending foxes h|
+5249|50|9|1|31|29451.55|0.07|0.03|A|F|1994-11-21|1994-11-19|1994-12-08|NONE|REG AIR|f the excuses. furiously fin|
+5249|31|7|2|44|40965.32|0.05|0.00|A|F|1994-12-28|1994-11-29|1994-12-29|TAKE BACK RETURN|MAIL|ole furiousl|
+5249|32|8|3|13|12116.39|0.09|0.00|R|F|1994-09-27|1994-10-20|1994-10-05|DELIVER IN PERSON|SHIP|ites. finally exp|
+5249|146|3|4|29|30338.06|0.00|0.05|A|F|1994-09-16|1994-11-03|1994-10-06|NONE|TRUCK| players. f|
+5249|158|6|5|12|12697.80|0.01|0.08|R|F|1994-12-28|1994-11-07|1995-01-15|COLLECT COD|MAIL|press depths could have to sleep carefu|
+5250|44|3|1|2|1888.08|0.08|0.04|N|O|1995-08-09|1995-10-10|1995-08-13|COLLECT COD|AIR|its. final pinto|
+5250|192|6|2|27|29489.13|0.10|0.05|N|O|1995-10-24|1995-09-03|1995-11-18|COLLECT COD|TRUCK|l forges are. furiously unusual pin|
+5251|139|10|1|36|37408.68|0.10|0.01|N|O|1995-07-16|1995-07-05|1995-07-28|DELIVER IN PERSON|FOB|slowly! bli|
+5252|141|10|1|13|13534.82|0.02|0.01|N|O|1996-03-02|1996-05-10|1996-03-11|NONE|FOB|boost fluffily across |
+5252|139|5|2|39|40526.07|0.06|0.05|N|O|1996-05-17|1996-04-23|1996-05-23|COLLECT COD|AIR|gular requests.|
+5252|195|9|3|9|9856.71|0.09|0.03|N|O|1996-05-30|1996-05-03|1996-06-26|TAKE BACK RETURN|RAIL|x. slyly special depos|
+5252|87|8|4|48|47379.84|0.01|0.06|N|O|1996-04-17|1996-03-19|1996-05-03|COLLECT COD|AIR|bold requests. furious|
+5252|68|5|5|24|23233.44|0.04|0.05|N|O|1996-05-11|1996-04-17|1996-05-12|COLLECT COD|REG AIR|posits after the fluffi|
+5252|3|10|6|41|37023.00|0.02|0.03|N|O|1996-03-16|1996-04-18|1996-03-17|NONE|TRUCK|ording to the blithely express somas sho|
+5253|31|2|1|35|32586.05|0.02|0.00|N|O|1995-07-23|1995-06-12|1995-08-03|DELIVER IN PERSON|AIR|ven deposits. careful|
+5253|150|7|2|38|39905.70|0.02|0.06|N|O|1995-08-03|1995-06-14|1995-08-27|DELIVER IN PERSON|REG AIR|onic dependencies are furiou|
+5253|14|5|3|9|8226.09|0.03|0.08|N|F|1995-06-08|1995-05-12|1995-06-23|DELIVER IN PERSON|REG AIR|lyly express deposits use furiou|
+5253|166|1|4|25|26654.00|0.04|0.03|A|F|1995-05-21|1995-06-13|1995-06-09|COLLECT COD|TRUCK|urts. even theodoli|
+5254|111|2|1|35|35388.85|0.01|0.07|A|F|1992-07-28|1992-09-05|1992-08-07|COLLECT COD|REG AIR|ntegrate carefully among the pending|
+5254|135|6|2|10|10351.30|0.05|0.04|A|F|1992-11-19|1992-10-20|1992-12-15|COLLECT COD|SHIP| accounts. silent deposit|
+5254|192|5|3|32|34950.08|0.00|0.08|A|F|1992-08-10|1992-09-21|1992-08-16|NONE|RAIL|ts impress closely furi|
+5254|163|2|4|45|47842.20|0.05|0.06|A|F|1992-11-11|1992-09-01|1992-12-07|COLLECT COD|REG AIR| wake. blithely silent excuse|
+5254|29|8|5|23|21367.46|0.02|0.06|A|F|1992-08-16|1992-09-05|1992-09-15|COLLECT COD|RAIL|lyly regular accounts. furiously pendin|
+5254|158|3|6|34|35977.10|0.09|0.02|R|F|1992-08-29|1992-10-16|1992-09-20|TAKE BACK RETURN|RAIL| furiously above the furiously |
+5254|20|7|7|9|8280.18|0.09|0.03|R|F|1992-07-29|1992-10-15|1992-08-20|TAKE BACK RETURN|REG AIR| wake blithely fluff|
+5255|131|7|1|2|2062.26|0.04|0.08|N|O|1996-09-27|1996-10-04|1996-10-04|DELIVER IN PERSON|RAIL|ajole blithely fluf|
+5255|172|10|2|30|32165.10|0.04|0.08|N|O|1996-09-20|1996-08-18|1996-10-09|TAKE BACK RETURN|AIR| to the silent requests cajole b|
+5255|130|3|3|41|42235.33|0.09|0.03|N|O|1996-08-21|1996-09-24|1996-09-05|COLLECT COD|FOB|tect blithely against t|
+5280|97|9|1|16|15953.44|0.02|0.03|N|O|1998-03-29|1998-01-28|1998-04-03|TAKE BACK RETURN|SHIP| foxes are furiously. theodoli|
+5280|176|5|2|46|49503.82|0.01|0.06|N|O|1998-01-04|1998-01-21|1998-02-03|TAKE BACK RETURN|FOB|efully carefully pen|
+5281|114|1|1|37|37522.07|0.05|0.02|N|O|1995-11-10|1996-01-31|1995-11-22|DELIVER IN PERSON|MAIL|ronic dependencies. fluffily final p|
+5281|105|2|2|38|38193.80|0.00|0.05|N|O|1996-02-17|1995-12-19|1996-02-29|NONE|RAIL|n asymptotes could wake about th|
+5281|127|2|3|23|23623.76|0.08|0.00|N|O|1995-12-30|1996-01-26|1996-01-23|COLLECT COD|REG AIR|. final theodolites cajole. ironic p|
+5281|87|8|4|48|47379.84|0.03|0.05|N|O|1996-01-31|1995-12-23|1996-02-08|TAKE BACK RETURN|REG AIR|ss the furiously |
+5281|43|10|5|33|31120.32|0.01|0.07|N|O|1996-03-01|1995-12-28|1996-03-05|COLLECT COD|RAIL|ly brave foxes. bold deposits above the |
+5282|118|2|1|36|36651.96|0.05|0.02|N|O|1998-05-20|1998-04-10|1998-06-14|DELIVER IN PERSON|TRUCK|re slyly accor|
+5282|52|10|2|32|30465.60|0.02|0.05|N|O|1998-03-01|1998-03-31|1998-03-03|NONE|FOB|onic deposits; furiou|
+5282|58|10|3|28|26825.40|0.03|0.06|N|O|1998-05-06|1998-04-24|1998-05-30|COLLECT COD|SHIP|fily final instruc|
+5283|5|2|1|20|18100.00|0.05|0.02|A|F|1994-09-16|1994-08-03|1994-10-15|TAKE BACK RETURN|TRUCK|al deposits? blithely even pinto beans|
+5283|186|7|2|1|1086.18|0.10|0.08|R|F|1994-06-20|1994-08-03|1994-07-01|COLLECT COD|FOB|deposits within the furio|
+5284|173|1|1|16|17170.72|0.04|0.02|N|O|1995-08-17|1995-08-23|1995-08-26|DELIVER IN PERSON|TRUCK|unts detect furiously even d|
+5284|44|7|2|24|22656.96|0.03|0.08|N|O|1995-10-21|1995-08-23|1995-10-27|COLLECT COD|AIR| haggle according |
+5285|193|5|1|31|33888.89|0.08|0.00|A|F|1994-04-17|1994-04-05|1994-05-09|NONE|RAIL|ubt. quickly blithe |
+5285|31|2|2|37|34448.11|0.09|0.02|R|F|1994-02-26|1994-02-18|1994-03-27|NONE|SHIP|uffily regu|
+5285|34|10|3|24|22416.72|0.02|0.04|A|F|1994-04-19|1994-04-03|1994-04-25|DELIVER IN PERSON|FOB|ess packages. quick, even deposits snooze b|
+5285|43|2|4|12|11316.48|0.05|0.06|A|F|1994-04-22|1994-04-07|1994-05-19|NONE|AIR| deposits-- quickly bold requests hag|
+5285|71|2|5|1|971.07|0.03|0.05|R|F|1994-03-14|1994-02-20|1994-04-10|COLLECT COD|TRUCK|e fluffily about the slyly special pa|
+5285|146|7|6|1|1046.14|0.06|0.01|R|F|1994-02-08|1994-04-02|1994-02-17|COLLECT COD|SHIP|ing deposits integra|
+5286|199|1|1|1|1099.19|0.01|0.07|N|O|1997-11-25|1997-11-07|1997-12-17|COLLECT COD|REG AIR|ly! furiously final pack|
+5286|97|1|2|7|6979.63|0.06|0.05|N|O|1997-10-23|1997-12-10|1997-11-20|TAKE BACK RETURN|RAIL|y express instructions sleep carefull|
+5286|16|10|3|3|2748.03|0.06|0.08|N|O|1997-12-04|1997-11-06|1997-12-09|COLLECT COD|MAIL|re fluffily|
+5286|40|6|4|6|5640.24|0.04|0.03|N|O|1997-10-15|1997-12-05|1997-11-12|COLLECT COD|RAIL|y special a|
+5286|186|7|5|38|41274.84|0.07|0.05|N|O|1997-11-29|1997-11-26|1997-12-15|TAKE BACK RETURN|SHIP|fluffily. special, ironic deposit|
+5286|138|9|6|24|24915.12|0.08|0.00|N|O|1997-09-27|1997-12-21|1997-09-30|COLLECT COD|TRUCK|s. express foxes of the|
+5287|39|10|1|32|30048.96|0.01|0.01|A|F|1994-01-29|1994-01-27|1994-02-08|NONE|RAIL|heodolites haggle caref|
+5312|61|6|1|27|25948.62|0.04|0.08|A|F|1995-04-20|1995-04-09|1995-04-25|COLLECT COD|SHIP|tructions cajol|
+5312|2|5|2|43|38786.00|0.05|0.08|A|F|1995-03-24|1995-05-07|1995-03-28|NONE|TRUCK|ly unusual|
+5313|17|1|1|34|31178.34|0.10|0.02|N|O|1997-08-07|1997-08-12|1997-08-24|DELIVER IN PERSON|FOB|ccording to the blithely final account|
+5313|13|10|2|17|15521.17|0.00|0.02|N|O|1997-09-02|1997-08-20|1997-09-07|NONE|SHIP|uests wake|
+5313|112|9|3|47|47569.17|0.06|0.08|N|O|1997-08-12|1997-08-18|1997-08-13|TAKE BACK RETURN|RAIL|pinto beans across the |
+5313|197|1|4|16|17555.04|0.08|0.00|N|O|1997-10-04|1997-08-02|1997-10-25|COLLECT COD|REG AIR|ckages wake carefully aga|
+5313|72|1|5|30|29162.10|0.06|0.08|N|O|1997-06-27|1997-07-18|1997-06-30|NONE|SHIP|nding packages use|
+5313|120|7|6|21|21422.52|0.05|0.05|N|O|1997-09-26|1997-09-02|1997-10-18|COLLECT COD|FOB|he blithely regular packages. quickly|
+5314|118|9|1|10|10181.10|0.07|0.05|N|O|1995-09-26|1995-07-24|1995-10-19|DELIVER IN PERSON|RAIL|latelets haggle final|
+5314|125|6|2|16|16401.92|0.00|0.04|N|O|1995-09-25|1995-07-08|1995-10-17|COLLECT COD|SHIP|hely unusual packages acc|
+5315|35|1|1|12|11220.36|0.08|0.06|R|F|1992-12-18|1993-01-16|1993-01-10|NONE|AIR|ccounts. furiously ironi|
+5315|179|10|2|39|42087.63|0.00|0.06|R|F|1992-11-09|1992-12-29|1992-12-07|NONE|SHIP|ly alongside of the ca|
+5316|108|1|1|29|29234.90|0.10|0.05|R|F|1994-03-28|1994-04-29|1994-04-09|DELIVER IN PERSON|REG AIR|ckly unusual foxes bo|
+5316|136|7|2|31|32120.03|0.00|0.08|A|F|1994-04-01|1994-04-21|1994-04-12|DELIVER IN PERSON|MAIL|s. deposits cajole around t|
+5317|82|3|1|29|28480.32|0.02|0.06|A|F|1994-11-28|1994-11-27|1994-12-16|COLLECT COD|FOB|oss the carefull|
+5317|171|2|2|18|19281.06|0.06|0.06|A|F|1995-01-02|1994-10-29|1995-01-16|NONE|RAIL|g to the blithely p|
+5317|120|4|3|37|37744.44|0.09|0.00|R|F|1994-09-15|1994-10-24|1994-09-23|NONE|TRUCK|totes nag theodolites. pend|
+5317|67|6|4|50|48353.00|0.09|0.01|A|F|1994-10-17|1994-10-25|1994-11-03|NONE|REG AIR|cajole furiously. accounts use quick|
+5317|95|8|5|19|18906.71|0.07|0.07|R|F|1994-12-15|1994-10-18|1994-12-27|NONE|MAIL|onic requests boost bli|
+5317|115|9|6|48|48725.28|0.01|0.03|A|F|1994-09-19|1994-11-25|1994-10-03|COLLECT COD|MAIL|ts about the packages cajole furio|
+5317|169|4|7|30|32074.80|0.07|0.07|A|F|1994-10-13|1994-10-31|1994-10-28|NONE|AIR|cross the attainments. slyly |
+5318|61|6|1|13|12493.78|0.10|0.04|R|F|1993-07-15|1993-06-25|1993-08-13|COLLECT COD|REG AIR|ly silent ideas. ideas haggle among the |
+5318|180|1|2|26|28084.68|0.00|0.04|R|F|1993-07-07|1993-05-23|1993-07-28|COLLECT COD|SHIP|al, express foxes. bold requests sleep alwa|
+5318|7|10|3|37|33559.00|0.07|0.05|A|F|1993-07-09|1993-06-22|1993-07-21|COLLECT COD|SHIP|ickly final deposi|
+5318|142|5|4|31|32306.34|0.01|0.04|R|F|1993-07-28|1993-05-06|1993-08-06|DELIVER IN PERSON|REG AIR|requests must sleep slyly quickly|
+5319|150|9|1|31|32554.65|0.04|0.07|N|O|1996-03-26|1996-03-07|1996-04-24|COLLECT COD|TRUCK|d carefully about the courts. fluffily spe|
+5319|44|3|2|39|36817.56|0.09|0.05|N|O|1996-05-17|1996-03-14|1996-06-11|NONE|TRUCK|unts. furiously silent|
+5344|19|3|1|6|5514.06|0.07|0.01|N|O|1998-08-04|1998-09-03|1998-08-11|TAKE BACK RETURN|REG AIR|ithely about the pending plate|
+5344|79|9|2|37|36225.59|0.03|0.07|N|O|1998-10-09|1998-07-26|1998-11-08|NONE|TRUCK|thely express packages|
+5344|67|8|3|26|25143.56|0.02|0.06|N|O|1998-08-27|1998-08-22|1998-09-24|NONE|AIR|furiously pending, silent multipliers.|
+5344|39|10|4|21|19719.63|0.03|0.01|N|O|1998-08-31|1998-09-06|1998-09-02|NONE|MAIL|xes. furiously even pinto beans sleep f|
+5345|83|4|1|3|2949.24|0.05|0.01|N|O|1997-12-10|1997-10-03|1998-01-05|COLLECT COD|SHIP|ites wake carefully unusual |
+5345|146|5|2|2|2092.28|0.10|0.02|N|O|1997-11-18|1997-10-12|1997-12-08|NONE|MAIL|ut the slyly specia|
+5345|192|5|3|46|50240.74|0.06|0.04|N|O|1997-10-06|1997-09-27|1997-10-18|COLLECT COD|REG AIR|slyly special deposits. fin|
+5345|114|4|4|37|37522.07|0.01|0.01|N|O|1997-11-01|1997-10-09|1997-11-26|DELIVER IN PERSON|AIR| along the ironically fina|
+5345|34|10|5|22|20548.66|0.02|0.02|N|O|1997-08-27|1997-11-22|1997-09-10|TAKE BACK RETURN|MAIL|leep slyly regular fox|
+5346|149|8|1|21|22031.94|0.07|0.08|R|F|1994-03-11|1994-03-07|1994-04-04|DELIVER IN PERSON|RAIL|integrate blithely a|
+5346|192|5|2|13|14198.47|0.04|0.04|A|F|1994-02-03|1994-02-05|1994-02-09|COLLECT COD|TRUCK|y. fluffily bold accounts grow. furio|
+5346|109|2|3|7|7063.70|0.08|0.05|A|F|1994-01-30|1994-03-26|1994-01-31|DELIVER IN PERSON|SHIP|equests use carefully care|
+5346|162|3|4|35|37175.60|0.06|0.02|A|F|1994-02-09|1994-03-01|1994-02-14|TAKE BACK RETURN|FOB|nic excuses cajole entic|
+5346|121|2|5|25|25528.00|0.05|0.06|R|F|1993-12-28|1994-03-19|1994-01-09|TAKE BACK RETURN|REG AIR|he ironic ideas are boldly slyly ironi|
+5346|33|9|6|6|5598.18|0.08|0.04|R|F|1994-03-01|1994-02-04|1994-03-09|NONE|REG AIR|escapades sleep furiously beside the |
+5346|80|9|7|41|40183.28|0.05|0.04|R|F|1994-01-10|1994-02-15|1994-01-26|TAKE BACK RETURN|REG AIR|fully close instructi|
+5347|83|4|1|48|47187.84|0.04|0.08|A|F|1995-02-25|1995-04-26|1995-03-26|NONE|SHIP|equests are slyly. blithely regu|
+5347|124|3|2|47|48133.64|0.02|0.01|N|F|1995-06-05|1995-03-29|1995-06-28|COLLECT COD|AIR|across the slyly bol|
+5347|23|2|3|34|31382.68|0.06|0.00|A|F|1995-05-18|1995-04-04|1995-06-02|DELIVER IN PERSON|SHIP| pending deposits. fluffily regular senti|
+5347|40|1|4|4|3760.16|0.06|0.03|A|F|1995-03-24|1995-04-03|1995-04-01|NONE|SHIP|ldly pending asymptotes ki|
+5347|131|2|5|21|21653.73|0.08|0.04|R|F|1995-04-01|1995-04-16|1995-04-23|NONE|SHIP|sly slyly final requests. careful|
+5347|56|1|6|6|5736.30|0.06|0.02|A|F|1995-04-11|1995-04-14|1995-05-02|NONE|TRUCK|lly unusual ideas. sl|
+5347|50|7|7|18|17100.90|0.01|0.01|N|F|1995-05-24|1995-05-07|1995-06-19|NONE|FOB|he ideas among the requests |
+5348|69|4|1|21|20350.26|0.10|0.04|N|O|1997-12-11|1997-12-24|1997-12-28|NONE|REG AIR| regular theodolites haggle car|
+5348|156|1|2|31|32740.65|0.07|0.02|N|O|1998-01-04|1997-12-09|1998-01-17|COLLECT COD|RAIL|are finally|
+5348|17|8|3|16|14672.16|0.06|0.08|N|O|1998-02-28|1997-12-25|1998-03-12|DELIVER IN PERSON|AIR|uriously thin pinto beans |
+5348|20|4|4|7|6440.14|0.04|0.00|N|O|1998-01-29|1997-12-20|1998-02-10|DELIVER IN PERSON|RAIL|even foxes. epitap|
+5348|2|5|5|37|33374.00|0.06|0.07|N|O|1997-12-01|1998-02-02|1997-12-07|NONE|FOB|y according to the carefully pending acco|
+5348|143|10|6|14|14603.96|0.06|0.05|N|O|1997-12-16|1998-01-12|1997-12-24|COLLECT COD|FOB|en pinto beans. somas cajo|
+5349|156|7|1|19|20066.85|0.06|0.01|N|O|1996-09-11|1996-11-18|1996-09-22|TAKE BACK RETURN|FOB|endencies use whithout the special |
+5349|168|3|2|14|14954.24|0.06|0.00|N|O|1996-11-07|1996-11-17|1996-11-20|TAKE BACK RETURN|TRUCK|fully regular |
+5349|4|5|3|6|5424.00|0.10|0.01|N|O|1996-12-30|1996-10-08|1997-01-01|DELIVER IN PERSON|MAIL|inal deposits affix carefully|
+5350|122|3|1|19|19420.28|0.02|0.06|R|F|1993-10-20|1993-11-15|1993-11-17|DELIVER IN PERSON|RAIL|romise slyly alongsi|
+5350|191|4|2|44|48012.36|0.04|0.06|R|F|1993-10-30|1993-11-23|1993-11-25|DELIVER IN PERSON|AIR|p above the ironic, pending dep|
+5350|54|9|3|12|11448.60|0.10|0.04|A|F|1994-01-30|1993-11-21|1994-02-15|COLLECT COD|REG AIR| cajole. even instructions haggle. blithe|
+5350|155|10|4|7|7386.05|0.08|0.00|R|F|1993-10-19|1993-12-28|1993-11-04|NONE|SHIP|alongside of th|
+5350|129|10|5|27|27786.24|0.07|0.04|A|F|1993-11-25|1993-12-27|1993-12-08|COLLECT COD|TRUCK|es. blithe theodolites haggl|
+5351|7|2|1|36|32652.00|0.06|0.05|N|O|1998-07-27|1998-07-06|1998-08-25|NONE|MAIL|ss the ironic, regular asymptotes cajole |
+5351|33|9|2|47|43852.41|0.04|0.01|N|O|1998-05-30|1998-08-08|1998-06-23|DELIVER IN PERSON|REG AIR|s. grouches cajole. sile|
+5351|106|3|3|2|2012.20|0.00|0.02|N|O|1998-05-12|1998-07-15|1998-05-24|NONE|TRUCK|g accounts wake furiously slyly even dolph|
+5376|61|6|1|42|40364.52|0.10|0.04|A|F|1994-09-20|1994-08-30|1994-09-29|TAKE BACK RETURN|REG AIR|y even asymptotes. courts are unusual pa|
+5376|91|4|2|44|43607.96|0.05|0.02|R|F|1994-08-30|1994-08-05|1994-09-07|COLLECT COD|AIR|ithe packages detect final theodolites. f|
+5376|65|6|3|18|17371.08|0.02|0.08|A|F|1994-10-29|1994-09-13|1994-11-01|COLLECT COD|MAIL| accounts boo|
+5377|79|8|1|40|39162.80|0.00|0.04|N|O|1997-05-21|1997-06-15|1997-05-26|DELIVER IN PERSON|AIR|lithely ironic theodolites are care|
+5377|30|3|2|17|15810.51|0.09|0.00|N|O|1997-07-05|1997-05-25|1997-07-22|COLLECT COD|RAIL|dencies. carefully regular re|
+5377|103|8|3|23|23071.30|0.07|0.08|N|O|1997-06-26|1997-07-13|1997-07-08|COLLECT COD|RAIL| silent wa|
+5377|104|7|4|12|12049.20|0.05|0.07|N|O|1997-05-08|1997-06-15|1997-05-15|DELIVER IN PERSON|MAIL| ironic, final|
+5377|173|3|5|27|28975.59|0.08|0.02|N|O|1997-07-11|1997-06-12|1997-08-08|TAKE BACK RETURN|MAIL|press theodolites. e|
+5378|155|3|1|39|41150.85|0.07|0.04|R|F|1992-11-25|1992-12-22|1992-12-02|COLLECT COD|AIR|ts are quickly around the|
+5378|62|9|2|46|44254.76|0.01|0.04|A|F|1993-02-17|1993-01-20|1993-02-26|COLLECT COD|REG AIR|into beans sleep. fu|
+5378|10|7|3|18|16380.18|0.02|0.03|R|F|1992-11-25|1992-12-21|1992-12-10|COLLECT COD|FOB|onic accounts was bold, |
+5379|199|1|1|40|43967.60|0.01|0.08|N|O|1995-10-01|1995-10-19|1995-10-30|COLLECT COD|MAIL|carefully final accounts haggle blithely. |
+5380|182|3|1|14|15150.52|0.10|0.01|N|O|1997-12-18|1997-12-03|1998-01-06|NONE|RAIL|final platelets.|
+5380|147|6|2|10|10471.40|0.09|0.05|N|O|1997-11-24|1998-01-10|1997-12-21|COLLECT COD|AIR|refully pending deposits. special, even t|
+5380|184|5|3|40|43367.20|0.02|0.08|N|O|1997-12-30|1997-11-27|1998-01-09|DELIVER IN PERSON|SHIP|ar asymptotes. blithely r|
+5380|66|3|4|6|5796.36|0.09|0.05|N|O|1997-11-15|1998-01-08|1997-12-11|COLLECT COD|MAIL|es. fluffily brave accounts across t|
+5380|107|8|5|48|48340.80|0.04|0.03|N|O|1997-12-01|1997-12-28|1997-12-05|DELIVER IN PERSON|FOB|encies haggle car|
+5381|188|9|1|37|40262.66|0.04|0.01|A|F|1993-04-08|1993-04-07|1993-04-12|DELIVER IN PERSON|SHIP|ly final deposits print carefully. unusua|
+5381|111|8|2|48|48533.28|0.04|0.03|R|F|1993-04-22|1993-04-17|1993-05-14|TAKE BACK RETURN|FOB|luffily spec|
+5381|192|3|3|13|14198.47|0.08|0.03|R|F|1993-05-09|1993-04-26|1993-05-25|NONE|FOB|s after the f|
+5381|168|3|4|17|18158.72|0.05|0.05|R|F|1993-05-25|1993-04-14|1993-06-17|NONE|MAIL|ckly final requests haggle qui|
+5381|63|8|5|49|47189.94|0.06|0.02|R|F|1993-05-08|1993-04-07|1993-06-03|NONE|FOB| accounts. regular, regula|
+5381|132|3|6|33|34060.29|0.10|0.00|A|F|1993-04-09|1993-04-03|1993-04-22|DELIVER IN PERSON|SHIP|ly special deposits |
+5381|44|3|7|31|29265.24|0.04|0.05|A|F|1993-04-10|1993-03-22|1993-04-13|TAKE BACK RETURN|MAIL|the carefully expre|
+5382|153|8|1|34|35807.10|0.03|0.03|R|F|1992-02-22|1992-02-18|1992-03-02|DELIVER IN PERSON|FOB|gular accounts. even accounts integrate|
+5382|55|3|2|13|12415.65|0.09|0.06|A|F|1992-01-16|1992-03-12|1992-02-06|NONE|MAIL|eodolites. final foxes |
+5382|149|10|3|3|3147.42|0.10|0.06|A|F|1992-03-22|1992-03-06|1992-04-19|TAKE BACK RETURN|AIR|efully unusua|
+5382|62|9|4|20|19241.20|0.08|0.02|A|F|1992-03-26|1992-02-17|1992-04-15|DELIVER IN PERSON|FOB|carefully regular accounts. slyly ev|
+5382|177|8|5|14|15080.38|0.02|0.02|A|F|1992-04-05|1992-04-05|1992-05-04|TAKE BACK RETURN|FOB| brave platelets. ev|
+5382|180|9|6|6|6481.08|0.02|0.01|A|F|1992-03-07|1992-04-02|1992-03-18|TAKE BACK RETURN|FOB|y final foxes by the sl|
+5382|105|2|7|48|48244.80|0.05|0.05|A|F|1992-02-14|1992-03-19|1992-02-25|DELIVER IN PERSON|REG AIR|nts integrate quickly ca|
+5383|96|7|1|12|11953.08|0.04|0.00|N|O|1995-07-02|1995-08-16|1995-08-01|TAKE BACK RETURN|AIR|y regular instructi|
+5408|102|7|1|2|2004.20|0.07|0.04|R|F|1992-08-21|1992-10-03|1992-08-28|DELIVER IN PERSON|MAIL|cross the dolphins h|
+5408|118|2|2|35|35633.85|0.04|0.05|R|F|1992-10-02|1992-10-17|1992-10-13|TAKE BACK RETURN|AIR|thely ironic requests alongside of the sl|
+5408|76|6|3|34|33186.38|0.10|0.02|A|F|1992-10-22|1992-08-25|1992-11-16|DELIVER IN PERSON|TRUCK|requests detect blithely a|
+5408|54|2|4|48|45794.40|0.04|0.05|R|F|1992-09-30|1992-08-27|1992-10-27|NONE|TRUCK|. furiously regular |
+5408|183|4|5|8|8665.44|0.03|0.07|A|F|1992-10-24|1992-09-06|1992-11-03|NONE|AIR|thely regular hocke|
+5409|194|8|1|27|29543.13|0.01|0.02|A|F|1992-02-14|1992-03-18|1992-02-23|DELIVER IN PERSON|AIR|eodolites |
+5409|104|5|2|38|38155.80|0.01|0.02|A|F|1992-03-17|1992-03-29|1992-04-13|NONE|REG AIR|onic, regular accounts! blithely even|
+5409|141|10|3|17|17699.38|0.07|0.00|A|F|1992-01-13|1992-04-05|1992-01-20|DELIVER IN PERSON|AIR|cross the sil|
+5409|1|8|4|9|8109.00|0.07|0.03|A|F|1992-02-15|1992-04-02|1992-02-28|DELIVER IN PERSON|AIR| unusual, unusual reques|
+5409|159|10|5|37|39188.55|0.06|0.04|R|F|1992-05-07|1992-02-10|1992-05-20|DELIVER IN PERSON|FOB|ously regular packages. packages|
+5409|64|3|6|14|13496.84|0.03|0.08|R|F|1992-02-14|1992-03-26|1992-02-29|DELIVER IN PERSON|AIR|osits cajole furiously|
+5410|117|8|1|48|48821.28|0.04|0.08|N|O|1998-09-27|1998-09-11|1998-10-01|TAKE BACK RETURN|AIR| about the slyly even courts. quickly regul|
+5410|105|8|2|41|41209.10|0.01|0.07|N|O|1998-08-25|1998-10-20|1998-09-01|DELIVER IN PERSON|REG AIR|sly. slyly ironic theodolites|
+5410|29|4|3|40|37160.80|0.07|0.08|N|O|1998-11-17|1998-10-02|1998-11-27|COLLECT COD|TRUCK|iously special accounts are along th|
+5410|50|7|4|8|7600.40|0.05|0.04|N|O|1998-09-12|1998-10-22|1998-09-22|DELIVER IN PERSON|TRUCK|ly. fluffily ironic platelets alon|
+5411|96|9|1|17|16933.53|0.05|0.01|N|O|1997-07-22|1997-07-14|1997-07-30|TAKE BACK RETURN|REG AIR| slyly slyly even deposits. carefully b|
+5411|113|7|2|10|10131.10|0.08|0.01|N|O|1997-07-19|1997-08-04|1997-07-26|TAKE BACK RETURN|MAIL|nding, special foxes unw|
+5411|56|7|3|5|4780.25|0.10|0.01|N|O|1997-09-12|1997-08-03|1997-09-23|DELIVER IN PERSON|FOB| bold, ironic theodo|
+5411|129|8|4|15|15436.80|0.08|0.05|N|O|1997-07-01|1997-07-15|1997-07-07|COLLECT COD|RAIL|attainments sleep slyly ironic|
+5411|4|5|5|19|17176.00|0.05|0.08|N|O|1997-05-25|1997-07-30|1997-06-19|COLLECT COD|RAIL|ial accounts according to the f|
+5412|54|9|1|2|1908.10|0.03|0.07|N|O|1998-04-14|1998-04-02|1998-04-19|TAKE BACK RETURN|REG AIR| sleep above the furiou|
+5412|66|1|2|48|46370.88|0.01|0.08|N|O|1998-02-22|1998-03-28|1998-03-18|TAKE BACK RETURN|TRUCK|s. slyly final packages cajole blithe|
+5412|74|2|3|31|30196.17|0.05|0.08|N|O|1998-03-23|1998-04-17|1998-04-10|NONE|SHIP|t the accounts detect slyly about the c|
+5412|97|10|4|26|25924.34|0.02|0.08|N|O|1998-01-22|1998-04-19|1998-02-17|NONE|AIR| the blithel|
+5413|126|7|1|48|49253.76|0.02|0.08|N|O|1998-01-25|1997-11-20|1998-02-22|COLLECT COD|SHIP| theodolites. furiously ironic instr|
+5413|142|9|2|37|38559.18|0.02|0.07|N|O|1997-12-08|1998-01-01|1997-12-13|COLLECT COD|TRUCK|usly bold instructions affix idly unusual, |
+5413|111|8|3|36|36399.96|0.02|0.07|N|O|1997-12-12|1997-11-28|1997-12-25|NONE|TRUCK|ular, regular ideas mold! final requests|
+5413|110|3|4|22|22222.42|0.02|0.08|N|O|1997-11-10|1997-11-24|1997-11-22|DELIVER IN PERSON|FOB|posits. quick|
+5413|189|10|5|5|5445.90|0.10|0.01|N|O|1997-11-28|1997-11-24|1997-12-05|NONE|RAIL|tes are al|
+5413|190|1|6|32|34886.08|0.02|0.03|N|O|1997-10-28|1998-01-03|1997-11-10|NONE|TRUCK|refully special package|
+5413|31|7|7|32|29792.96|0.06|0.07|N|O|1997-10-23|1997-12-09|1997-11-17|NONE|TRUCK|he quickly ironic ideas. slyly ironic ide|
+5414|68|9|1|40|38722.40|0.07|0.06|R|F|1993-04-07|1993-05-18|1993-04-23|COLLECT COD|AIR|ts are evenly across|
+5414|123|8|2|48|49109.76|0.06|0.07|R|F|1993-06-08|1993-05-14|1993-07-06|DELIVER IN PERSON|FOB| silent dolphins; fluffily regular tithe|
+5414|35|1|3|23|21505.69|0.10|0.00|A|F|1993-07-22|1993-05-26|1993-08-08|COLLECT COD|MAIL|e bold, express dolphins. spec|
+5414|133|4|4|15|15496.95|0.06|0.08|R|F|1993-05-18|1993-06-09|1993-05-27|DELIVER IN PERSON|REG AIR|e slyly about the carefully regula|
+5414|9|2|5|19|17271.00|0.01|0.05|R|F|1993-04-06|1993-05-12|1993-05-02|DELIVER IN PERSON|RAIL|ffily silent theodolites na|
+5414|98|1|6|28|27946.52|0.10|0.05|A|F|1993-03-27|1993-06-04|1993-04-07|TAKE BACK RETURN|SHIP|ts sleep sl|
+5415|102|5|1|44|44092.40|0.00|0.06|A|F|1992-08-19|1992-10-26|1992-09-17|TAKE BACK RETURN|TRUCK| requests. unusual theodolites sleep agains|
+5415|31|7|2|16|14896.48|0.08|0.00|A|F|1992-09-29|1992-09-12|1992-10-10|NONE|AIR|pinto beans haggle furiously|
+5415|102|7|3|6|6012.60|0.10|0.03|A|F|1992-10-28|1992-09-09|1992-11-20|COLLECT COD|RAIL|ges around the fur|
+5415|16|7|4|43|39388.43|0.01|0.02|R|F|1992-11-17|1992-09-14|1992-12-14|DELIVER IN PERSON|SHIP|yly blithely stealthy deposits. carefu|
+5415|161|6|5|11|11672.76|0.00|0.01|R|F|1992-11-22|1992-10-19|1992-12-10|DELIVER IN PERSON|SHIP|gle among t|
+5415|144|1|6|46|48030.44|0.03|0.03|R|F|1992-08-25|1992-09-10|1992-09-22|DELIVER IN PERSON|REG AIR|ve the fluffily |
+5415|153|4|7|11|11584.65|0.08|0.06|A|F|1992-08-21|1992-09-04|1992-08-23|NONE|TRUCK|unts maintain carefully unusual|
+5440|115|2|1|3|3045.33|0.02|0.08|N|O|1997-02-18|1997-02-28|1997-03-15|NONE|SHIP|y. accounts haggle along the blit|
+5441|164|1|1|3|3192.48|0.00|0.02|R|F|1994-08-12|1994-10-14|1994-09-01|TAKE BACK RETURN|REG AIR|are. unusual, |
+5441|131|2|2|49|50525.37|0.02|0.03|A|F|1994-09-23|1994-09-22|1994-10-22|NONE|FOB|ording to the furio|
+5441|144|3|3|33|34456.62|0.09|0.02|R|F|1994-10-09|1994-10-06|1994-10-30|DELIVER IN PERSON|TRUCK|ges. final instruction|
+5441|67|4|4|47|45451.82|0.07|0.08|R|F|1994-11-19|1994-10-16|1994-12-16|TAKE BACK RETURN|FOB|ounts wake slyly about the express instr|
+5442|42|5|1|16|15072.64|0.00|0.00|N|O|1998-04-12|1998-03-03|1998-05-04|TAKE BACK RETURN|RAIL|r packages. accounts haggle dependencies. f|
+5442|88|9|2|45|44463.60|0.08|0.01|N|O|1998-03-30|1998-02-24|1998-04-18|TAKE BACK RETURN|AIR|old slyly after |
+5442|61|8|3|12|11532.72|0.01|0.08|N|O|1998-04-15|1998-03-18|1998-05-05|DELIVER IN PERSON|TRUCK|fully final|
+5442|158|9|4|21|22221.15|0.07|0.06|N|O|1998-03-13|1998-02-19|1998-04-06|COLLECT COD|MAIL|ffily furiously ironic theodolites. furio|
+5442|16|7|5|25|22900.25|0.04|0.00|N|O|1998-03-29|1998-02-13|1998-04-13|TAKE BACK RETURN|REG AIR|ake furiously. slyly express th|
+5442|144|3|6|26|27147.64|0.08|0.07|N|O|1998-03-21|1998-03-21|1998-03-25|TAKE BACK RETURN|AIR|have to sleep furiously bold ideas. blith|
+5443|178|9|1|14|15094.38|0.02|0.00|N|O|1996-10-27|1996-11-11|1996-11-21|DELIVER IN PERSON|RAIL|s after the regular, regular deposits hag|
+5443|72|3|2|39|37910.73|0.03|0.07|N|O|1996-11-01|1996-11-30|1996-11-19|NONE|RAIL|gage carefully across the furiously|
+5443|160|5|3|25|26504.00|0.05|0.00|N|O|1996-12-07|1997-01-08|1997-01-05|NONE|FOB|use carefully above the pinto bea|
+5443|191|4|4|6|6547.14|0.05|0.02|N|O|1996-11-17|1996-12-03|1996-11-30|TAKE BACK RETURN|AIR|p fluffily foxe|
+5443|83|4|5|40|39323.20|0.03|0.03|N|O|1997-01-28|1996-12-10|1997-02-13|NONE|FOB|n courts. special re|
+5444|186|7|1|21|22809.78|0.01|0.07|A|F|1995-04-11|1995-04-25|1995-04-21|DELIVER IN PERSON|RAIL|ar packages haggle above th|
+5444|43|6|2|40|37721.60|0.05|0.08|N|O|1995-07-09|1995-04-25|1995-07-19|COLLECT COD|TRUCK|ously bold ideas. instructions wake slyl|
+5444|150|9|3|40|42006.00|0.08|0.01|A|F|1995-04-06|1995-05-08|1995-05-06|DELIVER IN PERSON|AIR| even packages.|
+5444|59|4|4|33|31648.65|0.05|0.04|N|O|1995-06-24|1995-04-24|1995-07-13|DELIVER IN PERSON|SHIP|ut the courts cajole blithely excuses|
+5444|171|9|5|21|22494.57|0.04|0.00|R|F|1995-05-05|1995-05-25|1995-05-29|TAKE BACK RETURN|REG AIR|aves serve sly|
+5444|20|7|6|21|19320.42|0.07|0.01|A|F|1995-03-30|1995-05-01|1995-03-31|COLLECT COD|AIR|furiously even theodolites.|
+5445|90|1|1|33|32672.97|0.08|0.07|A|F|1993-10-21|1993-10-14|1993-10-28|DELIVER IN PERSON|REG AIR|ests. final instructions|
+5445|131|2|2|12|12373.56|0.09|0.08|R|F|1993-11-02|1993-09-05|1993-11-26|COLLECT COD|FOB| slyly pending pinto beans was slyly al|
+5445|103|8|3|46|46142.60|0.04|0.07|A|F|1993-10-06|1993-09-15|1993-10-28|DELIVER IN PERSON|RAIL|old depend|
+5445|149|10|4|10|10491.40|0.08|0.06|A|F|1993-09-16|1993-10-05|1993-10-01|NONE|TRUCK|ncies abou|
+5445|13|10|5|14|12782.14|0.00|0.02|R|F|1993-11-19|1993-10-18|1993-12-07|NONE|RAIL| requests. bravely i|
+5446|190|1|1|27|29435.13|0.05|0.07|R|F|1994-07-21|1994-08-25|1994-08-17|TAKE BACK RETURN|RAIL|ously across the quic|
+5447|99|1|1|31|30971.79|0.09|0.03|N|O|1996-07-14|1996-05-07|1996-07-17|COLLECT COD|SHIP| foxes sleep. blithely unusual accounts det|
+5472|59|10|1|27|25894.35|0.09|0.06|A|F|1993-08-04|1993-07-07|1993-09-03|COLLECT COD|TRUCK|fily pending attainments. unus|
+5472|68|3|2|28|27105.68|0.00|0.03|A|F|1993-07-28|1993-05-28|1993-08-11|TAKE BACK RETURN|FOB|ffily pendin|
+5472|178|7|3|45|48517.65|0.06|0.02|R|F|1993-06-05|1993-05-14|1993-06-10|NONE|TRUCK| idle packages. furi|
+5472|184|5|4|37|40114.66|0.07|0.05|R|F|1993-06-15|1993-07-03|1993-07-09|DELIVER IN PERSON|RAIL|egrate carefully dependencies. |
+5472|75|6|5|40|39002.80|0.02|0.05|A|F|1993-04-13|1993-07-04|1993-05-04|NONE|REG AIR|e requests detect furiously. ruthlessly un|
+5472|167|2|6|39|41619.24|0.02|0.03|R|F|1993-04-18|1993-07-10|1993-05-12|TAKE BACK RETURN|MAIL|uriously carefully |
+5472|15|5|7|1|915.01|0.03|0.02|A|F|1993-04-14|1993-06-28|1993-04-16|NONE|RAIL|s use furiou|
+5473|48|5|1|9|8532.36|0.03|0.07|R|F|1992-06-03|1992-05-30|1992-06-09|TAKE BACK RETURN|AIR| excuses sleep blithely! regular dep|
+5473|70|1|2|27|26191.89|0.01|0.03|A|F|1992-04-06|1992-04-26|1992-04-29|TAKE BACK RETURN|MAIL|the deposits. warthogs wake fur|
+5473|15|5|3|33|30195.33|0.09|0.00|R|F|1992-05-18|1992-06-10|1992-06-13|TAKE BACK RETURN|MAIL|efully above the even, |
+5474|184|5|1|38|41198.84|0.01|0.08|A|F|1992-07-15|1992-07-16|1992-07-20|NONE|REG AIR| slyly beneath |
+5474|94|8|2|10|9940.90|0.06|0.00|R|F|1992-08-08|1992-08-10|1992-08-24|TAKE BACK RETURN|TRUCK|pinto bean|
+5474|48|1|3|31|29389.24|0.00|0.08|R|F|1992-08-02|1992-07-12|1992-08-04|NONE|TRUCK|the furiously express ideas. speci|
+5474|90|1|4|46|45544.14|0.03|0.04|A|F|1992-06-07|1992-07-11|1992-06-22|NONE|TRUCK|nstructions. furio|
+5475|183|4|1|10|10831.80|0.09|0.08|N|O|1996-07-19|1996-08-22|1996-07-23|COLLECT COD|AIR|ding to the deposits wake fina|
+5476|48|1|1|13|12324.52|0.01|0.04|N|O|1997-12-27|1997-12-08|1997-12-29|COLLECT COD|TRUCK|iously special ac|
+5476|20|4|2|17|15640.34|0.10|0.01|N|O|1998-02-02|1998-01-28|1998-02-14|COLLECT COD|FOB|ng dependencies until the f|
+5477|80|8|1|20|19601.60|0.03|0.01|N|O|1998-03-21|1998-02-09|1998-04-07|TAKE BACK RETURN|SHIP|platelets about the ironic|
+5477|77|7|2|21|20518.47|0.03|0.00|N|O|1998-01-28|1998-02-15|1998-02-24|TAKE BACK RETURN|SHIP|blate slyly. silent|
+5477|134|5|3|31|32058.03|0.04|0.01|N|O|1998-01-11|1998-01-30|1998-02-04|DELIVER IN PERSON|MAIL| special Tiresias cajole furiously. pending|
+5477|193|6|4|16|17491.04|0.00|0.01|N|O|1998-03-07|1998-03-12|1998-04-06|COLLECT COD|RAIL|regular, s|
+5477|96|9|5|23|22910.07|0.00|0.06|N|O|1998-01-04|1998-02-23|1998-01-24|NONE|REG AIR|telets wake blithely ab|
+5477|121|6|6|19|19401.28|0.10|0.03|N|O|1998-02-03|1998-01-30|1998-03-04|TAKE BACK RETURN|MAIL|ost carefully packages.|
+5478|8|9|1|39|35412.00|0.09|0.06|N|O|1996-08-19|1996-06-25|1996-09-08|DELIVER IN PERSON|SHIP|s. furiously |
+5478|2|5|2|47|42394.00|0.10|0.01|N|O|1996-08-15|1996-07-12|1996-08-31|NONE|RAIL| instructions; slyly even accounts hagg|
+5478|119|3|3|25|25477.75|0.09|0.07|N|O|1996-06-08|1996-07-12|1996-07-07|NONE|TRUCK|unusual, pending requests haggle accoun|
+5479|138|4|1|50|51906.50|0.02|0.02|A|F|1993-12-24|1994-02-14|1994-01-18|DELIVER IN PERSON|MAIL|ironic gifts. even dependencies sno|
+5479|104|5|2|19|19077.90|0.05|0.03|A|F|1994-01-22|1994-03-07|1994-02-11|DELIVER IN PERSON|SHIP|arefully bo|
+5504|68|5|1|4|3872.24|0.10|0.07|A|F|1993-04-30|1993-03-01|1993-05-22|DELIVER IN PERSON|AIR|into beans boost. |
+5504|177|8|2|7|7540.19|0.03|0.05|R|F|1993-04-25|1993-03-15|1993-05-06|NONE|TRUCK|packages detect furiously express reques|
+5504|160|2|3|29|30744.64|0.05|0.03|A|F|1993-01-28|1993-02-13|1993-02-27|NONE|SHIP|ajole carefully. care|
+5505|25|8|1|43|39775.86|0.07|0.01|N|O|1997-12-30|1997-11-28|1998-01-09|TAKE BACK RETURN|TRUCK|y alongside of the special requests.|
+5505|182|3|2|33|35711.94|0.05|0.08|N|O|1998-01-11|1997-11-11|1998-01-30|TAKE BACK RETURN|AIR|ithely unusual excuses integrat|
+5505|155|10|3|10|10551.50|0.06|0.01|N|O|1997-10-28|1997-11-27|1997-10-29|DELIVER IN PERSON|AIR| furiously special asym|
+5505|40|1|4|18|16920.72|0.04|0.04|N|O|1997-10-25|1997-12-12|1997-10-30|TAKE BACK RETURN|RAIL| to the quickly express pac|
+5505|162|9|5|46|48859.36|0.05|0.00|N|O|1998-01-06|1997-11-04|1998-02-04|TAKE BACK RETURN|SHIP|usly ironic dependencies haggle across |
+5506|140|1|1|2|2080.28|0.00|0.03|R|F|1994-02-04|1994-01-13|1994-02-17|COLLECT COD|MAIL|onic theodolites are fluffil|
+5506|160|1|2|6|6360.96|0.07|0.06|R|F|1994-02-21|1994-01-30|1994-02-27|DELIVER IN PERSON|MAIL|hely according to the furiously unusua|
+5507|10|5|1|23|20930.23|0.05|0.04|N|O|1998-09-04|1998-07-04|1998-09-18|TAKE BACK RETURN|AIR|ously slow packages poach whithout the|
+5507|138|9|2|48|49830.24|0.03|0.01|N|O|1998-08-03|1998-08-10|1998-08-24|DELIVER IN PERSON|AIR|yly idle deposits. final, final fox|
+5507|45|2|3|4|3780.16|0.04|0.06|N|O|1998-06-06|1998-07-02|1998-06-27|TAKE BACK RETURN|RAIL|into beans are|
+5507|67|6|4|22|21275.32|0.07|0.01|N|O|1998-07-08|1998-08-10|1998-07-22|DELIVER IN PERSON|TRUCK|gular ideas. carefully unu|
+5507|132|3|5|48|49542.24|0.06|0.01|N|O|1998-07-21|1998-07-15|1998-07-31|DELIVER IN PERSON|SHIP|uriously regular acc|
+5508|117|7|1|4|4068.44|0.10|0.04|N|O|1996-09-01|1996-08-02|1996-09-17|COLLECT COD|AIR|fluffily about the even |
+5509|197|10|1|3|3291.57|0.03|0.02|A|F|1994-06-14|1994-05-11|1994-06-17|NONE|SHIP| quickly fin|
+5509|99|3|2|17|16984.53|0.03|0.07|R|F|1994-07-01|1994-06-30|1994-07-31|COLLECT COD|AIR|ccounts wake ar|
+5509|93|7|3|30|29792.70|0.04|0.04|A|F|1994-07-23|1994-06-01|1994-08-08|NONE|AIR|counts haggle pinto beans. furiously |
+5509|100|3|4|45|45004.50|0.00|0.07|A|F|1994-07-24|1994-05-28|1994-08-20|COLLECT COD|AIR|counts sleep. f|
+5509|156|8|5|35|36965.25|0.04|0.03|A|F|1994-04-17|1994-06-29|1994-04-24|COLLECT COD|RAIL|c accounts. ca|
+5510|16|6|1|8|7328.08|0.01|0.01|A|F|1993-03-16|1993-03-29|1993-03-24|DELIVER IN PERSON|FOB|n packages boost sly|
+5510|20|10|2|46|42320.92|0.02|0.07|A|F|1993-03-12|1993-02-09|1993-03-19|NONE|TRUCK|silent packages cajole doggedly regular |
+5510|162|3|3|47|49921.52|0.03|0.01|A|F|1993-01-20|1993-03-25|1993-02-15|DELIVER IN PERSON|SHIP|riously even requests. slyly bold accou|
+5510|24|7|4|29|26796.58|0.09|0.08|A|F|1993-02-28|1993-03-28|1993-03-12|COLLECT COD|AIR|lithely fluffily ironic req|
+5511|165|4|1|16|17042.56|0.10|0.05|A|F|1995-02-02|1995-01-06|1995-02-19|TAKE BACK RETURN|RAIL|thely bold theodolites |
+5511|165|10|2|31|33019.96|0.09|0.01|A|F|1995-02-23|1995-01-21|1995-03-02|COLLECT COD|REG AIR|gular excuses. fluffily even pinto beans c|
+5511|128|3|3|49|50377.88|0.05|0.05|R|F|1994-12-21|1995-01-27|1994-12-26|NONE|REG AIR|bout the requests. theodolites |
+5511|122|7|4|4|4088.48|0.08|0.02|R|F|1994-12-28|1995-01-16|1995-01-24|TAKE BACK RETURN|RAIL|lphins. carefully blithe de|
+5511|9|2|5|23|20907.00|0.10|0.07|A|F|1995-03-11|1995-01-21|1995-03-27|TAKE BACK RETURN|TRUCK|ing dugouts |
+5511|188|9|6|5|5440.90|0.08|0.05|R|F|1994-12-29|1995-01-16|1995-01-24|DELIVER IN PERSON|MAIL|al theodolites. blithely final de|
+5511|143|2|7|23|23992.22|0.02|0.07|R|F|1995-02-03|1995-01-05|1995-02-18|COLLECT COD|REG AIR|ully deposits. warthogs hagg|
+5536|90|1|1|14|13861.26|0.08|0.06|N|O|1998-05-18|1998-05-08|1998-06-05|COLLECT COD|MAIL|instructions sleep |
+5536|62|1|2|20|19241.20|0.08|0.04|N|O|1998-05-08|1998-05-10|1998-05-31|DELIVER IN PERSON|REG AIR|equests mo|
+5536|197|9|3|35|38401.65|0.07|0.02|N|O|1998-05-19|1998-06-08|1998-06-05|NONE|MAIL|c, final theo|
+5536|9|10|4|30|27270.00|0.05|0.07|N|O|1998-04-15|1998-05-23|1998-05-03|NONE|FOB|arefully regular theodolites according|
+5536|141|2|5|11|11452.54|0.02|0.08|N|O|1998-03-18|1998-05-12|1998-03-28|TAKE BACK RETURN|FOB| snooze furio|
+5537|45|8|1|10|9450.40|0.05|0.08|N|O|1997-01-13|1996-12-25|1997-01-28|TAKE BACK RETURN|AIR| sleep carefully slyly bold depos|
+5537|150|9|2|15|15752.25|0.07|0.04|N|O|1997-01-13|1996-12-25|1997-01-27|COLLECT COD|AIR|eposits. permanently pending packag|
+5537|151|6|3|39|40994.85|0.03|0.00|N|O|1996-12-17|1996-11-08|1997-01-15|COLLECT COD|REG AIR| slyly bold packages are. qu|
+5537|97|1|4|38|37889.42|0.01|0.00|N|O|1996-11-06|1996-11-23|1996-11-12|TAKE BACK RETURN|MAIL|s above the carefully ironic deposits |
+5538|154|9|1|42|44274.30|0.05|0.00|A|F|1994-04-08|1994-03-17|1994-05-05|DELIVER IN PERSON|REG AIR|vely ironic accounts. furiously unusual acc|
+5538|121|2|2|4|4084.48|0.02|0.03|R|F|1994-03-21|1994-02-17|1994-04-11|TAKE BACK RETURN|REG AIR|ithely along the c|
+5538|19|3|3|38|34922.38|0.03|0.06|R|F|1994-03-17|1994-02-11|1994-04-10|TAKE BACK RETURN|FOB|ular pinto beans. silent ideas above |
+5538|78|6|4|9|8802.63|0.00|0.01|R|F|1993-12-26|1994-01-31|1994-01-03|TAKE BACK RETURN|REG AIR|encies across the blithely fina|
+5539|65|10|1|42|40532.52|0.10|0.08|A|F|1994-09-29|1994-09-17|1994-10-20|DELIVER IN PERSON|RAIL|ons across the carefully si|
+5540|181|2|1|42|45409.56|0.02|0.08|N|O|1996-11-12|1996-12-18|1996-12-05|TAKE BACK RETURN|RAIL|ss dolphins haggle |
+5540|102|3|2|2|2004.20|0.06|0.02|N|O|1996-12-12|1997-01-09|1996-12-25|DELIVER IN PERSON|MAIL|nic asymptotes could hav|
+5540|64|3|3|19|18317.14|0.01|0.03|N|O|1997-02-06|1996-11-18|1997-02-20|DELIVER IN PERSON|SHIP| slyly slyl|
+5540|72|10|4|24|23329.68|0.10|0.05|N|O|1997-01-09|1996-12-02|1997-01-23|COLLECT COD|FOB|deposits! ironic depths may engage-- b|
+5541|96|8|1|39|38847.51|0.08|0.05|N|O|1997-11-17|1997-12-27|1997-12-11|TAKE BACK RETURN|RAIL|ding theodolites haggle against the slyly |
+5542|189|10|1|6|6535.08|0.03|0.01|N|O|1996-06-14|1996-05-28|1996-07-11|DELIVER IN PERSON|TRUCK| foxes doubt. theodolites ca|
+5543|143|10|1|14|14603.96|0.02|0.03|R|F|1993-10-09|1993-12-09|1993-10-21|NONE|SHIP|ecial reque|
+5543|162|7|2|22|23367.52|0.04|0.00|A|F|1993-11-06|1993-11-02|1993-12-02|DELIVER IN PERSON|SHIP|instructions. deposits use quickly. ir|
+5543|67|6|3|3|2901.18|0.08|0.05|R|F|1993-11-18|1993-11-05|1993-12-17|NONE|FOB|ress, even |
+5543|147|10|4|8|8377.12|0.05|0.01|R|F|1993-10-28|1993-11-18|1993-11-07|NONE|SHIP|totes? iron|
+5543|80|1|5|32|31362.56|0.03|0.03|R|F|1993-10-04|1993-11-14|1993-11-03|DELIVER IN PERSON|AIR|ully around the |
+5543|184|5|6|1|1084.18|0.03|0.07|A|F|1993-10-29|1993-11-11|1993-11-23|TAKE BACK RETURN|FOB|uriously. slyly|
+5543|129|8|7|39|40135.68|0.06|0.00|R|F|1993-10-07|1993-11-15|1993-10-28|TAKE BACK RETURN|MAIL|l excuses are furiously. slyly unusual requ|
+5568|166|5|1|50|53308.00|0.05|0.05|N|O|1995-07-14|1995-09-04|1995-08-03|COLLECT COD|TRUCK|furious ide|
+5568|44|5|2|18|16992.72|0.01|0.08|N|O|1995-08-19|1995-08-18|1995-08-24|DELIVER IN PERSON|SHIP|structions haggle. carefully regular |
+5568|89|10|3|35|34617.80|0.08|0.07|N|O|1995-09-17|1995-09-04|1995-10-14|NONE|SHIP|lyly. blit|
+5569|29|4|1|25|23225.50|0.10|0.03|R|F|1993-06-29|1993-07-18|1993-07-05|TAKE BACK RETURN|TRUCK| deposits cajole above|
+5569|58|10|2|26|24909.30|0.09|0.06|A|F|1993-08-21|1993-07-22|1993-09-09|DELIVER IN PERSON|MAIL|pitaphs. ironic req|
+5569|55|3|3|48|45842.40|0.02|0.03|R|F|1993-06-16|1993-06-15|1993-07-09|COLLECT COD|SHIP|the fluffily|
+5569|147|10|4|19|19895.66|0.10|0.08|R|F|1993-07-30|1993-06-21|1993-08-13|TAKE BACK RETURN|FOB| detect ca|
+5569|59|1|5|15|14385.75|0.02|0.06|A|F|1993-06-29|1993-07-06|1993-07-05|DELIVER IN PERSON|MAIL|lithely bold requests boost fur|
+5570|161|6|1|37|39262.92|0.08|0.02|N|O|1996-08-29|1996-10-23|1996-09-11|NONE|RAIL|y ironic pin|
+5570|39|10|2|15|14085.45|0.09|0.02|N|O|1996-10-04|1996-10-05|1996-10-28|TAKE BACK RETURN|REG AIR|beans nag slyly special, regular pack|
+5570|60|1|3|29|27841.74|0.02|0.05|N|O|1996-10-12|1996-10-20|1996-11-08|TAKE BACK RETURN|SHIP|he silent, enticing requests.|
+5571|154|2|1|32|33732.80|0.05|0.01|R|F|1992-12-25|1993-03-01|1993-01-23|NONE|FOB| the blithely even packages nag q|
+5571|94|8|2|31|30816.79|0.09|0.07|R|F|1993-01-05|1993-01-18|1993-02-04|DELIVER IN PERSON|SHIP|uffily even accounts. quickly re|
+5571|92|6|3|18|17857.62|0.10|0.05|R|F|1993-03-11|1993-02-28|1993-04-03|COLLECT COD|REG AIR|uests haggle furiously pending d|
+5572|22|1|1|24|22128.48|0.08|0.08|R|F|1994-10-30|1994-10-02|1994-11-27|TAKE BACK RETURN|MAIL|ests cajole. evenly ironic exc|
+5572|172|10|2|27|28948.59|0.03|0.04|A|F|1994-08-29|1994-09-10|1994-08-30|TAKE BACK RETURN|SHIP| accounts. carefully final accoun|
+5572|87|8|3|19|18754.52|0.10|0.00|A|F|1994-08-12|1994-10-07|1994-09-01|DELIVER IN PERSON|RAIL|es. final, final requests wake blithely ag|
+5572|135|1|4|46|47615.98|0.02|0.01|R|F|1994-09-08|1994-10-14|1994-10-01|NONE|REG AIR|ully regular platelet|
+5572|24|3|5|34|31416.68|0.10|0.08|R|F|1994-10-22|1994-08-16|1994-11-08|NONE|TRUCK|asymptotes integrate. s|
+5572|101|2|6|14|14015.40|0.04|0.05|A|F|1994-11-02|1994-09-20|1994-11-03|COLLECT COD|RAIL|he fluffily express packages. fluffily fina|
+5572|26|1|7|24|22224.48|0.01|0.05|R|F|1994-09-26|1994-09-04|1994-10-22|DELIVER IN PERSON|FOB| beans. foxes sleep fluffily across th|
+5573|21|6|1|32|29472.64|0.05|0.07|N|O|1996-09-30|1996-10-25|1996-10-15|DELIVER IN PERSON|RAIL|egular depths haggl|
+5573|50|3|2|2|1900.10|0.01|0.07|N|O|1996-08-26|1996-09-29|1996-09-04|COLLECT COD|TRUCK| even foxes. specia|
+5573|11|8|3|46|41906.46|0.06|0.01|N|O|1996-11-04|1996-10-02|1996-11-15|DELIVER IN PERSON|MAIL|s haggle qu|
+5573|169|4|4|43|45973.88|0.10|0.03|N|O|1996-10-22|1996-11-03|1996-11-02|TAKE BACK RETURN|FOB| furiously pending packages against |
+5573|138|9|5|43|44639.59|0.05|0.04|N|O|1996-09-09|1996-09-24|1996-09-28|COLLECT COD|AIR| bold package|
+5574|185|6|1|46|49918.28|0.02|0.07|A|F|1992-06-20|1992-04-19|1992-07-11|NONE|FOB|arefully express requests wake furiousl|
+5574|33|4|2|21|19593.63|0.05|0.08|A|F|1992-03-22|1992-04-26|1992-04-16|TAKE BACK RETURN|TRUCK|fully final dugouts. express foxes nag |
+5574|119|6|3|27|27515.97|0.10|0.06|R|F|1992-05-08|1992-05-19|1992-06-05|TAKE BACK RETURN|REG AIR|ecial realms. furiously entici|
+5574|94|6|4|14|13917.26|0.09|0.01|R|F|1992-05-20|1992-04-09|1992-05-23|COLLECT COD|REG AIR| use slyly carefully special requests? slyl|
+5574|85|6|5|19|18716.52|0.05|0.03|A|F|1992-05-28|1992-04-24|1992-06-11|TAKE BACK RETURN|REG AIR|old deposits int|
+5575|58|10|1|7|6706.35|0.01|0.07|N|O|1995-10-01|1995-09-30|1995-10-06|NONE|FOB|s. slyly pending theodolites prin|
+5575|31|7|2|23|21413.69|0.04|0.02|N|O|1995-10-26|1995-10-09|1995-11-13|TAKE BACK RETURN|AIR|enticingly final requests. ironically|
+5575|63|8|3|16|15408.96|0.00|0.07|N|O|1995-08-17|1995-10-14|1995-08-30|NONE|RAIL|jole boldly beyond the final as|
+5575|110|1|4|7|7070.77|0.01|0.04|N|O|1995-10-15|1995-09-14|1995-10-18|DELIVER IN PERSON|RAIL|special requests. final, final |
+5600|187|8|1|34|36964.12|0.02|0.00|N|O|1997-03-22|1997-04-05|1997-04-09|TAKE BACK RETURN|MAIL|ly above the stealthy ideas. permane|
+5600|8|5|2|19|17252.00|0.00|0.01|N|O|1997-04-10|1997-03-24|1997-04-16|TAKE BACK RETURN|TRUCK|dencies. carefully p|
+5601|38|4|1|29|27202.87|0.09|0.04|A|F|1992-04-06|1992-02-24|1992-04-29|DELIVER IN PERSON|TRUCK| ironic ideas. final|
+5601|164|1|2|45|47887.20|0.10|0.07|A|F|1992-03-25|1992-04-03|1992-04-04|TAKE BACK RETURN|MAIL|ts-- blithely final accounts cajole. carefu|
+5601|73|4|3|38|36976.66|0.07|0.00|A|F|1992-01-08|1992-03-01|1992-01-09|TAKE BACK RETURN|REG AIR|ter the evenly final deposit|
+5601|148|5|4|12|12577.68|0.03|0.01|A|F|1992-02-27|1992-03-16|1992-03-27|COLLECT COD|TRUCK|ep carefully a|
+5602|176|4|1|9|9685.53|0.08|0.03|N|O|1997-10-14|1997-09-14|1997-11-11|COLLECT COD|FOB|lar foxes; quickly ironic ac|
+5602|62|7|2|31|29823.86|0.04|0.08|N|O|1997-09-04|1997-10-24|1997-09-07|NONE|TRUCK|rate fluffily regular platelets. blithel|
+5602|68|5|3|30|29041.80|0.04|0.00|N|O|1997-09-20|1997-10-25|1997-10-12|DELIVER IN PERSON|FOB|e slyly even packages. careful|
+5603|98|2|1|50|49904.50|0.03|0.02|A|F|1992-10-06|1992-08-20|1992-10-08|COLLECT COD|SHIP|final theodolites accor|
+5603|116|6|2|49|49789.39|0.06|0.05|A|F|1992-06-24|1992-07-28|1992-07-01|DELIVER IN PERSON|FOB|fully silent requests. carefully fin|
+5603|32|8|3|49|45669.47|0.00|0.02|R|F|1992-10-07|1992-07-21|1992-10-10|DELIVER IN PERSON|TRUCK|nic, pending dependencies print|
+5604|136|7|1|44|45589.72|0.05|0.01|N|O|1998-08-06|1998-07-08|1998-09-04|NONE|RAIL|efully ironi|
+5604|136|2|2|49|50770.37|0.10|0.00|N|O|1998-05-02|1998-07-07|1998-05-20|NONE|FOB|ove the regula|
+5604|78|8|3|10|9780.70|0.07|0.05|N|O|1998-08-03|1998-06-23|1998-08-04|COLLECT COD|SHIP|ly final realms wake blit|
+5605|87|8|1|50|49354.00|0.08|0.05|N|O|1996-08-26|1996-10-15|1996-09-04|TAKE BACK RETURN|RAIL|instructions sleep carefully ironic req|
+5605|151|2|2|7|7358.05|0.06|0.01|N|O|1996-12-13|1996-10-13|1996-12-15|TAKE BACK RETURN|FOB|lowly special courts nag among the furi|
+5605|173|2|3|3|3219.51|0.01|0.02|N|O|1996-09-01|1996-10-02|1996-09-20|TAKE BACK RETURN|AIR|posits. accounts boost. t|
+5605|55|3|4|45|42977.25|0.00|0.01|N|O|1996-09-05|1996-10-04|1996-09-13|COLLECT COD|FOB|ly unusual instructions. carefully ironic p|
+5605|70|7|5|39|37832.73|0.00|0.08|N|O|1996-12-13|1996-11-03|1996-12-24|DELIVER IN PERSON|REG AIR|cial deposits. theodolites w|
+5605|166|7|6|29|30918.64|0.08|0.08|N|O|1996-09-19|1996-10-22|1996-10-06|DELIVER IN PERSON|SHIP| quickly. quickly pending sen|
+5606|174|5|1|47|50485.99|0.10|0.04|N|O|1996-12-23|1997-01-31|1997-01-20|DELIVER IN PERSON|REG AIR|carefully final foxes. pending, final|
+5606|92|3|2|34|33731.06|0.09|0.06|N|O|1997-02-23|1997-02-08|1997-03-09|TAKE BACK RETURN|REG AIR|uses. slyly final |
+5606|127|8|3|46|47247.52|0.04|0.00|N|O|1997-03-11|1997-01-13|1997-03-23|DELIVER IN PERSON|REG AIR|ter the ironic accounts. even, ironic depos|
+5606|82|3|4|30|29462.40|0.08|0.04|N|O|1997-02-06|1997-01-26|1997-02-16|DELIVER IN PERSON|REG AIR| nag always. blithely express packages |
+5606|7|2|5|25|22675.00|0.06|0.00|N|O|1996-12-25|1997-01-12|1997-01-11|TAKE BACK RETURN|AIR|breach about the furiously bold |
+5606|154|5|6|3|3162.45|0.04|0.06|N|O|1997-01-11|1997-01-04|1997-02-08|COLLECT COD|AIR| sauternes. asympto|
+5606|74|5|7|46|44807.22|0.07|0.01|N|O|1997-02-01|1997-01-31|1997-02-15|DELIVER IN PERSON|TRUCK|ow requests wake around the regular accoun|
+5607|132|8|1|23|23738.99|0.02|0.06|R|F|1992-04-17|1992-02-12|1992-04-30|DELIVER IN PERSON|MAIL|the special, final patterns |
+5632|10|3|1|48|43680.48|0.06|0.06|N|O|1996-05-08|1996-03-24|1996-06-04|TAKE BACK RETURN|FOB|unts. decoys u|
+5632|106|7|2|21|21128.10|0.02|0.08|N|O|1996-03-22|1996-03-10|1996-04-10|NONE|AIR|refully regular pinto beans. ironic reques|
+5632|67|2|3|24|23209.44|0.04|0.06|N|O|1996-03-23|1996-04-02|1996-03-30|TAKE BACK RETURN|MAIL|beans detect. quickly final i|
+5633|160|2|1|28|29684.48|0.02|0.00|N|O|1998-08-14|1998-07-24|1998-08-26|TAKE BACK RETURN|SHIP|as boost quickly. unusual pinto |
+5633|102|3|2|10|10021.00|0.09|0.04|N|O|1998-07-15|1998-08-03|1998-08-03|COLLECT COD|AIR|its cajole fluffily fluffily special pinto|
+5633|46|7|3|27|25543.08|0.03|0.02|N|O|1998-09-28|1998-07-28|1998-10-12|DELIVER IN PERSON|AIR|ructions. even ideas haggle carefully r|
+5633|164|5|4|50|53208.00|0.02|0.05|N|O|1998-07-23|1998-07-09|1998-08-21|DELIVER IN PERSON|TRUCK|ts. slyly regular |
+5633|100|2|5|48|48004.80|0.01|0.05|N|O|1998-06-24|1998-07-22|1998-07-18|DELIVER IN PERSON|TRUCK|even courts haggle slyly at the requ|
+5633|107|2|6|1|1007.10|0.02|0.03|N|O|1998-09-29|1998-08-28|1998-10-19|NONE|RAIL|thely notornis: |
+5633|11|5|7|39|35529.39|0.02|0.08|N|O|1998-07-12|1998-07-03|1998-07-13|COLLECT COD|TRUCK|ding ideas cajole furiously after|
+5634|185|6|1|26|28214.68|0.10|0.08|N|O|1996-10-29|1996-09-15|1996-11-24|COLLECT COD|REG AIR|ptotes mold qu|
+5634|175|3|2|22|23653.74|0.02|0.05|N|O|1996-09-01|1996-08-31|1996-09-05|DELIVER IN PERSON|MAIL|silently unusual foxes above the blithely|
+5634|109|6|3|16|16145.60|0.08|0.02|N|O|1996-11-15|1996-09-14|1996-12-04|NONE|AIR|ess ideas are carefully pending, even re|
+5634|182|3|4|29|31383.22|0.00|0.01|N|O|1996-08-10|1996-10-29|1996-08-11|TAKE BACK RETURN|MAIL|ely final ideas. deposits sleep. reg|
+5634|1|2|5|1|901.00|0.04|0.02|N|O|1996-10-02|1996-10-21|1996-10-27|COLLECT COD|MAIL|ctions haggle carefully. carefully clo|
+5635|83|4|1|43|42272.44|0.03|0.00|R|F|1992-10-12|1992-09-29|1992-11-01|TAKE BACK RETURN|TRUCK|cross the d|
+5635|72|3|2|5|4860.35|0.05|0.08|R|F|1992-10-02|1992-11-05|1992-10-26|TAKE BACK RETURN|REG AIR|yly along the ironic, fi|
+5635|72|1|3|12|11664.84|0.09|0.02|A|F|1992-10-18|1992-09-24|1992-11-17|NONE|REG AIR|ke slyly against the carefully final req|
+5635|8|5|4|40|36320.00|0.03|0.01|A|F|1992-09-25|1992-11-05|1992-10-11|NONE|FOB|pending foxes. regular packages|
+5635|169|10|5|38|40628.08|0.05|0.06|A|F|1992-10-09|1992-09-25|1992-10-18|NONE|MAIL|ckly pendin|
+5635|162|9|6|23|24429.68|0.05|0.04|A|F|1992-08-24|1992-11-10|1992-09-21|NONE|AIR|ily pending packages. bold,|
+5635|137|3|7|32|33188.16|0.03|0.08|R|F|1992-11-24|1992-09-20|1992-12-17|TAKE BACK RETURN|TRUCK|slyly even|
+5636|70|9|1|18|17461.26|0.05|0.03|R|F|1995-05-14|1995-05-17|1995-06-12|DELIVER IN PERSON|REG AIR|slyly express requests. furiously pen|
+5636|70|5|2|26|25221.82|0.03|0.06|A|F|1995-03-05|1995-05-16|1995-03-23|TAKE BACK RETURN|AIR| furiously final pinto beans o|
+5636|90|1|3|21|20791.89|0.03|0.03|A|F|1995-03-13|1995-05-11|1995-03-24|COLLECT COD|AIR| are furiously unusual |
+5636|109|6|4|15|15136.50|0.03|0.04|R|F|1995-04-21|1995-04-30|1995-05-05|DELIVER IN PERSON|REG AIR|efully special|
+5636|47|4|5|13|12311.52|0.10|0.03|A|F|1995-05-11|1995-04-27|1995-05-26|COLLECT COD|AIR|en, fluffy accounts amon|
+5636|12|3|6|33|30096.33|0.06|0.04|A|F|1995-03-09|1995-04-05|1995-03-23|DELIVER IN PERSON|MAIL|ding to the |
+5636|134|10|7|24|24819.12|0.10|0.05|R|F|1995-04-12|1995-03-27|1995-04-16|DELIVER IN PERSON|RAIL|counts sleep furiously b|
+5637|47|4|1|14|13258.56|0.03|0.05|N|O|1996-07-20|1996-07-26|1996-08-14|COLLECT COD|MAIL|y bold deposits wak|
+5637|172|3|2|35|37525.95|0.09|0.08|N|O|1996-08-01|1996-08-04|1996-08-20|NONE|AIR|s sleep blithely alongside of the ironic|
+5637|96|10|3|22|21913.98|0.01|0.07|N|O|1996-08-28|1996-07-30|1996-09-17|COLLECT COD|REG AIR|nding requests are ca|
+5637|66|1|4|16|15456.96|0.03|0.03|N|O|1996-09-08|1996-08-31|1996-09-29|TAKE BACK RETURN|TRUCK|d packages. express requests|
+5637|196|7|5|10|10961.90|0.01|0.00|N|O|1996-08-25|1996-08-11|1996-09-23|TAKE BACK RETURN|MAIL|ickly ironic gifts. blithely even cour|
+5637|129|4|6|27|27786.24|0.01|0.05|N|O|1996-06-27|1996-08-09|1996-07-27|DELIVER IN PERSON|REG AIR|oss the carefully express warhorses|
+5638|138|9|1|45|46715.85|0.09|0.07|A|F|1994-05-17|1994-03-09|1994-06-15|NONE|TRUCK|ar foxes. fluffily pending accounts |
+5638|168|3|2|12|12817.92|0.02|0.05|A|F|1994-02-05|1994-04-01|1994-02-25|COLLECT COD|TRUCK|n, even requests. furiously ironic not|
+5638|162|9|3|21|22305.36|0.08|0.00|A|F|1994-03-13|1994-03-27|1994-03-17|DELIVER IN PERSON|TRUCK|press courts use f|
+5639|47|10|1|11|10417.44|0.09|0.02|R|F|1994-09-18|1994-07-10|1994-10-12|TAKE BACK RETURN|SHIP|g the unusual pinto beans caj|
+5664|122|1|1|25|25553.00|0.00|0.06|N|O|1998-10-29|1998-09-23|1998-11-25|COLLECT COD|FOB|eposits: furiously ironic grouch|
+5664|173|2|2|9|9658.53|0.07|0.05|N|O|1998-07-31|1998-08-26|1998-08-12|COLLECT COD|RAIL| ironic deposits haggle furiously. re|
+5664|53|4|3|31|29544.55|0.01|0.03|N|O|1998-11-10|1998-09-12|1998-12-07|TAKE BACK RETURN|FOB|ainst the never silent request|
+5664|138|9|4|33|34258.29|0.08|0.03|N|O|1998-08-29|1998-09-17|1998-09-25|DELIVER IN PERSON|RAIL|d the final |
+5664|112|2|5|44|44532.84|0.01|0.06|N|O|1998-09-24|1998-09-26|1998-10-23|NONE|TRUCK|ang thinly bold pa|
+5664|68|5|6|34|32914.04|0.09|0.01|N|O|1998-09-10|1998-10-05|1998-09-15|COLLECT COD|RAIL|st. fluffily pending foxes na|
+5664|182|3|7|9|9739.62|0.01|0.05|N|O|1998-11-04|1998-10-15|1998-11-20|TAKE BACK RETURN|REG AIR|yly. express ideas agai|
+5665|101|2|1|32|32035.20|0.00|0.02|A|F|1993-08-11|1993-08-01|1993-09-07|NONE|AIR|f the slyly even requests! regular request|
+5665|5|8|2|14|12670.00|0.02|0.00|R|F|1993-06-29|1993-09-16|1993-07-16|DELIVER IN PERSON|AIR|- special pinto beans sleep quickly blithel|
+5665|158|9|3|41|43384.15|0.09|0.02|A|F|1993-08-23|1993-09-22|1993-09-11|COLLECT COD|REG AIR| idle ideas across |
+5665|46|9|4|47|44463.88|0.01|0.01|A|F|1993-10-06|1993-09-19|1993-11-01|NONE|RAIL|s mold fluffily. final deposits along the|
+5666|122|5|1|7|7154.84|0.09|0.08|R|F|1994-05-10|1994-04-06|1994-05-21|NONE|FOB| ideas. regular packag|
+5666|36|7|2|14|13104.42|0.08|0.01|A|F|1994-02-27|1994-04-11|1994-03-06|DELIVER IN PERSON|TRUCK|lar deposits nag against the slyly final d|
+5666|193|6|3|39|42634.41|0.00|0.01|A|F|1994-05-13|1994-04-02|1994-06-12|DELIVER IN PERSON|TRUCK|the even, final foxes. quickly iron|
+5666|131|2|4|24|24747.12|0.07|0.01|R|F|1994-02-14|1994-03-09|1994-03-06|DELIVER IN PERSON|FOB|on the carefully pending asympto|
+5666|109|10|5|36|36327.60|0.07|0.07|R|F|1994-03-15|1994-03-16|1994-03-18|COLLECT COD|TRUCK|accounts. furiousl|
+5667|145|4|1|37|38670.18|0.09|0.06|N|O|1995-09-24|1995-09-17|1995-10-03|NONE|REG AIR|s cajole blit|
+5668|4|9|1|15|13560.00|0.03|0.04|A|F|1995-04-06|1995-05-12|1995-04-17|COLLECT COD|FOB| the express, pending requests. bo|
+5669|191|2|1|7|7638.33|0.06|0.06|N|O|1996-06-19|1996-07-07|1996-07-11|COLLECT COD|SHIP|yly regular requests lose blithely. careful|
+5669|156|8|2|2|2112.30|0.06|0.07|N|O|1996-08-04|1996-06-15|1996-08-20|NONE|SHIP| blithely excuses. slyly|
+5669|158|9|3|40|42326.00|0.00|0.02|N|O|1996-08-30|1996-06-15|1996-09-07|TAKE BACK RETURN|FOB|ar accounts alongside of the final, p|
+5669|90|1|4|31|30692.79|0.04|0.05|N|O|1996-08-05|1996-06-10|1996-08-29|COLLECT COD|AIR|to beans against the regular depo|
+5669|140|6|5|30|31204.20|0.07|0.01|N|O|1996-07-14|1996-07-28|1996-08-10|TAKE BACK RETURN|TRUCK|l accounts. care|
+5670|90|1|1|27|26732.43|0.10|0.06|R|F|1993-05-09|1993-05-30|1993-06-06|TAKE BACK RETURN|REG AIR| ideas promise bli|
+5670|186|7|2|43|46705.74|0.06|0.00|A|F|1993-07-09|1993-06-03|1993-07-14|DELIVER IN PERSON|FOB|ests in place of the carefully sly depos|
+5670|7|8|3|24|21768.00|0.09|0.04|A|F|1993-07-17|1993-07-01|1993-08-03|NONE|AIR|press, express requests haggle|
+5670|142|9|4|11|11463.54|0.06|0.06|R|F|1993-07-11|1993-06-26|1993-07-24|DELIVER IN PERSON|MAIL|etect furiously among the even pin|
+5671|120|7|1|25|25503.00|0.00|0.08|N|O|1998-04-17|1998-03-28|1998-05-06|DELIVER IN PERSON|AIR|cording to the quickly final requests-- |
+5671|129|8|2|46|47339.52|0.05|0.08|N|O|1998-03-28|1998-04-22|1998-04-19|TAKE BACK RETURN|MAIL|lar pinto beans detect care|
+5671|172|10|3|13|13938.21|0.10|0.06|N|O|1998-03-02|1998-04-03|1998-03-08|TAKE BACK RETURN|TRUCK|bold theodolites about|
+5671|111|1|4|42|42466.62|0.00|0.07|N|O|1998-02-17|1998-04-24|1998-03-17|TAKE BACK RETURN|SHIP|carefully slyly special deposit|
+5671|129|4|5|13|13378.56|0.09|0.00|N|O|1998-04-24|1998-03-26|1998-04-27|NONE|REG AIR|ers according to the ironic, unusual excu|
+5671|114|1|6|30|30423.30|0.09|0.07|N|O|1998-06-06|1998-04-15|1998-07-01|DELIVER IN PERSON|TRUCK|fily ironi|
+5696|137|3|1|28|29039.64|0.03|0.06|N|O|1995-07-03|1995-06-14|1995-07-27|COLLECT COD|REG AIR| the fluffily brave pearls |
+5696|59|1|2|46|44116.30|0.01|0.00|N|O|1995-08-10|1995-07-08|1995-08-25|COLLECT COD|AIR|ter the instruct|
+5696|167|2|3|42|44820.72|0.04|0.01|N|F|1995-06-06|1995-06-11|1995-06-19|TAKE BACK RETURN|SHIP|te furious|
+5696|98|10|4|20|19961.80|0.08|0.00|N|O|1995-06-25|1995-07-18|1995-07-16|NONE|TRUCK|silent, pending ideas sleep fluffil|
+5696|124|9|5|19|19458.28|0.07|0.05|N|O|1995-08-31|1995-06-13|1995-09-10|COLLECT COD|SHIP|unusual requests sleep furiously ru|
+5696|132|8|6|37|38188.81|0.04|0.05|N|O|1995-07-21|1995-06-23|1995-08-19|NONE|RAIL| carefully expres|
+5696|102|9|7|6|6012.60|0.07|0.05|N|O|1995-08-03|1995-07-15|1995-09-01|DELIVER IN PERSON|REG AIR|n patterns lose slyly fina|
+5697|55|7|1|24|22921.20|0.10|0.07|R|F|1992-10-27|1992-11-28|1992-11-20|NONE|RAIL|uffily iro|
+5697|16|10|2|43|39388.43|0.06|0.02|R|F|1992-12-08|1992-12-03|1992-12-17|TAKE BACK RETURN|FOB|blithely reg|
+5697|56|8|3|42|40154.10|0.03|0.01|A|F|1992-12-19|1992-12-08|1993-01-03|COLLECT COD|TRUCK|inal theodolites cajole after the bli|
+5698|11|8|1|30|27330.30|0.01|0.05|A|F|1994-05-26|1994-08-16|1994-06-19|COLLECT COD|AIR|its. quickly regular foxes aro|
+5698|163|4|2|25|26579.00|0.08|0.07|R|F|1994-08-06|1994-06-21|1994-08-25|NONE|SHIP| asymptotes sleep slyly above the|
+5698|155|3|3|45|47481.75|0.03|0.01|A|F|1994-06-23|1994-08-13|1994-07-02|NONE|FOB|ng excuses. slyly express asymptotes|
+5698|58|6|4|15|14370.75|0.07|0.08|R|F|1994-06-29|1994-07-03|1994-07-02|COLLECT COD|REG AIR|ly ironic frets haggle carefully |
+5698|140|1|5|37|38485.18|0.06|0.06|A|F|1994-06-30|1994-06-23|1994-07-22|TAKE BACK RETURN|SHIP|ts. even, ironic |
+5698|188|9|6|1|1088.18|0.06|0.04|R|F|1994-05-31|1994-07-10|1994-06-03|DELIVER IN PERSON|MAIL|nts. slyly quiet pinto beans nag carefu|
+5699|2|7|1|24|21648.00|0.01|0.07|A|F|1992-10-21|1992-09-04|1992-11-04|COLLECT COD|AIR|kages. fin|
+5699|55|10|2|26|24831.30|0.06|0.06|R|F|1992-08-11|1992-09-21|1992-08-14|COLLECT COD|MAIL|y final deposits wake fluffily u|
+5699|18|2|3|48|44064.48|0.10|0.05|R|F|1992-11-23|1992-10-20|1992-11-29|DELIVER IN PERSON|TRUCK|s. carefully regul|
+5699|55|3|4|46|43932.30|0.08|0.02|A|F|1992-11-28|1992-09-23|1992-12-27|TAKE BACK RETURN|FOB|o the slyly|
+5699|28|7|5|21|19488.42|0.02|0.02|A|F|1992-10-13|1992-09-30|1992-10-19|NONE|MAIL|lyly final pla|
+5699|191|5|6|30|32735.70|0.08|0.05|R|F|1992-11-13|1992-10-01|1992-12-11|DELIVER IN PERSON|AIR| the carefully final |
+5699|129|8|7|45|46310.40|0.09|0.06|A|F|1992-09-23|1992-10-22|1992-10-04|DELIVER IN PERSON|SHIP|rmanent packages sleep across the f|
+5700|168|5|1|24|25635.84|0.09|0.00|N|O|1997-12-26|1998-01-28|1998-01-18|DELIVER IN PERSON|REG AIR|ix carefully |
+5700|123|8|2|30|30693.60|0.00|0.06|N|O|1998-04-19|1998-03-13|1998-04-27|COLLECT COD|MAIL|ly blithely final instructions. fl|
+5700|126|5|3|23|23600.76|0.03|0.05|N|O|1998-01-30|1998-01-31|1998-01-31|NONE|REG AIR| wake quickly carefully fluffy hockey|
+5701|54|2|1|17|16218.85|0.02|0.05|N|O|1997-03-27|1997-04-08|1997-04-21|DELIVER IN PERSON|RAIL|tes. quickly final a|
+5702|77|7|1|44|42991.08|0.06|0.02|R|F|1994-01-04|1993-11-25|1994-01-22|NONE|RAIL|lites. carefully final requests doze b|
+5702|86|7|2|37|36484.96|0.10|0.05|R|F|1993-12-14|1993-10-21|1994-01-08|NONE|FOB|ix slyly. regular instructions slee|
+5702|131|7|3|44|45369.72|0.00|0.02|R|F|1993-11-28|1993-12-02|1993-12-22|NONE|TRUCK|ake according to th|
+5702|63|8|4|31|29854.86|0.00|0.04|A|F|1994-01-04|1993-10-22|1994-01-26|DELIVER IN PERSON|TRUCK|pinto beans. blithely |
+5703|88|9|1|2|1976.16|0.09|0.01|R|F|1993-05-29|1993-07-26|1993-06-05|TAKE BACK RETURN|REG AIR|nts against the blithely sile|
+5728|44|1|1|47|44369.88|0.10|0.05|A|F|1994-12-13|1995-01-25|1994-12-25|TAKE BACK RETURN|MAIL|nd the bravely final deposits. final ideas|
+5728|159|1|2|40|42366.00|0.05|0.08|A|F|1995-03-28|1995-01-17|1995-04-14|TAKE BACK RETURN|SHIP|final deposits. theodolite|
+5729|143|4|1|5|5215.70|0.07|0.00|R|F|1994-11-27|1994-11-11|1994-12-23|TAKE BACK RETURN|MAIL|s. even sheaves nag courts. |
+5729|107|10|2|39|39276.90|0.10|0.00|A|F|1995-01-22|1994-11-21|1995-02-13|TAKE BACK RETURN|MAIL|. special pl|
+5729|12|3|3|50|45600.50|0.00|0.05|R|F|1994-12-09|1994-12-31|1994-12-24|TAKE BACK RETURN|AIR|ly special sentiments. car|
+5730|151|2|1|2|2102.30|0.08|0.00|N|O|1998-02-24|1998-03-15|1998-03-11|COLLECT COD|SHIP|ely ironic foxes. carefu|
+5730|200|1|2|9|9901.80|0.10|0.01|N|O|1998-03-05|1998-02-02|1998-03-28|DELIVER IN PERSON|MAIL|s lose blithely. specia|
+5731|192|6|1|13|14198.47|0.02|0.04|N|O|1997-07-30|1997-06-23|1997-08-13|COLLECT COD|RAIL|ngside of the quickly regular depos|
+5731|105|6|2|11|11056.10|0.00|0.08|N|O|1997-06-06|1997-07-08|1997-06-25|NONE|MAIL| furiously final accounts wake. d|
+5731|111|2|3|6|6066.66|0.01|0.04|N|O|1997-07-02|1997-07-01|1997-07-08|COLLECT COD|SHIP|sits integrate slyly close platelets. quick|
+5731|14|1|4|6|5484.06|0.03|0.06|N|O|1997-09-07|1997-06-20|1997-09-20|TAKE BACK RETURN|RAIL|rs. quickly regular theo|
+5731|195|6|5|19|20808.61|0.08|0.02|N|O|1997-06-29|1997-06-27|1997-07-15|NONE|REG AIR|ly unusual ideas above the |
+5732|139|5|1|26|27017.38|0.02|0.07|N|O|1997-08-18|1997-10-25|1997-09-12|TAKE BACK RETURN|TRUCK|totes cajole according to the theodolites.|
+5733|33|4|1|39|36388.17|0.01|0.07|A|F|1993-03-22|1993-05-24|1993-04-04|DELIVER IN PERSON|FOB|side of the|
+5734|183|4|1|29|31412.22|0.05|0.01|N|O|1997-12-01|1997-12-08|1997-12-23|NONE|RAIL|structions cajole final, express |
+5734|150|3|2|6|6300.90|0.07|0.00|N|O|1997-10-27|1997-12-19|1997-11-02|COLLECT COD|RAIL|s. regular platelets cajole furiously. regu|
+5734|67|8|3|10|9670.60|0.01|0.03|N|O|1997-12-28|1997-12-24|1998-01-24|DELIVER IN PERSON|TRUCK|equests; accounts above|
+5735|60|1|1|41|39362.46|0.01|0.01|R|F|1994-12-23|1995-02-10|1995-01-22|COLLECT COD|MAIL|lthily ruthless i|
+5760|1|8|1|6|5406.00|0.09|0.03|R|F|1994-07-30|1994-07-31|1994-08-16|COLLECT COD|REG AIR|ng the acco|
+5760|6|1|2|24|21744.00|0.04|0.05|A|F|1994-07-15|1994-07-04|1994-08-08|NONE|MAIL|s. bravely ironic accounts among|
+5760|148|5|3|8|8385.12|0.07|0.04|A|F|1994-09-06|1994-08-03|1994-10-06|NONE|AIR|l accounts among the carefully even de|
+5760|123|4|4|19|19439.28|0.10|0.01|R|F|1994-08-02|1994-08-02|1994-08-15|COLLECT COD|SHIP|sits nag. even, regular ideas cajole b|
+5760|166|1|5|6|6396.96|0.03|0.07|R|F|1994-06-09|1994-07-06|1994-06-16|DELIVER IN PERSON|MAIL| shall have to cajole along the |
+5761|47|6|1|41|38828.64|0.08|0.00|N|O|1998-07-31|1998-08-09|1998-08-08|TAKE BACK RETURN|TRUCK|pecial deposits. qu|
+5761|108|9|2|36|36291.60|0.00|0.07|N|O|1998-09-07|1998-09-21|1998-09-11|TAKE BACK RETURN|TRUCK| pinto beans thrash alongside of the pendi|
+5761|198|2|3|49|53811.31|0.04|0.08|N|O|1998-07-14|1998-08-20|1998-07-25|NONE|SHIP|ly bold accounts wake above the|
+5762|175|6|1|6|6451.02|0.05|0.02|N|O|1997-04-07|1997-03-25|1997-05-02|NONE|AIR|ironic dependencies doze carefu|
+5762|102|9|2|27|27056.70|0.02|0.08|N|O|1997-02-21|1997-05-08|1997-03-23|NONE|REG AIR|across the bold ideas. carefully sp|
+5762|89|10|3|40|39563.20|0.00|0.08|N|O|1997-04-30|1997-05-09|1997-05-08|COLLECT COD|SHIP|al instructions. furiousl|
+5762|133|4|4|47|48557.11|0.05|0.06|N|O|1997-03-02|1997-03-23|1997-03-19|NONE|RAIL|equests sleep after the furiously ironic pa|
+5762|25|6|5|28|25900.56|0.02|0.06|N|O|1997-02-22|1997-03-25|1997-02-24|TAKE BACK RETURN|SHIP|ic foxes among the blithely qui|
+5762|12|6|6|12|10944.12|0.00|0.06|N|O|1997-04-18|1997-04-27|1997-05-11|DELIVER IN PERSON|REG AIR|ages are abo|
+5763|131|2|1|32|32996.16|0.02|0.06|N|O|1998-07-16|1998-09-13|1998-08-02|DELIVER IN PERSON|FOB|ding instruct|
+5763|136|2|2|23|23830.99|0.09|0.04|N|O|1998-07-25|1998-09-21|1998-08-15|DELIVER IN PERSON|SHIP|re after the blithel|
+5763|13|3|3|25|22825.25|0.01|0.02|N|O|1998-10-04|1998-08-16|1998-10-09|DELIVER IN PERSON|REG AIR|inal theodolites. even re|
+5763|121|6|4|47|47992.64|0.09|0.00|N|O|1998-08-22|1998-09-22|1998-09-04|NONE|REG AIR|gle slyly. slyly final re|
+5763|123|4|5|8|8184.96|0.06|0.05|N|O|1998-09-23|1998-09-15|1998-09-27|DELIVER IN PERSON|TRUCK|foxes wake slyly. car|
+5763|190|1|6|9|9811.71|0.08|0.02|N|O|1998-09-24|1998-09-01|1998-10-02|NONE|AIR| deposits. instru|
+5764|101|2|1|28|28030.80|0.04|0.04|A|F|1993-12-07|1993-12-20|1993-12-26|TAKE BACK RETURN|RAIL|sleep furi|
+5764|200|3|2|20|22004.00|0.10|0.05|A|F|1993-10-17|1993-12-24|1993-10-18|TAKE BACK RETURN|FOB|ng to the fluffily qu|
+5764|188|9|3|4|4352.72|0.03|0.05|A|F|1993-10-25|1993-12-23|1993-11-06|DELIVER IN PERSON|AIR|ily regular courts haggle|
+5765|162|7|1|31|32926.96|0.00|0.06|A|F|1995-01-11|1995-02-13|1995-01-23|TAKE BACK RETURN|AIR|r foxes. ev|
+5765|124|9|2|29|29699.48|0.07|0.08|A|F|1994-12-29|1995-02-01|1995-01-26|NONE|RAIL|nic requests. deposits wake quickly among |
+5765|139|10|3|31|32213.03|0.05|0.01|R|F|1995-03-01|1995-01-23|1995-03-31|TAKE BACK RETURN|REG AIR|the furiou|
+5765|152|4|4|46|48398.90|0.07|0.07|R|F|1995-03-13|1995-02-12|1995-03-20|DELIVER IN PERSON|MAIL|ccounts sleep about th|
+5765|174|3|5|48|51560.16|0.09|0.02|A|F|1995-03-30|1995-01-14|1995-04-09|DELIVER IN PERSON|SHIP|theodolites integrate furiously|
+5765|83|4|6|41|40306.28|0.04|0.00|A|F|1994-12-31|1995-02-11|1995-01-17|TAKE BACK RETURN|SHIP| furiously. slyly sile|
+5765|42|5|7|21|19782.84|0.05|0.04|R|F|1995-04-05|1995-02-12|1995-05-05|COLLECT COD|TRUCK|ole furiously. quick, special dependencies |
+5766|188|9|1|1|1088.18|0.10|0.01|R|F|1994-01-16|1993-11-16|1994-01-23|NONE|MAIL|blithely regular the|
+5766|149|8|2|39|40916.46|0.02|0.07|A|F|1993-10-24|1993-12-07|1993-11-08|DELIVER IN PERSON|SHIP| furiously unusual courts. slyly final pear|
+5766|118|8|3|4|4072.44|0.08|0.08|R|F|1993-11-10|1993-10-30|1993-12-01|COLLECT COD|TRUCK|ly even requests. furiou|
+5767|167|8|1|11|11738.76|0.08|0.01|A|F|1992-06-02|1992-05-30|1992-06-08|NONE|TRUCK|instructions. carefully final accou|
+5767|69|8|2|15|14535.90|0.07|0.05|R|F|1992-06-05|1992-07-28|1992-06-08|DELIVER IN PERSON|MAIL|warthogs. carefully unusual g|
+5767|191|3|3|42|45829.98|0.06|0.01|R|F|1992-07-31|1992-06-09|1992-08-09|COLLECT COD|TRUCK| blithe deposi|
+5767|153|4|4|34|35807.10|0.06|0.01|R|F|1992-06-02|1992-06-23|1992-06-17|NONE|FOB|sits among the|
+5767|46|7|5|36|34057.44|0.03|0.00|A|F|1992-07-17|1992-06-10|1992-07-19|COLLECT COD|AIR|ake carefully. packages |
+5792|178|8|1|34|36657.78|0.08|0.07|R|F|1993-05-23|1993-06-25|1993-06-12|NONE|RAIL|requests are against t|
+5792|157|5|2|47|49686.05|0.10|0.00|A|F|1993-06-08|1993-05-10|1993-06-26|COLLECT COD|AIR|regular, ironic excuses n|
+5792|183|4|3|32|34661.76|0.05|0.08|R|F|1993-06-26|1993-05-23|1993-07-07|COLLECT COD|RAIL|s are slyly against the ev|
+5792|14|8|4|14|12796.14|0.09|0.02|A|F|1993-07-28|1993-06-17|1993-08-27|DELIVER IN PERSON|RAIL|olites print carefully|
+5792|102|9|5|31|31065.10|0.02|0.01|A|F|1993-06-17|1993-05-05|1993-07-01|COLLECT COD|TRUCK|s? furiously even instructions |
+5793|53|5|1|20|19061.00|0.05|0.03|N|O|1997-10-05|1997-09-04|1997-10-30|COLLECT COD|AIR|e carefully ex|
+5793|170|5|2|41|43876.97|0.06|0.06|N|O|1997-08-04|1997-10-10|1997-08-12|DELIVER IN PERSON|TRUCK|snooze quick|
+5793|43|4|3|8|7544.32|0.07|0.03|N|O|1997-08-16|1997-09-08|1997-08-28|COLLECT COD|AIR|al foxes l|
+5793|148|7|4|48|50310.72|0.02|0.02|N|O|1997-09-27|1997-08-23|1997-10-27|DELIVER IN PERSON|REG AIR|quickly enticing excuses use slyly abov|
+5794|158|9|1|42|44442.30|0.06|0.05|R|F|1993-06-29|1993-05-30|1993-07-28|COLLECT COD|REG AIR|he careful|
+5794|115|2|2|14|14211.54|0.09|0.02|R|F|1993-04-19|1993-07-02|1993-05-18|COLLECT COD|SHIP|uriously carefully ironic reque|
+5794|7|8|3|15|13605.00|0.09|0.06|R|F|1993-06-25|1993-06-27|1993-07-09|NONE|MAIL|blithely regular ideas. final foxes haggle |
+5794|137|3|4|47|48745.11|0.00|0.08|A|F|1993-07-16|1993-06-21|1993-08-05|TAKE BACK RETURN|REG AIR|quests. blithely final excu|
+5795|193|6|1|34|37168.46|0.09|0.05|A|F|1992-08-21|1992-07-30|1992-08-27|COLLECT COD|REG AIR|al instructions must affix along the ironic|
+5796|58|3|1|27|25867.35|0.10|0.00|N|O|1996-04-06|1996-02-29|1996-04-20|DELIVER IN PERSON|RAIL|s wake quickly aro|
+5797|61|6|1|17|16338.02|0.09|0.03|N|O|1997-12-13|1998-01-12|1997-12-23|NONE|REG AIR|the ironic, even theodoli|
+5798|127|8|1|2|2054.24|0.09|0.00|N|O|1998-05-25|1998-06-22|1998-06-09|COLLECT COD|FOB|e furiously across |
+5798|124|9|2|14|14337.68|0.06|0.05|N|O|1998-04-01|1998-06-14|1998-04-27|NONE|RAIL|he special, bold packages. carefully iron|
+5798|134|5|3|22|22750.86|0.02|0.01|N|O|1998-06-24|1998-06-06|1998-07-20|COLLECT COD|TRUCK|sits poach carefully|
+5798|146|3|4|40|41845.60|0.08|0.06|N|O|1998-07-09|1998-06-24|1998-07-16|NONE|TRUCK| integrate carefu|
+5798|149|8|5|7|7343.98|0.06|0.07|N|O|1998-06-06|1998-05-10|1998-06-07|NONE|SHIP|ts against the blithely final p|
+5798|38|4|6|9|8442.27|0.06|0.02|N|O|1998-05-05|1998-05-25|1998-05-09|DELIVER IN PERSON|REG AIR|e blithely|
+5798|115|9|7|32|32483.52|0.08|0.01|N|O|1998-04-27|1998-05-03|1998-05-08|TAKE BACK RETURN|REG AIR|ubt blithely above the |
+5799|95|6|1|41|40798.69|0.04|0.02|N|O|1995-11-13|1995-10-31|1995-11-16|COLLECT COD|TRUCK|al accounts sleep ruthlessl|
+5799|100|3|2|30|30003.00|0.03|0.08|N|O|1995-09-12|1995-09-13|1995-09-19|NONE|RAIL| furiously s|
+5824|77|7|1|40|39082.80|0.06|0.06|N|O|1997-01-14|1997-01-17|1997-02-02|NONE|REG AIR|he final packag|
+5824|182|3|2|42|45451.56|0.09|0.00|N|O|1997-02-01|1997-02-20|1997-02-07|COLLECT COD|SHIP|ts sleep. carefully regular accounts h|
+5824|73|1|3|16|15569.12|0.03|0.02|N|O|1997-02-13|1997-01-07|1997-02-17|TAKE BACK RETURN|TRUCK|sly express Ti|
+5824|92|5|4|32|31746.88|0.03|0.02|N|O|1997-02-16|1997-01-24|1997-02-20|DELIVER IN PERSON|RAIL|ven requests. |
+5824|107|8|5|44|44312.40|0.08|0.03|N|O|1997-01-24|1997-01-31|1997-02-11|COLLECT COD|TRUCK|fily fluffily bold|
+5825|159|7|1|23|24360.45|0.10|0.05|R|F|1995-05-10|1995-04-28|1995-05-13|DELIVER IN PERSON|TRUCK| special pinto beans. dependencies haggl|
+5826|144|1|1|4|4176.56|0.03|0.06|N|O|1998-07-31|1998-09-10|1998-08-27|NONE|AIR| packages across the fluffily spec|
+5826|64|5|2|18|17353.08|0.04|0.01|N|O|1998-07-17|1998-09-03|1998-07-22|NONE|SHIP|atelets use above t|
+5827|187|8|1|30|32615.40|0.03|0.05|N|O|1998-11-11|1998-09-27|1998-11-30|DELIVER IN PERSON|RAIL|ounts may c|
+5827|103|6|2|23|23071.30|0.09|0.05|N|O|1998-11-16|1998-09-14|1998-11-17|COLLECT COD|RAIL|ans. furiously special instruct|
+5827|164|1|3|3|3192.48|0.03|0.06|N|O|1998-10-17|1998-09-29|1998-10-28|DELIVER IN PERSON|MAIL|uses eat along the furiously|
+5827|200|1|4|26|28605.20|0.06|0.00|N|O|1998-07-29|1998-09-24|1998-07-30|DELIVER IN PERSON|SHIP|arefully special packages wake thin|
+5827|112|9|5|38|38460.18|0.03|0.06|N|O|1998-10-18|1998-08-27|1998-10-23|TAKE BACK RETURN|TRUCK|ly ruthless accounts|
+5827|17|4|6|14|12838.14|0.05|0.01|N|O|1998-08-31|1998-09-06|1998-09-13|TAKE BACK RETURN|RAIL|rges. fluffily pending |
+5828|2|9|1|28|25256.00|0.10|0.03|A|F|1994-05-15|1994-05-20|1994-06-08|DELIVER IN PERSON|MAIL| special ideas haggle slyly ac|
+5828|158|3|2|37|39151.55|0.01|0.00|R|F|1994-06-07|1994-05-30|1994-06-17|NONE|RAIL|e carefully spec|
+5829|40|1|1|4|3760.16|0.01|0.02|N|O|1997-03-01|1997-02-17|1997-03-22|NONE|TRUCK|ithely; accounts cajole ideas. regular foxe|
+5829|107|10|2|40|40284.00|0.04|0.01|N|O|1997-04-21|1997-02-12|1997-05-04|COLLECT COD|TRUCK| the carefully ironic accounts. a|
+5829|129|8|3|6|6174.72|0.05|0.06|N|O|1997-01-22|1997-03-12|1997-02-02|TAKE BACK RETURN|AIR|sts. slyly special fo|
+5829|90|1|4|42|41583.78|0.02|0.07|N|O|1997-03-26|1997-04-01|1997-03-30|COLLECT COD|REG AIR|pearls. slyly bold deposits solve final|
+5829|191|5|5|49|53468.31|0.05|0.01|N|O|1997-01-31|1997-03-13|1997-02-18|NONE|MAIL| ironic excuses use fluf|
+5829|18|5|6|17|15606.17|0.09|0.02|N|O|1997-04-10|1997-03-29|1997-04-22|COLLECT COD|AIR|after the furiously ironic ideas no|
+5829|78|9|7|27|26407.89|0.08|0.04|N|O|1997-02-25|1997-03-31|1997-03-03|DELIVER IN PERSON|AIR|ns about the excuses are c|
+5830|160|2|1|29|30744.64|0.10|0.02|R|F|1993-06-19|1993-05-10|1993-07-13|DELIVER IN PERSON|REG AIR|y bold excuses|
+5831|191|2|1|2|2182.38|0.10|0.01|N|O|1997-02-09|1997-01-20|1997-03-07|TAKE BACK RETURN|TRUCK|quickly silent req|
+5831|74|3|2|33|32144.31|0.04|0.03|N|O|1996-11-20|1997-01-18|1996-12-18|TAKE BACK RETURN|MAIL| instructions wake. slyly sil|
+5831|82|3|3|6|5892.48|0.05|0.07|N|O|1997-01-29|1997-01-14|1997-02-09|NONE|MAIL|ly ironic accounts nag pendin|
+5831|13|10|4|46|41998.46|0.06|0.02|N|O|1997-02-24|1997-01-18|1997-03-02|COLLECT COD|MAIL|ly final pa|
+5831|43|4|5|37|34892.48|0.05|0.01|N|O|1997-01-17|1997-02-08|1997-02-01|NONE|FOB|uriously even requests|
+5856|4|1|1|1|904.00|0.03|0.02|A|F|1994-12-29|1995-01-07|1995-01-10|TAKE BACK RETURN|MAIL|tly. special deposits wake blithely even|
+5856|35|6|2|35|32726.05|0.09|0.02|R|F|1994-11-24|1994-12-23|1994-11-30|COLLECT COD|AIR|excuses. finally ir|
+5856|153|4|3|39|41072.85|0.05|0.03|A|F|1995-01-18|1995-01-11|1995-01-19|DELIVER IN PERSON|TRUCK|uickly quickly fluffy in|
+5857|58|9|1|25|23951.25|0.03|0.02|N|O|1997-12-02|1997-12-17|1997-12-08|DELIVER IN PERSON|REG AIR|ding platelets. pending excu|
+5857|195|9|2|50|54759.50|0.06|0.07|N|O|1997-12-04|1997-12-16|1997-12-20|NONE|TRUCK|y regular d|
+5857|68|3|3|1|968.06|0.03|0.01|N|O|1998-02-01|1997-12-09|1998-02-20|TAKE BACK RETURN|SHIP|instructions detect final reques|
+5857|118|2|4|12|12217.32|0.03|0.08|N|O|1998-01-24|1997-12-27|1998-02-10|TAKE BACK RETURN|AIR|counts. express, final|
+5857|192|4|5|14|15290.66|0.07|0.07|N|O|1997-12-10|1998-01-06|1998-01-04|TAKE BACK RETURN|TRUCK|ffily pendin|
+5857|93|5|6|49|48661.41|0.00|0.04|N|O|1998-01-23|1997-12-12|1998-01-28|DELIVER IN PERSON|REG AIR|egular pinto beans|
+5858|121|4|1|20|20422.40|0.02|0.06|A|F|1992-07-23|1992-08-26|1992-07-24|COLLECT COD|SHIP|uffily unusual pinto beans sleep|
+5858|16|7|2|36|32976.36|0.00|0.05|A|F|1992-09-25|1992-08-16|1992-10-11|NONE|SHIP|osits wake quickly quickly sile|
+5858|148|5|3|7|7336.98|0.08|0.02|A|F|1992-10-07|1992-08-16|1992-10-15|TAKE BACK RETURN|REG AIR|. doggedly regular packages use pendin|
+5858|164|9|4|46|48951.36|0.07|0.06|R|F|1992-09-07|1992-10-06|1992-10-06|DELIVER IN PERSON|MAIL|posits withi|
+5858|161|8|5|18|19100.88|0.00|0.07|A|F|1992-11-05|1992-10-08|1992-12-03|NONE|TRUCK|al excuses. bold|
+5858|154|9|6|7|7379.05|0.04|0.00|A|F|1992-09-14|1992-10-01|1992-10-01|TAKE BACK RETURN|RAIL|dly pending ac|
+5858|11|5|7|50|45550.50|0.06|0.00|R|F|1992-07-20|1992-10-07|1992-07-25|NONE|TRUCK|r the ironic ex|
+5859|175|4|1|50|53758.50|0.07|0.01|N|O|1997-07-08|1997-06-20|1997-07-27|COLLECT COD|MAIL|ly regular deposits use. ironic|
+5859|9|6|2|17|15453.00|0.03|0.03|N|O|1997-05-15|1997-06-30|1997-05-26|DELIVER IN PERSON|AIR|ly ironic requests. quickly unusual pin|
+5859|46|3|3|33|31219.32|0.10|0.04|N|O|1997-07-08|1997-06-22|1997-07-18|TAKE BACK RETURN|TRUCK|eposits unwind furiously final pinto bea|
+5859|93|4|4|40|39723.60|0.09|0.02|N|O|1997-08-05|1997-06-17|1997-08-20|NONE|REG AIR|l dependenci|
+5859|153|8|5|35|36860.25|0.00|0.08|N|O|1997-05-28|1997-07-14|1997-06-15|COLLECT COD|TRUCK|egular acco|
+5859|44|5|6|9|8496.36|0.01|0.02|N|O|1997-06-15|1997-06-06|1997-06-20|NONE|RAIL|ges boost quickly. blithely r|
+5859|191|5|7|27|29462.13|0.05|0.08|N|O|1997-07-30|1997-07-08|1997-08-08|NONE|MAIL| across th|
+5860|51|3|1|10|9510.50|0.04|0.04|A|F|1992-03-11|1992-03-30|1992-03-31|NONE|MAIL|ual patterns try to eat carefully above|
+5861|191|5|1|32|34918.08|0.00|0.03|N|O|1997-05-27|1997-05-29|1997-05-28|TAKE BACK RETURN|MAIL|nt asymptotes. carefully express request|
+5861|86|7|2|6|5916.48|0.10|0.03|N|O|1997-07-28|1997-05-18|1997-08-24|TAKE BACK RETURN|TRUCK|olites. slyly|
+5862|113|7|1|4|4052.44|0.09|0.06|N|O|1997-06-04|1997-04-26|1997-06-19|NONE|TRUCK|yly silent deposit|
+5862|2|7|2|29|26158.00|0.03|0.05|N|O|1997-04-02|1997-04-16|1997-04-04|NONE|FOB|e fluffily. furiously|
+5863|161|10|1|45|47752.20|0.07|0.06|A|F|1993-12-19|1994-01-25|1994-01-05|NONE|REG AIR| deposits are ab|
+5863|160|8|2|21|22263.36|0.09|0.03|R|F|1994-01-13|1994-01-09|1994-01-28|DELIVER IN PERSON|FOB|atelets nag blithely furi|
+5888|62|7|1|46|44254.76|0.02|0.00|N|O|1996-11-18|1996-11-05|1996-12-08|TAKE BACK RETURN|FOB|yly final accounts hag|
+5888|112|3|2|24|24290.64|0.03|0.01|N|O|1996-11-07|1996-11-30|1996-11-20|COLLECT COD|SHIP|ing to the spe|
+5889|77|7|1|17|16610.19|0.09|0.02|N|O|1995-07-01|1995-08-12|1995-07-25|NONE|AIR|blithely pending packages. flu|
+5890|113|4|1|38|38498.18|0.01|0.08|A|F|1993-02-14|1992-12-09|1993-02-27|COLLECT COD|FOB| accounts. carefully final asymptotes|
+5891|85|6|1|22|21671.76|0.00|0.06|R|F|1993-01-01|1993-02-18|1993-01-14|DELIVER IN PERSON|TRUCK|iresias cajole deposits. special, ir|
+5891|186|7|2|9|9775.62|0.03|0.07|R|F|1993-01-20|1993-02-27|1993-02-10|COLLECT COD|REG AIR|cajole carefully |
+5891|30|9|3|10|9300.30|0.08|0.01|A|F|1993-04-14|1993-02-07|1993-04-15|DELIVER IN PERSON|RAIL|nding requests. b|
+5892|148|9|1|7|7336.98|0.02|0.03|N|O|1995-06-26|1995-07-18|1995-07-25|COLLECT COD|AIR|e furiously. quickly even deposits da|
+5892|150|9|2|37|38855.55|0.09|0.06|N|O|1995-08-12|1995-06-11|1995-09-05|NONE|REG AIR|maintain. bold, expre|
+5892|3|4|3|28|25284.00|0.03|0.06|N|O|1995-08-16|1995-07-06|1995-08-22|DELIVER IN PERSON|MAIL|ithely unusual accounts will have to integ|
+5892|75|6|4|23|22426.61|0.08|0.04|R|F|1995-05-18|1995-07-06|1995-05-29|COLLECT COD|MAIL| foxes nag slyly about the qui|
+5893|134|10|1|43|44467.59|0.05|0.02|R|F|1992-11-02|1992-09-27|1992-11-21|TAKE BACK RETURN|RAIL|s. regular courts above the carefully silen|
+5893|2|9|2|2|1804.00|0.10|0.04|R|F|1992-07-18|1992-09-10|1992-08-12|NONE|RAIL|ckages wake sly|
+5894|8|5|1|23|20884.00|0.04|0.08|A|F|1994-09-05|1994-10-27|1994-09-13|NONE|TRUCK| furiously even deposits haggle alw|
+5894|79|8|2|48|46995.36|0.04|0.08|A|F|1994-09-04|1994-11-03|1994-09-17|NONE|TRUCK| asymptotes among the blithely silent |
+5895|15|9|1|38|34770.38|0.05|0.08|N|O|1997-04-05|1997-03-06|1997-05-03|DELIVER IN PERSON|RAIL|ts are furiously. regular, final excuses |
+5895|122|3|2|47|48039.64|0.04|0.06|N|O|1997-04-27|1997-03-17|1997-05-07|DELIVER IN PERSON|AIR|r packages wake carefull|
+5895|84|5|3|49|48219.92|0.03|0.07|N|O|1997-03-15|1997-02-17|1997-04-04|NONE|TRUCK|permanent foxes. packages|
+5895|146|7|4|31|32430.34|0.03|0.01|N|O|1997-03-03|1997-03-30|1997-03-08|TAKE BACK RETURN|TRUCK| final deposits nod slyly careful|
+5895|200|1|5|20|22004.00|0.07|0.00|N|O|1997-04-30|1997-02-07|1997-05-08|DELIVER IN PERSON|AIR|gular deposits wake blithely carefully fin|
+5895|78|7|6|15|14671.05|0.08|0.08|N|O|1997-04-19|1997-03-09|1997-05-13|TAKE BACK RETURN|RAIL|silent package|
+5920|187|8|1|50|54359.00|0.06|0.00|A|F|1995-03-13|1995-01-03|1995-03-31|TAKE BACK RETURN|RAIL|across the carefully pending platelets|
+5920|58|9|2|24|22993.20|0.01|0.05|A|F|1994-12-28|1995-01-21|1994-12-31|DELIVER IN PERSON|FOB|fully regular dolphins. furiousl|
+5920|117|1|3|2|2034.22|0.08|0.07|A|F|1995-02-18|1995-01-13|1995-03-04|NONE|SHIP| evenly spe|
+5920|12|2|4|28|25536.28|0.06|0.02|R|F|1994-12-17|1995-02-13|1994-12-31|NONE|SHIP|le slyly slyly even deposits. f|
+5920|100|4|5|42|42004.20|0.09|0.08|A|F|1994-12-18|1995-01-07|1995-01-14|COLLECT COD|AIR|lar, ironic dependencies sno|
+5921|99|3|1|44|43959.96|0.07|0.01|R|F|1994-07-14|1994-06-30|1994-07-15|NONE|TRUCK|ain about the special|
+5921|146|9|2|25|26153.50|0.06|0.01|A|F|1994-05-19|1994-06-15|1994-06-17|COLLECT COD|TRUCK|nd the slyly regular deposits. quick|
+5921|68|5|3|17|16457.02|0.06|0.01|R|F|1994-05-20|1994-05-26|1994-05-23|NONE|FOB|final asymptotes. even packages boost |
+5921|28|7|4|26|24128.52|0.03|0.04|A|F|1994-05-03|1994-07-06|1994-05-06|NONE|AIR|hy dependenc|
+5921|143|10|5|41|42768.74|0.04|0.02|R|F|1994-04-13|1994-05-31|1994-04-26|DELIVER IN PERSON|AIR|nusual, regular theodol|
+5921|115|6|6|5|5075.55|0.02|0.00|R|F|1994-06-01|1994-05-07|1994-06-10|COLLECT COD|TRUCK|eas cajole across the final, fi|
+5922|196|10|1|9|9865.71|0.07|0.00|N|O|1996-12-04|1997-01-20|1996-12-08|DELIVER IN PERSON|RAIL|haggle slyly even packages. packages|
+5922|157|2|2|37|39114.55|0.01|0.04|N|O|1996-12-19|1996-12-16|1997-01-15|COLLECT COD|RAIL|s wake slyly. requests cajole furiously asy|
+5922|90|1|3|35|34653.15|0.08|0.00|N|O|1996-12-12|1997-01-21|1997-01-01|DELIVER IN PERSON|SHIP|accounts. regu|
+5922|66|7|4|13|12558.78|0.08|0.07|N|O|1997-03-08|1996-12-26|1997-04-03|DELIVER IN PERSON|FOB|sly special accounts wake ironically.|
+5922|57|5|5|39|37324.95|0.04|0.07|N|O|1997-03-04|1997-01-17|1997-03-25|TAKE BACK RETURN|SHIP|e of the instructions. quick|
+5922|179|9|6|10|10791.70|0.04|0.01|N|O|1997-02-23|1996-12-26|1997-03-04|NONE|REG AIR|sly regular deposits haggle quickly ins|
+5923|177|8|1|27|29083.59|0.08|0.03|N|O|1997-08-16|1997-06-27|1997-08-29|DELIVER IN PERSON|RAIL|arefully i|
+5923|119|3|2|42|42802.62|0.01|0.08|N|O|1997-09-16|1997-07-23|1997-09-27|COLLECT COD|REG AIR|y regular theodolites w|
+5923|108|5|3|2|2016.20|0.06|0.05|N|O|1997-06-19|1997-07-31|1997-06-28|TAKE BACK RETURN|TRUCK|express patterns. even deposits|
+5923|174|4|4|46|49411.82|0.05|0.04|N|O|1997-07-29|1997-07-23|1997-08-23|COLLECT COD|SHIP|nto beans cajole blithe|
+5923|59|4|5|35|33566.75|0.04|0.05|N|O|1997-07-21|1997-07-11|1997-08-01|DELIVER IN PERSON|AIR|sts affix unusual, final requests. request|
+5924|176|5|1|38|40894.46|0.06|0.05|N|O|1995-12-17|1995-12-11|1996-01-06|TAKE BACK RETURN|AIR|ions cajole carefully along the |
+5924|53|1|2|49|46699.45|0.04|0.00|N|O|1995-10-25|1995-12-11|1995-11-08|NONE|MAIL|inly final excuses. blithely regular requ|
+5924|17|8|3|24|22008.24|0.09|0.08|N|O|1996-01-12|1995-12-13|1996-01-25|COLLECT COD|REG AIR| use carefully. special, e|
+5925|87|8|1|42|41457.36|0.05|0.02|N|O|1996-03-05|1996-01-13|1996-03-10|COLLECT COD|SHIP|to the furiously|
+5925|125|4|2|31|31778.72|0.03|0.03|N|O|1996-01-02|1995-12-14|1996-01-07|TAKE BACK RETURN|FOB|e slyly. furiously regular deposi|
+5925|89|10|3|50|49454.00|0.03|0.04|N|O|1996-02-14|1996-01-10|1996-02-15|NONE|TRUCK|es. stealthily express pains print bli|
+5925|54|9|4|30|28621.50|0.02|0.07|N|O|1996-02-21|1996-02-11|1996-03-10|NONE|TRUCK| the packa|
+5925|160|1|5|41|43466.56|0.00|0.06|N|O|1996-02-03|1995-12-24|1996-02-20|NONE|SHIP| across the pending deposits nag caref|
+5925|50|9|6|48|45602.40|0.02|0.00|N|O|1996-02-03|1996-01-19|1996-03-04|DELIVER IN PERSON|REG AIR| haggle after the fo|
+5926|90|1|1|8|7920.72|0.02|0.00|R|F|1994-07-17|1994-07-20|1994-08-11|COLLECT COD|MAIL|gle furiously express foxes. bo|
+5926|50|9|2|27|25651.35|0.09|0.05|A|F|1994-07-05|1994-08-11|1994-08-02|DELIVER IN PERSON|MAIL|ironic requests|
+5926|127|8|3|46|47247.52|0.01|0.03|R|F|1994-09-05|1994-08-12|1994-09-11|COLLECT COD|RAIL|ts integrate. courts haggl|
+5926|190|1|4|23|25074.37|0.01|0.02|A|F|1994-07-23|1994-08-10|1994-07-27|DELIVER IN PERSON|FOB|ickly special packages among |
+5927|90|1|1|44|43563.96|0.04|0.05|N|O|1997-11-29|1997-11-21|1997-12-13|DELIVER IN PERSON|TRUCK|rding to the special, final decoy|
+5927|115|2|2|8|8120.88|0.04|0.05|N|O|1997-09-24|1997-11-15|1997-10-22|TAKE BACK RETURN|SHIP|ilent dependencies nod c|
+5927|167|6|3|32|34149.12|0.10|0.07|N|O|1997-12-26|1997-10-27|1997-12-31|COLLECT COD|AIR|telets. carefully bold accounts was|
+5952|200|2|1|49|53909.80|0.10|0.02|N|O|1997-06-30|1997-07-10|1997-07-02|COLLECT COD|AIR|e furiously regular|
+5952|191|5|2|11|12003.09|0.10|0.05|N|O|1997-05-13|1997-06-04|1997-05-27|DELIVER IN PERSON|FOB|y nag blithely aga|
+5952|71|2|3|43|41756.01|0.01|0.01|N|O|1997-06-29|1997-06-06|1997-07-15|COLLECT COD|MAIL|posits sleep furiously quickly final p|
+5952|158|3|4|23|24337.45|0.00|0.07|N|O|1997-05-13|1997-06-27|1997-05-20|NONE|TRUCK|e blithely packages. eve|
+5953|129|10|1|36|37048.32|0.03|0.00|R|F|1992-05-28|1992-06-24|1992-05-29|DELIVER IN PERSON|FOB| cajole furio|
+5953|13|7|2|34|31042.34|0.03|0.04|A|F|1992-05-04|1992-06-12|1992-06-02|NONE|RAIL|hockey players use furiously against th|
+5953|162|9|3|5|5310.80|0.07|0.06|A|F|1992-04-10|1992-04-27|1992-04-14|NONE|SHIP|s. blithely |
+5953|169|8|4|23|24590.68|0.09|0.02|R|F|1992-06-05|1992-06-03|1992-06-29|TAKE BACK RETURN|FOB|he silent ideas. silent foxes po|
+5954|147|6|1|8|8377.12|0.03|0.00|A|F|1993-03-27|1993-01-22|1993-04-04|TAKE BACK RETURN|AIR|unusual th|
+5954|81|2|2|40|39243.20|0.02|0.01|A|F|1992-12-30|1993-01-16|1993-01-09|COLLECT COD|RAIL|iously ironic deposits after|
+5954|94|8|3|20|19881.80|0.09|0.07|A|F|1992-12-25|1993-02-05|1992-12-31|COLLECT COD|REG AIR| accounts wake carefu|
+5954|145|4|4|20|20902.80|0.00|0.01|R|F|1993-02-27|1993-01-04|1993-03-08|NONE|TRUCK|ke furiously blithely special packa|
+5954|100|4|5|35|35003.50|0.04|0.06|A|F|1993-03-17|1993-02-06|1993-04-10|NONE|SHIP|tions maintain slyly. furious|
+5954|193|5|6|39|42634.41|0.04|0.08|A|F|1993-02-27|1993-02-25|1993-03-29|DELIVER IN PERSON|REG AIR| always regular dolphins. furiously p|
+5955|140|1|1|14|14561.96|0.08|0.08|N|O|1995-06-22|1995-05-23|1995-06-24|DELIVER IN PERSON|TRUCK| unusual, bold theodolit|
+5955|62|7|2|15|14430.90|0.08|0.07|R|F|1995-04-22|1995-05-28|1995-04-27|NONE|FOB|y final accounts above the regu|
+5955|112|9|3|40|40484.40|0.03|0.00|R|F|1995-04-01|1995-06-11|1995-04-27|NONE|FOB|oss the fluffily regular|
+5956|155|3|1|10|10551.50|0.04|0.05|N|O|1998-07-27|1998-07-04|1998-08-21|NONE|MAIL|ic packages am|
+5956|55|7|2|23|21966.15|0.08|0.03|N|O|1998-06-06|1998-07-10|1998-06-15|DELIVER IN PERSON|RAIL|ly slyly special |
+5956|175|5|3|47|50532.99|0.04|0.06|N|O|1998-09-06|1998-06-29|1998-09-18|TAKE BACK RETURN|MAIL|lyly express theodol|
+5956|20|10|4|40|36800.80|0.09|0.05|N|O|1998-06-11|1998-07-19|1998-06-21|NONE|MAIL|final theodolites sleep carefully ironic c|
+5957|15|9|1|37|33855.37|0.07|0.00|A|F|1994-04-18|1994-02-19|1994-05-11|NONE|AIR| ideas use ruthlessly.|
+5957|59|4|2|46|44116.30|0.04|0.08|A|F|1994-01-23|1994-01-30|1994-02-07|NONE|SHIP|platelets. furiously unusual requests |
+5957|2|7|3|17|15334.00|0.01|0.01|A|F|1994-01-24|1994-02-16|1994-02-08|TAKE BACK RETURN|SHIP|. final, pending packages|
+5957|132|3|4|29|29931.77|0.01|0.03|R|F|1994-02-24|1994-03-04|1994-03-08|COLLECT COD|REG AIR|sits. final, even asymptotes cajole quickly|
+5957|88|9|5|40|39523.20|0.04|0.04|R|F|1994-01-07|1994-02-05|1994-01-26|DELIVER IN PERSON|SHIP|ironic asymptotes sleep blithely again|
+5957|6|1|6|41|37146.00|0.10|0.07|R|F|1994-03-25|1994-02-20|1994-03-31|DELIVER IN PERSON|MAIL|es across the regular requests maint|
+5957|159|1|7|32|33892.80|0.10|0.07|A|F|1994-03-05|1994-02-20|1994-03-09|NONE|TRUCK| boost carefully across the |
+5958|149|8|1|33|34621.62|0.02|0.04|N|O|1995-09-24|1995-12-12|1995-10-05|COLLECT COD|MAIL|lar, regular accounts wake furi|
+5958|43|6|2|23|21689.92|0.03|0.04|N|O|1995-09-26|1995-10-19|1995-09-27|COLLECT COD|SHIP|regular requests. bold, bold deposits unwin|
+5958|153|8|3|42|44232.30|0.10|0.00|N|O|1995-12-12|1995-10-19|1996-01-09|NONE|AIR|n accounts. final, ironic packages |
+5958|39|10|4|18|16902.54|0.04|0.05|N|O|1995-12-02|1995-10-17|1995-12-22|COLLECT COD|FOB|regular requests haggle|
+5958|132|8|5|32|33028.16|0.06|0.00|N|O|1995-09-20|1995-12-10|1995-10-14|COLLECT COD|REG AIR|e carefully special theodolites. carefully |
+5959|135|1|1|49|50721.37|0.07|0.03|R|F|1992-07-16|1992-08-09|1992-08-14|DELIVER IN PERSON|SHIP|usual packages haggle slyly pi|
+5959|147|8|2|17|17801.38|0.09|0.07|R|F|1992-06-10|1992-07-06|1992-06-23|COLLECT COD|MAIL|ackages. blithely ex|
+5959|5|6|3|4|3620.00|0.04|0.03|R|F|1992-06-14|1992-07-05|1992-07-01|NONE|MAIL|gular requests ar|
+5959|196|7|4|13|14250.47|0.03|0.00|A|F|1992-07-29|1992-07-13|1992-08-20|COLLECT COD|SHIP|ar forges. deposits det|
+5959|40|6|5|37|34781.48|0.04|0.01|R|F|1992-06-05|1992-07-18|1992-06-29|NONE|TRUCK|endencies. brai|
+5959|119|3|6|35|35668.85|0.03|0.00|A|F|1992-05-27|1992-06-19|1992-06-23|NONE|TRUCK|ely silent deposits. |
+5959|43|10|7|47|44322.88|0.02|0.01|R|F|1992-08-28|1992-07-24|1992-09-09|TAKE BACK RETURN|RAIL|deposits. slyly special cou|
+5984|70|5|1|13|12610.91|0.06|0.07|R|F|1994-10-16|1994-09-06|1994-11-11|NONE|MAIL|lar platelets. f|
+5984|102|3|2|25|25052.50|0.05|0.08|R|F|1994-10-06|1994-07-21|1994-10-28|COLLECT COD|RAIL|gular accounts. even packages nag slyly|
+5984|1|4|3|8|7208.00|0.10|0.00|R|F|1994-09-17|1994-08-28|1994-09-25|COLLECT COD|RAIL|its. express,|
+5984|190|1|4|35|38156.65|0.00|0.01|A|F|1994-08-25|1994-08-05|1994-08-31|DELIVER IN PERSON|SHIP|le fluffily regula|
+5985|86|7|1|4|3944.32|0.02|0.02|A|F|1995-05-04|1995-04-01|1995-05-17|DELIVER IN PERSON|MAIL|ole along the quickly slow d|
+5986|79|7|1|26|25455.82|0.00|0.00|R|F|1992-08-10|1992-05-23|1992-08-24|TAKE BACK RETURN|SHIP|e fluffily ironic ideas. silent |
+5986|196|8|2|25|27404.75|0.03|0.06|A|F|1992-06-16|1992-07-17|1992-06-29|TAKE BACK RETURN|MAIL| instructions. slyly regular de|
+5986|30|5|3|1|930.03|0.07|0.06|A|F|1992-05-21|1992-06-21|1992-05-24|DELIVER IN PERSON|REG AIR|fix quickly quickly final deposits. fluffil|
+5986|90|1|4|31|30692.79|0.00|0.03|A|F|1992-08-21|1992-06-29|1992-09-14|NONE|AIR|structions! furiously pending instructi|
+5986|136|7|5|6|6216.78|0.05|0.02|A|F|1992-07-16|1992-06-10|1992-07-29|DELIVER IN PERSON|RAIL|al foxes within the slyly speci|
+5987|23|2|1|1|923.02|0.01|0.04|N|O|1996-09-13|1996-10-29|1996-09-21|DELIVER IN PERSON|REG AIR|refully final excuses haggle furiously ag|
+5987|176|5|2|20|21523.40|0.10|0.06|N|O|1996-11-28|1996-09-17|1996-12-05|TAKE BACK RETURN|RAIL|ing excuses nag quickly always bold|
+5987|92|3|3|43|42659.87|0.08|0.04|N|O|1996-10-30|1996-10-13|1996-11-12|NONE|AIR|theodolites wake above the furiously b|
+5987|97|1|4|37|36892.33|0.08|0.08|N|O|1996-10-15|1996-10-27|1996-11-09|NONE|MAIL|le furiously carefully special |
+5988|172|1|1|41|43958.97|0.08|0.03|R|F|1994-01-20|1994-02-06|1994-02-10|COLLECT COD|AIR|the pending, express reque|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/nation.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/nation.tbl
new file mode 100644
index 0000000..ed3fd5b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/nation.tbl
@@ -0,0 +1,25 @@
+0|ALGERIA|0| haggle. carefully final deposits detect slyly agai|
+1|ARGENTINA|1|al foxes promise slyly according to the regular accounts. bold requests alon|
+2|BRAZIL|1|y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special |
+3|CANADA|1|eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold|
+4|EGYPT|4|y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d|
+5|ETHIOPIA|0|ven packages wake quickly. regu|
+6|FRANCE|3|refully final requests. regular, ironi|
+7|GERMANY|3|l platelets. regular accounts x-ray: unusual, regular acco|
+8|INDIA|2|ss excuses cajole slyly across the packages. deposits print aroun|
+9|INDONESIA|2| slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull|
+10|IRAN|4|efully alongside of the slyly final dependencies. |
+11|IRAQ|4|nic deposits boost atop the quickly final requests? quickly regula|
+12|JAPAN|2|ously. final, express gifts cajole a|
+13|JORDAN|4|ic deposits are blithely about the carefully regular pa|
+14|KENYA|0| pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t|
+15|MOROCCO|0|rns. blithely bold courts among the closely regular packages use furiously bold platelets?|
+16|MOZAMBIQUE|0|s. ironic, unusual asymptotes wake blithely r|
+17|PERU|1|platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun|
+18|CHINA|2|c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos|
+19|ROMANIA|3|ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account|
+20|SAUDI ARABIA|4|ts. silent requests haggle. closely express packages sleep across the blithely|
+21|VIETNAM|2|hely enticingly express accounts. even, final |
+22|RUSSIA|3| requests against the platelets use never according to the quickly regular pint|
+23|UNITED KINGDOM|3|eans boost carefully special requests. accounts are. carefull|
+24|UNITED STATES|1|y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/orders-part1.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/orders-part1.tbl
new file mode 100644
index 0000000..ed78053
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/orders-part1.tbl
@@ -0,0 +1,750 @@
+1|37|O|131251.81|1996-01-02|5-LOW|Clerk#000000951|0|nstructions sleep furiously among |
+2|79|O|40183.29|1996-12-01|1-URGENT|Clerk#000000880|0| foxes. pending accounts at the pending, silent asymptot|
+3|124|F|160882.76|1993-10-14|5-LOW|Clerk#000000955|0|sly final accounts boost. carefully regular ideas cajole carefully. depos|
+4|137|O|31084.79|1995-10-11|5-LOW|Clerk#000000124|0|sits. slyly regular warthogs cajole. regular, regular theodolites acro|
+5|46|F|86615.25|1994-07-30|5-LOW|Clerk#000000925|0|quickly. bold deposits sleep slyly. packages use slyly|
+6|56|F|36468.55|1992-02-21|4-NOT SPECIFIED|Clerk#000000058|0|ggle. special, final requests are against the furiously specia|
+7|40|O|171488.73|1996-01-10|2-HIGH|Clerk#000000470|0|ly special requests |
+32|131|O|116923.00|1995-07-16|2-HIGH|Clerk#000000616|0|ise blithely bold, regular requests. quickly unusual dep|
+33|67|F|99798.76|1993-10-27|3-MEDIUM|Clerk#000000409|0|uriously. furiously final request|
+34|62|O|41670.02|1998-07-21|3-MEDIUM|Clerk#000000223|0|ly final packages. fluffily final deposits wake blithely ideas. spe|
+35|128|O|148789.52|1995-10-23|4-NOT SPECIFIED|Clerk#000000259|0|zzle. carefully enticing deposits nag furio|
+36|116|O|38988.98|1995-11-03|1-URGENT|Clerk#000000358|0| quick packages are blithely. slyly silent accounts wake qu|
+37|88|F|113701.89|1992-06-03|3-MEDIUM|Clerk#000000456|0|kly regular pinto beans. carefully unusual waters cajole never|
+38|125|O|46366.56|1996-08-21|4-NOT SPECIFIED|Clerk#000000604|0|haggle blithely. furiously express ideas haggle blithely furiously regular re|
+39|82|O|219707.84|1996-09-20|3-MEDIUM|Clerk#000000659|0|ole express, ironic requests: ir|
+64|34|F|20065.73|1994-07-16|3-MEDIUM|Clerk#000000661|0|wake fluffily. sometimes ironic pinto beans about the dolphin|
+65|17|P|65883.92|1995-03-18|1-URGENT|Clerk#000000632|0|ular requests are blithely pending orbits-- even requests against the deposit|
+66|130|F|79258.24|1994-01-20|5-LOW|Clerk#000000743|0|y pending requests integrate|
+67|58|O|116227.05|1996-12-19|4-NOT SPECIFIED|Clerk#000000547|0|symptotes haggle slyly around the furiously iron|
+68|29|O|215135.72|1998-04-18|3-MEDIUM|Clerk#000000440|0| pinto beans sleep carefully. blithely ironic deposits haggle furiously acro|
+69|85|F|162176.23|1994-06-04|4-NOT SPECIFIED|Clerk#000000330|0| depths atop the slyly thin deposits detect among the furiously silent accou|
+70|65|F|84651.80|1993-12-18|5-LOW|Clerk#000000322|0| carefully ironic request|
+71|4|O|178821.73|1998-01-24|4-NOT SPECIFIED|Clerk#000000271|0| express deposits along the blithely regul|
+96|109|F|55090.67|1994-04-17|2-HIGH|Clerk#000000395|0|oost furiously. pinto|
+97|22|F|68908.31|1993-01-29|3-MEDIUM|Clerk#000000547|0|hang blithely along the regular accounts. furiously even ideas after the|
+98|106|F|51004.44|1994-09-25|1-URGENT|Clerk#000000448|0|c asymptotes. quickly regular packages should have to nag re|
+99|89|F|92326.79|1994-03-13|4-NOT SPECIFIED|Clerk#000000973|0|e carefully ironic packages. pending|
+100|148|O|141311.01|1998-02-28|4-NOT SPECIFIED|Clerk#000000577|0|heodolites detect slyly alongside of the ent|
+101|28|O|95591.40|1996-03-17|3-MEDIUM|Clerk#000000419|0|ding accounts above the slyly final asymptote|
+102|1|O|113954.89|1997-05-09|2-HIGH|Clerk#000000596|0| slyly according to the asymptotes. carefully final packages integrate furious|
+103|31|O|95563.95|1996-06-20|4-NOT SPECIFIED|Clerk#000000090|0|ges. carefully unusual instructions haggle quickly regular f|
+128|74|F|36333.34|1992-06-15|1-URGENT|Clerk#000000385|0|ns integrate fluffily. ironic asymptotes after the regular excuses nag around |
+129|73|F|188124.55|1992-11-19|5-LOW|Clerk#000000859|0|ing tithes. carefully pending deposits boost about the silently express |
+130|37|F|115717.37|1992-05-08|2-HIGH|Clerk#000000036|0|le slyly unusual, regular packages? express deposits det|
+131|94|F|96596.81|1994-06-08|3-MEDIUM|Clerk#000000625|0|after the fluffily special foxes integrate s|
+132|28|F|118802.62|1993-06-11|3-MEDIUM|Clerk#000000488|0|sits are daringly accounts. carefully regular foxes sleep slyly about the|
+133|44|O|80437.72|1997-11-29|1-URGENT|Clerk#000000738|0|usly final asymptotes |
+134|7|F|154260.84|1992-05-01|4-NOT SPECIFIED|Clerk#000000711|0|lar theodolites boos|
+135|61|O|174569.88|1995-10-21|4-NOT SPECIFIED|Clerk#000000804|0|l platelets use according t|
+160|83|O|86076.86|1996-12-19|4-NOT SPECIFIED|Clerk#000000342|0|thely special sauternes wake slyly of t|
+161|17|F|19056.99|1994-08-31|2-HIGH|Clerk#000000322|0|carefully! special instructions sin|
+162|16|O|2158.13|1995-05-08|3-MEDIUM|Clerk#000000378|0|nts hinder fluffily ironic instructions. express, express excuses |
+163|88|O|125170.86|1997-09-05|3-MEDIUM|Clerk#000000379|0|y final packages. final foxes since the quickly even|
+164|1|F|202660.52|1992-10-21|5-LOW|Clerk#000000209|0|cajole ironic courts. slyly final ideas are slyly. blithely final Tiresias sub|
+165|28|F|141824.23|1993-01-30|4-NOT SPECIFIED|Clerk#000000292|0|across the blithely regular accounts. bold|
+166|109|O|93335.60|1995-09-12|2-HIGH|Clerk#000000440|0|lets. ironic, bold asymptotes kindle|
+167|121|F|52982.23|1993-01-04|4-NOT SPECIFIED|Clerk#000000731|0|s nag furiously bold excuses. fluffily iron|
+192|83|O|133002.55|1997-11-25|5-LOW|Clerk#000000483|0|y unusual platelets among the final instructions integrate rut|
+193|80|F|48053.18|1993-08-08|1-URGENT|Clerk#000000025|0|the furiously final pin|
+194|62|F|114097.63|1992-04-05|3-MEDIUM|Clerk#000000352|0|egular requests haggle slyly regular, regular pinto beans. asymptote|
+195|136|F|120053.52|1993-12-28|3-MEDIUM|Clerk#000000216|0|old forges are furiously sheaves. slyly fi|
+196|65|F|33248.04|1993-03-17|2-HIGH|Clerk#000000988|0|beans boost at the foxes. silent foxes|
+197|34|P|100290.07|1995-04-07|2-HIGH|Clerk#000000969|0|solve quickly about the even braids. carefully express deposits affix care|
+198|112|O|125792.83|1998-01-02|4-NOT SPECIFIED|Clerk#000000331|0|its. carefully ironic requests sleep. furiously express fox|
+199|53|O|80592.44|1996-03-07|2-HIGH|Clerk#000000489|0|g theodolites. special packag|
+224|4|F|155680.60|1994-06-18|4-NOT SPECIFIED|Clerk#000000642|0|r the quickly thin courts. carefully|
+225|34|P|165890.47|1995-05-25|1-URGENT|Clerk#000000177|0|s. blithely ironic accounts wake quickly fluffily special acc|
+226|128|F|180119.22|1993-03-10|2-HIGH|Clerk#000000756|0|s are carefully at the blithely ironic acc|
+227|10|O|46076.46|1995-11-10|5-LOW|Clerk#000000919|0| express instructions. slyly regul|
+228|46|F|2638.98|1993-02-25|1-URGENT|Clerk#000000562|0|es was slyly among the regular foxes. blithely regular dependenci|
+229|112|F|142290.77|1993-12-29|1-URGENT|Clerk#000000628|0|he fluffily even instructions. furiously i|
+230|103|F|107231.60|1993-10-27|1-URGENT|Clerk#000000520|0|odolites. carefully quick requ|
+231|91|F|141554.06|1994-09-29|2-HIGH|Clerk#000000446|0| packages haggle slyly after the carefully ironic instruct|
+256|125|F|106315.25|1993-10-19|4-NOT SPECIFIED|Clerk#000000834|0|he fluffily final ideas might are final accounts. carefully f|
+257|124|O|7102.74|1998-03-28|3-MEDIUM|Clerk#000000680|0|ts against the sly warhorses cajole slyly accounts|
+258|43|F|186669.10|1993-12-29|1-URGENT|Clerk#000000167|0|dencies. blithely quick packages cajole. ruthlessly final accounts|
+259|44|F|75661.70|1993-09-29|4-NOT SPECIFIED|Clerk#000000601|0|ages doubt blithely against the final foxes. carefully express deposits dazzle|
+260|106|O|179292.14|1996-12-10|3-MEDIUM|Clerk#000000960|0|lently regular pinto beans sleep after the slyly e|
+261|47|F|201003.12|1993-06-29|3-MEDIUM|Clerk#000000310|0|ully fluffily brave instructions. furiousl|
+262|31|O|108443.84|1995-11-25|4-NOT SPECIFIED|Clerk#000000551|0|l packages. blithely final pinto beans use carefu|
+263|118|F|79782.56|1994-05-17|2-HIGH|Clerk#000000088|0| pending instructions. blithely un|
+288|8|O|163794.53|1997-02-21|1-URGENT|Clerk#000000109|0|uriously final requests. even, final ideas det|
+289|104|O|131092.67|1997-02-10|3-MEDIUM|Clerk#000000103|0|sily. slyly special excuse|
+290|118|F|62814.89|1994-01-01|4-NOT SPECIFIED|Clerk#000000735|0|efully dogged deposits. furiou|
+291|142|F|66817.05|1994-03-13|1-URGENT|Clerk#000000923|0|dolites. carefully regular pinto beans cajol|
+292|23|F|30783.05|1992-01-13|2-HIGH|Clerk#000000193|0|g pinto beans will have to sleep f|
+293|31|F|37248.78|1992-10-02|2-HIGH|Clerk#000000629|0|re bold, ironic deposits. platelets c|
+294|52|F|30059.47|1993-07-16|3-MEDIUM|Clerk#000000499|0|kly according to the frays. final dolphins affix quickly |
+295|19|F|89345.99|1994-09-29|2-HIGH|Clerk#000000155|0| unusual pinto beans play. regular ideas haggle|
+320|1|O|39835.54|1997-11-21|2-HIGH|Clerk#000000573|0|ar foxes nag blithely|
+321|124|F|62251.15|1993-03-21|3-MEDIUM|Clerk#000000289|0|equests run. blithely final dependencies after the deposits wake caref|
+322|134|F|127068.89|1992-03-19|1-URGENT|Clerk#000000158|0|fully across the slyly bold packages. packages against the quickly regular i|
+323|40|F|79683.42|1994-03-26|1-URGENT|Clerk#000000959|0|arefully pending foxes sleep blithely. slyly express accoun|
+324|106|F|26868.85|1992-03-20|1-URGENT|Clerk#000000352|0| about the ironic, regular deposits run blithely against the excuses|
+325|41|F|71543.41|1993-10-17|5-LOW|Clerk#000000844|0|ly sometimes pending pa|
+326|76|O|229165.17|1995-06-04|2-HIGH|Clerk#000000466|0| requests. furiously ironic asymptotes mold carefully alongside of the blit|
+327|145|P|24468.16|1995-04-17|5-LOW|Clerk#000000992|0|ng the slyly final courts. slyly even escapades eat |
+352|107|F|16003.86|1994-03-08|2-HIGH|Clerk#000000932|0|ke slyly bold pinto beans. blithely regular accounts against the spe|
+353|2|F|179984.42|1993-12-31|5-LOW|Clerk#000000449|0| quiet ideas sleep. even instructions cajole slyly. silently spe|
+354|139|O|157062.70|1996-03-14|2-HIGH|Clerk#000000511|0|ly regular ideas wake across the slyly silent ideas. final deposits eat b|
+355|71|F|69447.25|1994-06-14|5-LOW|Clerk#000000532|0|s. sometimes regular requests cajole. regular, pending accounts a|
+356|148|F|162786.67|1994-06-30|4-NOT SPECIFIED|Clerk#000000944|0|as wake along the bold accounts. even, |
+357|61|O|98723.11|1996-10-09|2-HIGH|Clerk#000000301|0|e blithely about the express, final accounts. quickl|
+358|4|F|226806.66|1993-09-20|2-HIGH|Clerk#000000392|0|l, silent instructions are slyly. silently even de|
+359|79|F|142891.22|1994-12-19|3-MEDIUM|Clerk#000000934|0|n dolphins. special courts above the carefully ironic requests use|
+384|115|F|122785.82|1992-03-03|5-LOW|Clerk#000000206|0|, even accounts use furiously packages. slyly ironic pla|
+385|34|O|50724.06|1996-03-22|5-LOW|Clerk#000000600|0|hless accounts unwind bold pain|
+386|61|F|90380.40|1995-01-25|2-HIGH|Clerk#000000648|0| haggle quickly. stealthily bold asymptotes haggle among the furiously even re|
+387|4|O|130647.18|1997-01-26|4-NOT SPECIFIED|Clerk#000000768|0| are carefully among the quickly even deposits. furiously silent req|
+388|46|F|120533.46|1992-12-16|4-NOT SPECIFIED|Clerk#000000356|0|ar foxes above the furiously ironic deposits nag slyly final reque|
+389|127|F|1984.14|1994-02-17|2-HIGH|Clerk#000000062|0|ing to the regular asymptotes. final, pending foxes about the blithely sil|
+390|103|O|168562.27|1998-04-07|5-LOW|Clerk#000000404|0|xpress asymptotes use among the regular, final pinto b|
+391|112|F|13282.23|1994-11-17|2-HIGH|Clerk#000000256|0|orges thrash fluffil|
+416|41|F|71362.50|1993-09-27|5-LOW|Clerk#000000294|0| the accounts. fluffily bold depo|
+417|55|F|91982.29|1994-02-06|3-MEDIUM|Clerk#000000468|0|ironic, even packages. thinly unusual accounts sleep along the slyly unusual |
+418|95|P|33124.96|1995-04-13|4-NOT SPECIFIED|Clerk#000000643|0|. furiously ironic instruc|
+419|118|O|111597.96|1996-10-01|3-MEDIUM|Clerk#000000376|0|osits. blithely pending theodolites boost carefully|
+420|91|O|198039.23|1995-10-31|4-NOT SPECIFIED|Clerk#000000756|0|leep carefully final excuses. fluffily pending requests unwind carefully above|
+421|40|F|1084.38|1992-02-22|5-LOW|Clerk#000000405|0|egular, even packages according to the final, un|
+422|74|O|106045.89|1997-05-31|4-NOT SPECIFIED|Clerk#000000049|0|aggle carefully across the accounts. regular accounts eat fluffi|
+423|104|O|26981.31|1996-06-01|1-URGENT|Clerk#000000674|0|quests. deposits cajole quickly. furiously bold accounts haggle q|
+448|149|O|114978.03|1995-08-21|3-MEDIUM|Clerk#000000597|0| regular, express foxes use blithely. quic|
+449|97|O|41605.63|1995-07-20|2-HIGH|Clerk#000000841|0|. furiously regular theodolites affix blithely |
+450|49|P|153386.61|1995-03-05|4-NOT SPECIFIED|Clerk#000000293|0|d theodolites. boldly bold foxes since the pack|
+451|100|O|104664.40|1998-05-25|5-LOW|Clerk#000000048|0|nic pinto beans. theodolites poach carefully; |
+452|61|O|2007.48|1997-10-14|1-URGENT|Clerk#000000498|0|t, unusual instructions above the blithely bold pint|
+453|46|O|216826.73|1997-05-26|5-LOW|Clerk#000000504|0|ss foxes. furiously regular ideas sleep according to t|
+454|49|O|23198.24|1995-12-27|5-LOW|Clerk#000000890|0|dolites sleep carefully blithely regular deposits. quickly regul|
+455|13|O|138010.76|1996-12-04|1-URGENT|Clerk#000000796|0| about the final platelets. dependen|
+480|73|F|20530.97|1993-05-08|5-LOW|Clerk#000000004|0|ealthy pinto beans. fluffily regular requests along the special sheaves wake |
+481|31|F|117827.18|1992-10-08|2-HIGH|Clerk#000000230|0|ly final ideas. packages haggle fluffily|
+482|127|O|136634.34|1996-03-26|1-URGENT|Clerk#000000295|0|ts. deposits wake: final acco|
+483|35|O|39793.05|1995-07-11|2-HIGH|Clerk#000000025|0|cross the carefully final e|
+484|55|O|219920.62|1997-01-03|3-MEDIUM|Clerk#000000545|0|grouches use. furiously bold accounts maintain. bold, regular deposits|
+485|101|O|110432.76|1997-03-26|2-HIGH|Clerk#000000105|0| regular ideas nag thinly furiously s|
+486|52|O|185968.15|1996-03-11|4-NOT SPECIFIED|Clerk#000000803|0|riously dolphins. fluffily ironic requ|
+487|109|F|48502.79|1992-08-18|1-URGENT|Clerk#000000086|0|ithely unusual courts eat accordi|
+512|64|P|124661.48|1995-05-20|5-LOW|Clerk#000000814|0|ding requests. carefully express theodolites was quickly. furious|
+513|61|O|63703.92|1995-05-01|2-HIGH|Clerk#000000522|0|regular packages. pinto beans cajole carefully against the even|
+514|76|O|104585.77|1996-04-04|2-HIGH|Clerk#000000094|0| cajole furiously. slyly final excuses cajole. slyly special instructions |
+515|142|F|153720.22|1993-08-29|4-NOT SPECIFIED|Clerk#000000700|0|eposits are furiously furiously silent pinto beans. pending pack|
+516|44|O|10677.86|1998-04-21|2-HIGH|Clerk#000000305|0|lar, unusual platelets are carefully. even courts sleep bold, final pinto bea|
+517|10|O|82197.79|1997-04-07|5-LOW|Clerk#000000359|0|slyly pending deposits cajole quickly packages. furiou|
+518|145|O|223537.09|1998-02-08|2-HIGH|Clerk#000000768|0| the carefully bold accounts. quickly regular excuses are|
+519|64|O|95731.50|1997-10-31|1-URGENT|Clerk#000000985|0|ains doze furiously against the f|
+544|94|F|47627.89|1993-02-17|2-HIGH|Clerk#000000145|0|the special, final accounts. dogged dolphins|
+545|64|O|23476.12|1995-11-07|2-HIGH|Clerk#000000537|0|as. blithely final hockey players about th|
+546|145|O|14790.37|1996-11-01|2-HIGH|Clerk#000000041|0|osits sleep. slyly special dolphins about the q|
+547|100|O|96855.29|1996-06-22|3-MEDIUM|Clerk#000000976|0|ing accounts eat. carefully regular packa|
+548|124|F|99088.75|1994-09-21|1-URGENT|Clerk#000000435|0|arefully express instru|
+549|110|F|141679.41|1992-07-13|1-URGENT|Clerk#000000196|0|ideas alongside of |
+550|25|O|33123.28|1995-08-02|1-URGENT|Clerk#000000204|0|t requests. blithely |
+551|91|O|46355.83|1995-05-30|1-URGENT|Clerk#000000179|0|xpress accounts boost quic|
+576|31|O|18307.45|1997-05-13|3-MEDIUM|Clerk#000000955|0|l requests affix regular requests. final account|
+577|56|F|34768.68|1994-12-19|5-LOW|Clerk#000000154|0| deposits engage stealthil|
+578|94|O|70392.02|1997-01-10|5-LOW|Clerk#000000281|0|e blithely even packages. slyly pending platelets bes|
+579|68|O|120828.12|1998-03-11|2-HIGH|Clerk#000000862|0| regular instructions. blithely even p|
+580|61|O|88219.12|1997-07-05|2-HIGH|Clerk#000000314|0|tegrate fluffily regular accou|
+581|70|O|126066.00|1997-02-23|4-NOT SPECIFIED|Clerk#000000239|0| requests. even requests use slyly. blithely ironic |
+582|50|O|129004.81|1997-10-21|1-URGENT|Clerk#000000378|0|n pinto beans print a|
+583|49|O|127817.38|1997-03-19|3-MEDIUM|Clerk#000000792|0|efully express requests. a|
+608|26|O|62567.99|1996-02-28|3-MEDIUM|Clerk#000000995|0|nic waters wake slyly slyly expre|
+609|127|F|21088.59|1994-06-01|3-MEDIUM|Clerk#000000348|0|- ironic gifts believe furiously ca|
+610|52|O|175142.28|1995-08-02|1-URGENT|Clerk#000000610|0|totes. ironic, unusual packag|
+611|106|F|73907.63|1993-01-27|1-URGENT|Clerk#000000401|0|ounts detect furiously ac|
+612|82|F|145695.42|1992-10-21|3-MEDIUM|Clerk#000000759|0|boost quickly quickly final excuses. final foxes use bravely afte|
+613|139|O|33396.35|1995-06-18|2-HIGH|Clerk#000000172|0|ts hinder among the deposits. fluffily ironic depos|
+614|134|F|218116.21|1992-12-01|2-HIGH|Clerk#000000388|0| deposits! even, daring theodol|
+615|67|F|32890.89|1992-05-09|5-LOW|Clerk#000000388|0|t to promise asymptotes. packages haggle alongside of the fluffil|
+640|97|F|145495.62|1993-01-23|2-HIGH|Clerk#000000433|0|r, unusual accounts boost carefully final ideas. slyly silent theod|
+641|133|F|120626.49|1993-08-30|5-LOW|Clerk#000000175|0|ents cajole furiously about the quickly silent pac|
+642|40|F|22994.51|1993-12-16|3-MEDIUM|Clerk#000000357|0| among the requests wake slyly alongside of th|
+643|58|P|180396.95|1995-03-25|2-HIGH|Clerk#000000354|0|g dependencies. regular accounts |
+644|8|F|201268.06|1992-05-01|1-URGENT|Clerk#000000550|0| blithely unusual platelets haggle ironic, special excuses. excuses unwi|
+645|115|F|234763.73|1994-12-03|2-HIGH|Clerk#000000090|0|quickly daring theodolites across the regu|
+646|52|F|142070.65|1994-11-22|2-HIGH|Clerk#000000203|0|carefully even foxes. fina|
+647|143|O|56449.23|1997-08-07|1-URGENT|Clerk#000000270|0|egular pearls. carefully express asymptotes are. even account|
+672|109|F|89877.09|1994-04-14|5-LOW|Clerk#000000106|0|egular requests are furiously according to |
+673|80|F|21137.08|1994-03-10|1-URGENT|Clerk#000000448|0| special pinto beans use quickly furiously even depende|
+674|34|F|27204.60|1992-08-29|5-LOW|Clerk#000000448|0|ully special deposits. furiously final warhorses affix carefully. fluffily f|
+675|13|O|125188.72|1997-07-31|2-HIGH|Clerk#000000168|0|ffily between the careful|
+676|38|O|163966.67|1996-12-13|2-HIGH|Clerk#000000248|0|the final deposits. special, pending|
+677|124|F|147915.68|1993-11-24|3-MEDIUM|Clerk#000000824|0|uriously special pinto beans cajole carefully. fi|
+678|131|F|135761.05|1993-02-27|5-LOW|Clerk#000000530|0|. blithely final somas about the|
+679|49|O|8945.03|1995-12-15|2-HIGH|Clerk#000000853|0|tealthy, final pinto beans haggle slyly. pending platelets about the special, |
+704|85|O|56210.26|1996-11-21|3-MEDIUM|Clerk#000000682|0|blithely pending platelets wake alongside of the final, iron|
+705|43|O|83773.49|1997-02-13|4-NOT SPECIFIED|Clerk#000000294|0|ithely regular dependencies. express, even packages sleep slyly pending t|
+706|148|O|23973.60|1995-09-09|1-URGENT|Clerk#000000448|0|g the packages. deposits caj|
+707|118|F|58218.35|1994-11-20|3-MEDIUM|Clerk#000000199|0| ideas about the silent, bold deposits nag dolphins|
+708|32|O|100445.59|1998-07-03|3-MEDIUM|Clerk#000000101|0|lphins cajole about t|
+709|37|O|72055.87|1998-04-21|1-URGENT|Clerk#000000461|0|ons alongside of the carefully bold pinto bea|
+710|133|F|208974.42|1993-01-02|5-LOW|Clerk#000000026|0| regular, regular requests boost. fluffily re|
+711|64|F|92484.70|1993-09-23|4-NOT SPECIFIED|Clerk#000000856|0|its. fluffily regular gifts are furi|
+736|47|O|130204.17|1998-06-21|5-LOW|Clerk#000000881|0|refully of the final pi|
+737|121|F|12984.85|1992-04-26|5-LOW|Clerk#000000233|0|ake blithely express, ironic theodolites. blithely special accounts wa|
+738|22|F|114145.18|1993-03-02|4-NOT SPECIFIED|Clerk#000000669|0|ly even foxes. furiously regular accounts cajole ca|
+739|1|O|159171.69|1998-05-31|5-LOW|Clerk#000000900|0| against the slyly ironic packages nag slyly ironic|
+740|44|O|83490.99|1995-07-16|3-MEDIUM|Clerk#000000583|0|courts haggle furiously across the final, regul|
+741|106|O|47985.98|1998-07-07|2-HIGH|Clerk#000000295|0|ic instructions. slyly express instructions solv|
+742|103|F|207632.55|1994-12-23|5-LOW|Clerk#000000543|0|equests? slyly ironic dolphins boost carefully above the blithely|
+743|79|O|23614.89|1996-10-04|4-NOT SPECIFIED|Clerk#000000933|0|eans. furiously ironic deposits sleep carefully carefully qui|
+768|98|O|220636.82|1996-08-20|3-MEDIUM|Clerk#000000411|0|jole slyly ironic packages. slyly even idea|
+769|80|F|43092.76|1993-06-02|3-MEDIUM|Clerk#000000172|0|ggle furiously. ironic packages haggle slyly. bold platelets affix s|
+770|32|O|64271.75|1998-05-23|5-LOW|Clerk#000000572|0|heodolites. furiously special pinto beans cajole pac|
+771|46|O|105302.05|1995-06-17|1-URGENT|Clerk#000000105|0|s. furiously final instructions across the deposit|
+772|97|F|128234.96|1993-04-17|2-HIGH|Clerk#000000430|0|s boost blithely fluffily idle ideas? fluffily even pin|
+773|133|F|146862.27|1993-09-26|3-MEDIUM|Clerk#000000307|0|tions are quickly accounts. accounts use bold, even pinto beans. gifts ag|
+774|80|O|145857.60|1995-12-04|1-URGENT|Clerk#000000883|0|tealthily even depths|
+775|134|F|59455.61|1995-03-18|1-URGENT|Clerk#000000191|0|kly express requests. fluffily silent accounts poach furiously|
+800|56|O|87892.38|1998-07-14|2-HIGH|Clerk#000000213|0|y alongside of the pending packages? final platelets nag fluffily carefu|
+801|118|F|127717.72|1992-02-18|1-URGENT|Clerk#000000186|0|iously from the furiously enticing reques|
+802|137|F|156381.95|1995-01-05|1-URGENT|Clerk#000000516|0|posits. ironic, pending requests cajole. even theodol|
+803|16|O|27629.66|1997-04-29|5-LOW|Clerk#000000260|0|ic instructions. even deposits haggle furiously at the deposits-- regular de|
+804|50|F|94400.43|1993-03-12|3-MEDIUM|Clerk#000000931|0|s. blithely final foxes are about the packag|
+805|127|O|90042.41|1995-07-05|4-NOT SPECIFIED|Clerk#000000856|0|y according to the fluffily |
+806|131|O|26839.16|1996-06-20|2-HIGH|Clerk#000000240|0| the ironic packages wake carefully fina|
+807|145|F|222392.53|1993-11-24|3-MEDIUM|Clerk#000000012|0|refully special tithes. blithely regular accoun|
+832|29|F|68494.08|1992-04-19|5-LOW|Clerk#000000495|0|xes. bravely regular packages sleep up the furiously bold accou|
+833|56|F|49033.69|1994-02-13|3-MEDIUM|Clerk#000000437|0|ts haggle quickly across the slyl|
+834|43|F|46459.92|1994-05-23|3-MEDIUM|Clerk#000000805|0| sleep. quickly even foxes are boldly. slyly express requests use slyly|
+835|65|O|62430.67|1995-10-08|4-NOT SPECIFIED|Clerk#000000416|0|s about the carefully special foxes haggle quickly about the|
+836|70|O|72843.48|1996-11-25|4-NOT SPECIFIED|Clerk#000000729|0|ely bold excuses sleep regular ideas. furiously unusual ideas wake furiou|
+837|116|F|60918.41|1994-06-15|4-NOT SPECIFIED|Clerk#000000563|0|kages sleep slyly above the ironic, final orbits|
+838|17|O|82918.36|1998-01-29|5-LOW|Clerk#000000213|0| slyly around the slyly even|
+839|28|O|70182.63|1995-08-08|1-URGENT|Clerk#000000951|0|the carefully even platelets. furiously unusual fo|
+864|139|O|74710.74|1997-08-17|1-URGENT|Clerk#000000036|0|ly after the slyly regular deposits. express, regular asymptotes nag ca|
+865|4|F|70430.54|1993-05-04|3-MEDIUM|Clerk#000000337|0|. special packages wake after the carefully final accounts. express pinto be|
+866|40|F|4766.19|1992-11-28|3-MEDIUM|Clerk#000000718|0|ins after the even, even accounts nod blithel|
+867|26|F|7471.75|1993-11-16|3-MEDIUM|Clerk#000000877|0|pades nag quickly final, |
+868|104|F|127345.45|1992-06-09|4-NOT SPECIFIED|Clerk#000000782|0|onic theodolites print carefully. blithely dogge|
+869|136|O|58932.19|1997-01-12|2-HIGH|Clerk#000000245|0|ar sheaves are slowly. slyly even attainments boost theodolites. furiously|
+870|34|F|40492.37|1993-06-20|4-NOT SPECIFIED|Clerk#000000123|0|blithely ironic ideas nod. sly, r|
+871|16|O|172861.58|1995-11-15|5-LOW|Clerk#000000882|0|oss the ironic theodolites.|
+896|2|F|169847.63|1993-03-09|1-URGENT|Clerk#000000187|0|inal packages eat blithely according to the warhorses. furiously quiet de|
+897|49|P|57697.44|1995-03-20|1-URGENT|Clerk#000000316|0| wake quickly against |
+898|55|F|101020.75|1993-06-03|2-HIGH|Clerk#000000611|0|. unusual pinto beans haggle quickly across |
+899|109|O|125562.09|1998-04-08|5-LOW|Clerk#000000575|0|rts engage carefully final theodolites.|
+900|46|F|120073.51|1994-10-01|4-NOT SPECIFIED|Clerk#000000060|0| fluffily express deposits nag furiousl|
+901|13|O|81826.12|1998-07-21|4-NOT SPECIFIED|Clerk#000000929|0|lyly even foxes are furious, silent requests. requests about the quickly |
+902|10|F|37348.62|1994-07-27|4-NOT SPECIFIED|Clerk#000000811|0|yly final requests over the furiously regula|
+903|11|O|109351.87|1995-07-07|4-NOT SPECIFIED|Clerk#000000793|0|e slyly about the final pl|
+928|67|F|228136.49|1995-03-02|5-LOW|Clerk#000000450|0|ithely express pinto beans. |
+929|83|F|109301.02|1992-10-02|2-HIGH|Clerk#000000160|0|its. furiously even foxes affix carefully finally silent accounts. express req|
+930|131|F|199102.23|1994-12-17|1-URGENT|Clerk#000000004|0| accounts nag slyly. ironic, ironic accounts wake blithel|
+931|103|F|117909.23|1992-12-07|1-URGENT|Clerk#000000881|0|ss packages haggle furiously express, regular deposits. even, e|
+932|41|O|40234.50|1997-05-16|2-HIGH|Clerk#000000218|0|ly express instructions boost furiously reg|
+933|97|F|71349.30|1992-08-05|4-NOT SPECIFIED|Clerk#000000752|0|ial courts wake permanently against the furiously regular ideas. unusual |
+934|52|O|17213.59|1996-07-03|1-URGENT|Clerk#000000229|0|ts integrate carefully. sly, regular deposits af|
+935|50|O|97733.87|1997-09-24|5-LOW|Clerk#000000180|0|iously final deposits cajole. blithely even packages |
+960|35|F|63537.13|1994-09-21|3-MEDIUM|Clerk#000000120|0|regular accounts. requests|
+961|56|P|158893.16|1995-06-04|4-NOT SPECIFIED|Clerk#000000720|0|ons nag furiously among the quickl|
+962|37|F|98258.73|1994-05-06|5-LOW|Clerk#000000463|0|ments nag deposits. fluffily ironic a|
+963|26|F|53287.25|1994-05-26|3-MEDIUM|Clerk#000000497|0|uses haggle carefully. slyly even dependencies after the packages ha|
+964|76|O|131146.47|1995-05-20|3-MEDIUM|Clerk#000000657|0|print blithely ironic, careful theodolit|
+965|70|P|41758.44|1995-05-15|5-LOW|Clerk#000000218|0|iously special packages. slyly pending requests are carefully |
+966|14|O|120516.93|1998-04-30|2-HIGH|Clerk#000000239|0|special deposits. furious|
+967|110|F|179287.95|1992-06-21|3-MEDIUM|Clerk#000000167|0|excuses engage quickly bold dep|
+992|55|O|133665.12|1997-11-11|3-MEDIUM|Clerk#000000875|0|ts. regular pinto beans thrash carefully sl|
+993|80|O|198238.65|1995-09-10|3-MEDIUM|Clerk#000000894|0|quickly express accounts among the furiously bol|
+994|2|F|41433.48|1994-04-20|5-LOW|Clerk#000000497|0|ole. slyly bold excuses nag caref|
+995|116|P|135157.92|1995-05-31|3-MEDIUM|Clerk#000000439|0|deas. blithely final deposits play. express accounts wake blithely caref|
+996|71|O|47447.63|1997-12-29|1-URGENT|Clerk#000000497|0|arefully final packages into the slyly final requests affix blit|
+997|109|O|27561.82|1997-05-19|2-HIGH|Clerk#000000651|0|ly express depths. furiously final requests haggle furiously. carefu|
+998|32|F|65269.38|1994-11-26|4-NOT SPECIFIED|Clerk#000000956|0|ronic dolphins. ironic, bold ideas haggle furiously furious|
+999|61|F|145249.13|1993-09-05|5-LOW|Clerk#000000464|0|pitaphs sleep. regular accounts use. f|
+1024|4|O|176084.63|1997-12-23|5-LOW|Clerk#000000903|0| blithely. even, express theodolites cajole slyly across|
+1025|103|F|82034.03|1995-05-05|2-HIGH|Clerk#000000376|0|ross the slyly final pa|
+1026|73|O|36464.76|1997-06-04|5-LOW|Clerk#000000223|0|s wake blithely. special acco|
+1027|128|F|112770.89|1992-06-03|3-MEDIUM|Clerk#000000241|0|equests cajole. slyly final pinto bean|
+1028|70|F|153864.67|1994-01-01|2-HIGH|Clerk#000000131|0|ts are. final, silent deposits are among the fl|
+1029|130|F|47440.91|1994-06-21|2-HIGH|Clerk#000000700|0|quests sleep. slyly even foxes wake quickly final theodolites. clo|
+1030|134|F|16346.94|1994-06-15|5-LOW|Clerk#000000422|0|ully ironic accounts sleep carefully. requests are carefully alongside of the |
+1031|4|F|128024.71|1994-09-01|3-MEDIUM|Clerk#000000448|0|s; ironic theodolites along the carefully ex|
+1056|28|F|38446.39|1995-02-11|1-URGENT|Clerk#000000125|0|t, even deposits hang about the slyly special i|
+1057|76|F|108107.42|1992-02-20|1-URGENT|Clerk#000000124|0|cuses dazzle carefully careful, ironic pinto beans. carefully even theod|
+1058|53|F|89359.11|1993-04-26|3-MEDIUM|Clerk#000000373|0|kly pending courts haggle. blithely regular sheaves integrate carefully fi|
+1059|127|F|198360.22|1994-02-27|1-URGENT|Clerk#000000104|0|en accounts. carefully bold packages cajole daringly special depende|
+1060|140|F|121994.04|1993-02-21|3-MEDIUM|Clerk#000000989|0|l platelets sleep quickly slyly special requests. furiously |
+1061|103|O|166947.75|1998-05-15|5-LOW|Clerk#000000576|0|uests sleep at the packages. fur|
+1062|106|O|39805.04|1997-01-15|1-URGENT|Clerk#000000152|0|eposits use blithely |
+1063|37|F|41392.31|1994-04-02|2-HIGH|Clerk#000000024|0|deposits nag quickly regular deposits. quickl|
+1088|148|F|47120.41|1992-05-21|5-LOW|Clerk#000000347|0|counts are blithely. platelets print. carefully |
+1089|49|O|103192.74|1996-05-04|4-NOT SPECIFIED|Clerk#000000226|0|ns haggle ruthlessly. even requests are quickly abov|
+1090|19|O|32929.30|1997-11-15|2-HIGH|Clerk#000000300|0| furiously regular platelets haggle along the slyly unusual foxes! |
+1091|83|O|35795.22|1996-08-27|1-URGENT|Clerk#000000549|0| even pinto beans haggle quickly alongside of the eve|
+1092|124|P|85552.21|1995-03-04|3-MEDIUM|Clerk#000000006|0|re quickly along the blithe|
+1093|101|O|79189.58|1997-07-31|4-NOT SPECIFIED|Clerk#000000159|0| after the carefully ironic requests. carefully ironic packages wake fluffil|
+1094|145|O|9006.25|1997-12-24|3-MEDIUM|Clerk#000000570|0|beans affix furiously about the pending, even deposits. finally pendi|
+1095|145|O|178491.24|1995-08-22|3-MEDIUM|Clerk#000000709|0|sly bold requests cajole carefully according to|
+1120|140|O|107958.62|1997-11-07|3-MEDIUM|Clerk#000000319|0|lly special requests. slyly pending platelets are quickly pending requ|
+1121|29|O|241837.88|1997-01-13|3-MEDIUM|Clerk#000000541|0|r escapades. deposits above the fluffily bold requests hag|
+1122|121|O|179747.47|1997-01-10|1-URGENT|Clerk#000000083|0|uffily carefully final theodolites. furiously express packages affix|
+1123|73|O|93259.93|1996-08-03|3-MEDIUM|Clerk#000000929|0|uriously pending requests. slyly regular instruction|
+1124|80|O|141858.97|1998-07-30|5-LOW|Clerk#000000326|0|regular pinto beans along the fluffily silent packages|
+1125|25|F|80438.38|1994-10-27|2-HIGH|Clerk#000000510|0|ithely final requests. i|
+1126|145|O|59982.31|1998-01-28|4-NOT SPECIFIED|Clerk#000000928|0|d slyly regular ideas: special ideas believe slyly. slyly ironic sheaves w|
+1127|58|O|103320.91|1995-09-19|4-NOT SPECIFIED|Clerk#000000397|0|usly silent, regular pinto beans. blithely express requests boos|
+1152|49|F|51775.54|1994-08-14|4-NOT SPECIFIED|Clerk#000000496|0|equests. deposits ab|
+1153|121|O|220727.97|1996-04-18|5-LOW|Clerk#000000059|0| across the pending deposi|
+1154|37|F|192417.85|1992-02-15|1-URGENT|Clerk#000000268|0|old asymptotes are special requests. blithely even deposits sleep furiously|
+1155|149|O|126902.81|1997-10-06|2-HIGH|Clerk#000000164|0|c deposits haggle among the ironic, even requests. carefully ironic sheaves n|
+1156|133|O|217682.81|1996-10-19|1-URGENT|Clerk#000000200|0| blithely ironic dolphins. furiously pendi|
+1157|97|O|85394.06|1998-01-14|4-NOT SPECIFIED|Clerk#000000207|0|out the regular excuses boost carefully against the furio|
+1158|142|O|31075.51|1996-06-30|2-HIGH|Clerk#000000549|0|integrate slyly furiously ironic deposit|
+1159|70|F|55553.68|1992-09-18|3-MEDIUM|Clerk#000000992|0|ts may sleep. requests according to the|
+1184|89|O|39700.29|1997-10-26|5-LOW|Clerk#000000777|0|iously even packages haggle fluffily care|
+1185|74|F|47033.21|1992-08-24|5-LOW|Clerk#000000344|0| even escapades are. package|
+1186|59|O|82026.18|1996-08-15|4-NOT SPECIFIED|Clerk#000000798|0|ingly regular pinto beans: instructi|
+1187|134|F|85948.02|1992-11-20|3-MEDIUM|Clerk#000000047|0|s after the furiously final deposits boost slyly under the|
+1188|20|O|54655.07|1996-04-11|2-HIGH|Clerk#000000256|0|ully ironic deposits. slyl|
+1189|46|F|71017.99|1994-04-09|1-URGENT|Clerk#000000243|0|f the even accounts. courts print blithely ironic accounts. sile|
+1190|13|O|31043.39|1997-03-16|5-LOW|Clerk#000000575|0|ccounts above the foxes integrate carefully after the |
+1191|112|O|28623.04|1995-11-07|3-MEDIUM|Clerk#000000011|0|uests nag furiously. carefully even requests|
+1216|122|F|68056.57|1992-12-07|5-LOW|Clerk#000000918|0|nal foxes around the e|
+1217|7|F|40982.08|1992-04-26|4-NOT SPECIFIED|Clerk#000000538|0| foxes nag quickly. ironic excuses nod. blithely pending|
+1218|10|F|99834.47|1994-06-20|4-NOT SPECIFIED|Clerk#000000994|0|s cajole. special, silent deposits about the theo|
+1219|28|O|10163.56|1995-10-05|3-MEDIUM|Clerk#000000800|0|od carefully. slyly final dependencies across the even fray|
+1220|49|O|122157.14|1996-08-29|1-URGENT|Clerk#000000712|0|inal theodolites wake. fluffily ironic asymptotes cajol|
+1221|14|F|117397.16|1992-04-19|4-NOT SPECIFIED|Clerk#000000852|0| detect against the silent, even deposits. carefully ironic|
+1222|10|F|47623.94|1993-02-05|3-MEDIUM|Clerk#000000811|0|theodolites use quickly even accounts. carefully final asympto|
+1223|10|O|26714.67|1996-05-25|4-NOT SPECIFIED|Clerk#000000238|0|posits was blithely fr|
+1248|49|F|210713.88|1992-01-02|1-URGENT|Clerk#000000890|0|t the carefully regular dugouts. s|
+1249|149|F|45889.09|1994-01-05|1-URGENT|Clerk#000000095|0|al ideas sleep above the pending pin|
+1250|37|F|12907.62|1992-09-29|4-NOT SPECIFIED|Clerk#000000652|0|ts after the fluffily pending instructions use slyly about the s|
+1251|38|O|109536.55|1997-10-30|1-URGENT|Clerk#000000276|0|, brave sauternes. deposits boost fluffily.|
+1252|149|O|93403.05|1997-08-04|5-LOW|Clerk#000000348|0|ng the slyly regular excuses. special courts nag furiously blithely e|
+1253|115|F|92730.74|1993-01-26|1-URGENT|Clerk#000000775|0| requests sleep furiously even foxes. ruthless packag|
+1254|70|O|94649.25|1995-12-22|1-URGENT|Clerk#000000607|0| pinto beans. carefully regular request|
+1255|122|F|62518.31|1994-05-30|4-NOT SPECIFIED|Clerk#000000798|0|ct slyly regular accounts. quick|
+1280|97|F|91664.85|1993-01-11|5-LOW|Clerk#000000160|0|posits thrash quickly after the theodolites. furiously iro|
+1281|62|F|165454.51|1994-12-11|1-URGENT|Clerk#000000430|0|counts. carefully pending accounts eat |
+1282|116|F|61297.42|1992-02-29|4-NOT SPECIFIED|Clerk#000000168|0|he quickly special packages. furiously final re|
+1283|118|O|202623.92|1996-08-30|4-NOT SPECIFIED|Clerk#000000260|0| pinto beans boost slyly ac|
+1284|134|O|106122.38|1996-01-07|2-HIGH|Clerk#000000492|0|s. blithely silent deposits s|
+1285|11|F|139124.72|1992-06-01|1-URGENT|Clerk#000000423|0|cial deposits cajole after the ironic requests. p|
+1286|109|F|207291.83|1993-05-14|4-NOT SPECIFIED|Clerk#000000939|0| deposits use carefully from the excuses. slyly bold p|
+1287|19|F|131432.42|1994-07-05|2-HIGH|Clerk#000000288|0|ly ironic dolphins integrate furiously among the final packages. st|
+1312|112|F|58111.00|1994-05-19|3-MEDIUM|Clerk#000000538|0|n, express accounts across the ironic|
+1313|148|F|46598.65|1994-09-13|1-URGENT|Clerk#000000774|0|ld accounts. regular deposits cajole. ironically pending theodolites use car|
+1314|143|F|56207.66|1994-05-13|3-MEDIUM|Clerk#000000485|0|ickly blithe packages nod ideas. furiously bold braids boost around the car|
+1315|22|O|121935.23|1998-03-22|5-LOW|Clerk#000000840|0|final theodolites alongside of the carefu|
+1316|16|F|163746.47|1993-12-03|1-URGENT|Clerk#000000857|0|ully bold theodolites? pending, bold pin|
+1317|100|P|139714.71|1995-05-19|2-HIGH|Clerk#000000373|0|sts. furiously special deposits lose fur|
+1318|128|O|81663.65|1998-06-27|3-MEDIUM|Clerk#000000581|0|s hang bold requests. pending, re|
+1319|32|O|31103.83|1996-09-27|2-HIGH|Clerk#000000257|0|y across the ruthlessly ironic accounts. unusu|
+1344|17|F|43809.37|1992-04-16|5-LOW|Clerk#000000178|0|omise close, silent requests. pending theodolites boost pending |
+1345|95|F|111207.93|1992-10-28|5-LOW|Clerk#000000447|0| regular tithes. quickly fluffy de|
+1346|76|F|171975.62|1992-06-18|2-HIGH|Clerk#000000374|0|ges sleep quickly-- even pint|
+1347|41|O|173444.60|1997-06-20|5-LOW|Clerk#000000977|0|he furiously even foxes use carefully express req|
+1348|19|O|94135.77|1998-04-18|5-LOW|Clerk#000000206|0|tly. quickly even deposi|
+1349|64|O|46376.09|1997-10-26|1-URGENT|Clerk#000000543|0|yly! blithely special theodolites cajole. unusual, reg|
+1350|52|F|49305.98|1993-08-24|1-URGENT|Clerk#000000635|0|iously about the blithely special a|
+1351|106|O|24637.96|1998-04-20|1-URGENT|Clerk#000000012|0| cajole. regular, special re|
+1376|47|O|23984.88|1997-05-04|4-NOT SPECIFIED|Clerk#000000730|0|der furiously final, final frets. carefull|
+1377|20|O|108334.30|1998-04-24|4-NOT SPECIFIED|Clerk#000000625|0|lly across the blithely express accounts. ironic excuses promise carefully de|
+1378|20|O|118495.12|1996-03-09|4-NOT SPECIFIED|Clerk#000000705|0| furiously even tithes cajole slyly among the quick|
+1379|65|O|84627.76|1998-05-25|5-LOW|Clerk#000000861|0|y deposits are caref|
+1380|137|O|94969.41|1996-07-07|3-MEDIUM|Clerk#000000969|0|inal deposits wake slyly daringly even requests. bold, even foxe|
+1381|127|O|58212.22|1998-05-25|3-MEDIUM|Clerk#000000107|0|even requests breach after the bold, ironic instructions. slyly even|
+1382|133|F|173522.71|1993-08-17|5-LOW|Clerk#000000241|0|fully final packages sl|
+1383|121|F|34797.72|1993-04-27|2-HIGH|Clerk#000000785|0|ts. express requests sleep blithel|
+1408|55|O|183965.61|1997-12-26|4-NOT SPECIFIED|Clerk#000000942|0|t the quickly final asymptotes. unusual|
+1409|143|F|72440.52|1992-12-31|4-NOT SPECIFIED|Clerk#000000065|0|ructions. furiously unusual excuses are regular, unusual theodolites. fin|
+1410|113|O|114879.19|1997-04-12|5-LOW|Clerk#000000123|0|iously along the bravely regular dolphins. pinto beans cajole furiously sp|
+1411|95|F|164462.61|1994-12-21|2-HIGH|Clerk#000000566|0|s. furiously special excuses across the pending pinto beans haggle sp|
+1412|53|F|78676.54|1993-03-13|4-NOT SPECIFIED|Clerk#000000083|0|uffily daring theodolit|
+1413|91|O|75733.58|1997-06-14|3-MEDIUM|Clerk#000000342|0|, ironic instructions. carefully even packages dazzle|
+1414|77|O|38057.81|1995-08-16|1-URGENT|Clerk#000000883|0|ccounts. ironic foxes haggle car|
+1415|79|F|24654.79|1994-05-29|4-NOT SPECIFIED|Clerk#000000601|0|rays. blithely final ideas affix quickl|
+1440|98|O|50201.16|1995-08-10|5-LOW|Clerk#000000956|0| pending requests. closely s|
+1441|122|O|156477.94|1997-03-06|4-NOT SPECIFIED|Clerk#000000156|0|ter the excuses. ironic dependencies m|
+1442|112|F|7108.12|1994-07-05|4-NOT SPECIFIED|Clerk#000000935|0|nal pinto beans. slyly ironic ideas cajol|
+1443|44|O|44672.03|1996-12-16|5-LOW|Clerk#000000185|0|x blithely against the carefully final somas. even asymptotes are. quickly spe|
+1444|134|F|207907.60|1994-12-06|3-MEDIUM|Clerk#000000783|0|ove the bold accounts cajole fluffily about|
+1445|115|F|154653.32|1995-01-10|3-MEDIUM|Clerk#000000211|0|even packages wake fluffily |
+1446|41|O|27663.16|1998-02-16|5-LOW|Clerk#000000274|0|lly regular notornis above the requests sleep final accounts! |
+1447|91|F|108171.38|1992-10-15|2-HIGH|Clerk#000000880|0|inly against the blithely pending excuses. regular, pe|
+1472|149|O|65331.05|1996-10-06|5-LOW|Clerk#000000303|0|y special dolphins around the final dependencies wake quick|
+1473|94|O|80624.38|1997-03-17|3-MEDIUM|Clerk#000000960|0|furiously close accoun|
+1474|70|F|51697.18|1995-01-09|1-URGENT|Clerk#000000438|0|detect quickly above the carefully even |
+1475|5|O|185496.66|1997-11-12|2-HIGH|Clerk#000000972|0|cally final packages boost. blithely ironic packa|
+1476|145|O|18795.62|1996-06-27|2-HIGH|Clerk#000000673|0|ding accounts hinder alongside of the quickly pending requests. fluf|
+1477|76|O|231831.35|1997-08-24|5-LOW|Clerk#000000612|0|ly bold foxes. final ideas would cajo|
+1478|50|O|20791.50|1997-08-03|2-HIGH|Clerk#000000827|0|lessly. carefully express|
+1479|16|O|31471.04|1995-12-16|4-NOT SPECIFIED|Clerk#000000697|0|he furiously even foxes. thinly bold deposits|
+1504|2|F|89399.40|1992-08-28|3-MEDIUM|Clerk#000000381|0|, brave deposits. bold de|
+1505|37|F|55892.35|1992-08-21|2-HIGH|Clerk#000000544|0|s. slyly ironic packages cajole. carefully regular packages haggle |
+1506|148|F|195844.84|1992-09-21|3-MEDIUM|Clerk#000000620|0| dependencies. accounts affix blithely slowly unusual deposits. slyly regular |
+1507|121|F|96166.92|1993-10-14|3-MEDIUM|Clerk#000000305|0|stealthy, ironic de|
+1508|103|O|151282.65|1998-04-10|5-LOW|Clerk#000000117|0| after the furiously regular pinto beans hang slyly quickly ironi|
+1509|64|F|180455.98|1993-07-08|5-LOW|Clerk#000000770|0|the regular ideas. regul|
+1510|53|O|154590.05|1996-09-17|5-LOW|Clerk#000000128|0|ld carefully. furiously final asymptotes haggle furiously|
+1511|79|O|59651.38|1996-12-22|4-NOT SPECIFIED|Clerk#000000386|0|ts above the depend|
+1536|94|O|5184.26|1997-01-26|3-MEDIUM|Clerk#000000117|0|ges are! furiously final deposits cajole iron|
+1537|109|F|108317.51|1992-02-15|4-NOT SPECIFIED|Clerk#000000862|0|g to the even deposits. ironic, final packages |
+1538|29|O|179554.41|1995-06-18|4-NOT SPECIFIED|Clerk#000000258|0| instructions. regular theod|
+1539|112|F|39612.63|1995-03-10|5-LOW|Clerk#000000840|0|nstructions boost pa|
+1540|16|F|128014.15|1992-08-05|2-HIGH|Clerk#000000927|0|r ideas hinder blithe|
+1541|94|P|47286.32|1995-05-18|1-URGENT|Clerk#000000906|0|y. slyly ironic warhorses around the furiously regul|
+1542|143|F|132972.24|1993-09-15|3-MEDIUM|Clerk#000000435|0|t the furiously close deposits do was f|
+1543|52|O|139047.22|1997-02-20|1-URGENT|Clerk#000000398|0|unts. furiously pend|
+1568|17|O|76119.72|1997-01-30|4-NOT SPECIFIED|Clerk#000000554|0|d notornis. carefully |
+1569|104|O|87803.55|1998-04-02|5-LOW|Clerk#000000786|0|orbits. fluffily even decoys serve blithely. furiously furious realms nag acro|
+1570|124|O|35589.57|1998-03-16|1-URGENT|Clerk#000000745|0|pinto beans haggle furiousl|
+1571|103|F|151404.78|1992-12-05|2-HIGH|Clerk#000000565|0|ously furiously bold warthogs. slyly ironic instructions are quickly a|
+1572|11|O|47232.79|1996-02-24|2-HIGH|Clerk#000000994|0|fluffily ironic accounts haggle blithely final platelets! slyly regular foxes|
+1573|148|F|86918.57|1992-12-28|2-HIGH|Clerk#000000940|0|ess, ironic deposits use along the carefu|
+1574|134|O|179923.54|1996-12-12|3-MEDIUM|Clerk#000000809|0| ideas hinder after the carefully unusual |
+1575|145|O|197031.52|1995-09-13|3-MEDIUM|Clerk#000000497|0|. furiously regular dep|
+1600|94|F|130515.61|1993-03-03|3-MEDIUM|Clerk#000000627|0|tions cajole quietly above the regular, silent requests. slyly fin|
+1601|53|F|73962.95|1994-08-27|5-LOW|Clerk#000000469|0|ent deposits are ca|
+1602|1|F|4225.26|1993-08-05|5-LOW|Clerk#000000660|0|deposits. busily silent instructions haggle furiously. fin|
+1603|2|F|29305.47|1993-07-31|4-NOT SPECIFIED|Clerk#000000869|0|s. slyly silent deposits boo|
+1604|113|F|107139.29|1993-07-17|5-LOW|Clerk#000000512|0|lithely silent waters. blithely unusual packages alongside |
+1605|58|O|130687.64|1998-04-24|4-NOT SPECIFIED|Clerk#000000616|0|sleep furiously? ruthless, even pinto beans |
+1606|53|O|115877.40|1997-04-17|4-NOT SPECIFIED|Clerk#000000550|0|r requests. quickly even platelets breach before the ironically|
+1607|149|O|166335.03|1995-12-16|2-HIGH|Clerk#000000498|0| bold, pending foxes haggle. slyly silent |
+1632|67|O|183286.33|1997-01-08|3-MEDIUM|Clerk#000000351|0|onic requests are accounts. bold a|
+1633|16|O|52359.51|1995-10-14|2-HIGH|Clerk#000000666|0|y silent accounts sl|
+1634|70|O|145898.47|1996-09-10|1-URGENT|Clerk#000000360|0|arefully blithely ironic requests. slyly unusual instructions alongside|
+1635|4|O|70232.26|1997-02-13|3-MEDIUM|Clerk#000000958|0|s. slyly ironic requests affix slyly |
+1636|79|O|172021.87|1997-06-17|3-MEDIUM|Clerk#000000457|0|ding requests. slyly ironic courts wake quickl|
+1637|73|F|180912.15|1995-02-08|4-NOT SPECIFIED|Clerk#000000189|0| final accounts. blithely silent ideas cajole bravely. carefully express |
+1638|139|O|172436.30|1997-08-13|2-HIGH|Clerk#000000643|0|he fluffily regular asymp|
+1639|5|O|104166.56|1995-08-20|4-NOT SPECIFIED|Clerk#000000939|0|haggle furiously. final requests detect furious|
+1664|64|O|178060.22|1996-03-03|1-URGENT|Clerk#000000090|0|y quickly even asymptotes. furiously regular packages haggle quickly fin|
+1665|76|F|4819.91|1994-05-08|2-HIGH|Clerk#000000920|0|ly regular packages are fluffily even ideas. fluffily final|
+1666|95|O|128367.97|1995-10-18|1-URGENT|Clerk#000000849|0|ffily pending dependencies wake fluffily. pending, final accounts |
+1667|5|O|125030.37|1997-10-10|2-HIGH|Clerk#000000103|0|e accounts. slyly express accounts must are a|
+1668|142|O|137576.19|1997-07-12|4-NOT SPECIFIED|Clerk#000000148|0|eodolites. carefully dogged dolphins haggle q|
+1669|2|O|24362.39|1997-06-09|3-MEDIUM|Clerk#000000663|0|er ironic requests detect furiously blithely sp|
+1670|25|O|89999.72|1997-05-24|2-HIGH|Clerk#000000320|0|unusual dependencies. furiously special platelets main|
+1671|35|O|104391.11|1996-07-27|4-NOT SPECIFIED|Clerk#000000275|0|ly. slyly pending requests was above the |
+1696|4|O|102665.03|1998-01-08|4-NOT SPECIFIED|Clerk#000000041|0|bravely bold accounts above the quickly bold|
+1697|76|O|122621.31|1996-10-07|1-URGENT|Clerk#000000815|0|o x-ray blithely. pl|
+1698|40|O|141118.87|1997-04-23|2-HIGH|Clerk#000000432|0|slyly. carefully express deposit|
+1699|85|F|66408.29|1993-12-30|1-URGENT|Clerk#000000125|0|jole blithely. furiously un|
+1700|65|O|89143.36|1996-06-15|3-MEDIUM|Clerk#000000328|0|ely final dolphins wake sometimes above the quietly regular deposits. fur|
+1701|130|F|72835.95|1992-05-19|2-HIGH|Clerk#000000395|0|furiously. regular, close theodoli|
+1702|67|P|194119.31|1995-05-07|2-HIGH|Clerk#000000300|0|around the carefully final deposits cajole carefully according to the b|
+1703|134|F|121220.59|1993-01-28|3-MEDIUM|Clerk#000000463|0| pinto beans poach. bold courts boost. regular, express deposits at|
+1728|64|O|131604.34|1996-05-22|2-HIGH|Clerk#000000711|0|beans. slyly regular instructions sleep! slyly final packages|
+1729|133|F|12137.76|1992-05-19|2-HIGH|Clerk#000000158|0|pending foxes wake. accounts|
+1730|124|O|150886.49|1998-07-24|5-LOW|Clerk#000000794|0| fluffily pending deposits serve. furiously even requests wake furiou|
+1731|128|O|190490.78|1996-01-06|1-URGENT|Clerk#000000268|0|lithely regular, final instructions. ironic, express packages are above|
+1732|146|F|179854.51|1993-11-29|5-LOW|Clerk#000000903|0|inal requests integrate dolph|
+1733|148|O|165489.52|1996-05-12|2-HIGH|Clerk#000000789|0|e carefully according to the accounts. furiously pending instructions sleep|
+1734|7|F|44002.53|1994-06-11|2-HIGH|Clerk#000000722|0| final ideas haggle. blithely quick foxes sleep busily bold ideas. i|
+1735|22|F|98541.95|1992-12-27|1-URGENT|Clerk#000000458|0|ully idle requests wake qu|
+1760|115|O|82151.12|1996-05-17|5-LOW|Clerk#000000917|0| deposits. busily regular deposits wake blithely along the furiously even re|
+1761|106|F|211925.95|1993-12-24|2-HIGH|Clerk#000000817|0|efully slyly bold frets. packages boost b|
+1762|77|F|202227.17|1994-08-20|4-NOT SPECIFIED|Clerk#000000653|0|ly ironic packages. furi|
+1763|121|O|140685.01|1996-10-29|2-HIGH|Clerk#000000321|0|es. bold dependencies haggle furiously along |
+1764|29|F|47384.71|1992-03-25|1-URGENT|Clerk#000000182|0|. slyly final packages integrate carefully acro|
+1765|73|O|36551.43|1995-12-03|4-NOT SPECIFIED|Clerk#000000490|0| regular excuses wake slyly|
+1766|139|O|41032.81|1996-10-12|2-HIGH|Clerk#000000983|0|unusual deposits affix quickly beyond the carefully s|
+1767|25|P|136582.60|1995-03-14|2-HIGH|Clerk#000000327|0|eposits use carefully carefully regular platelets. quickly regular packages al|
+1792|49|F|107919.86|1993-11-09|5-LOW|Clerk#000000102|0|ructions haggle along the pending packages. carefully speci|
+1793|19|F|82504.56|1992-07-12|4-NOT SPECIFIED|Clerk#000000291|0|regular packages cajole. blithely special packages according to the final d|
+1794|140|O|179462.21|1997-09-28|1-URGENT|Clerk#000000686|0|ally silent pinto beans. regular package|
+1795|94|F|146849.33|1994-03-19|2-HIGH|Clerk#000000815|0| quickly final packages! blithely dogged accounts c|
+1796|47|F|33755.47|1992-11-21|2-HIGH|Clerk#000000245|0|eans use furiously around th|
+1797|125|O|51494.47|1996-05-07|3-MEDIUM|Clerk#000000508|0|quiet platelets haggle since the quickly ironic instructi|
+1798|52|O|46393.97|1997-07-28|1-URGENT|Clerk#000000741|0|al foxes are blithe|
+1799|61|F|46815.93|1994-03-07|4-NOT SPECIFIED|Clerk#000000339|0|ns sleep furiously final waters. blithely regular instructions h|
+1824|49|F|81351.53|1994-05-05|1-URGENT|Clerk#000000972|0|e blithely fluffily|
+1825|148|F|150582.77|1993-12-05|3-MEDIUM|Clerk#000000345|0|ironic, final accou|
+1826|82|F|124719.97|1992-04-16|4-NOT SPECIFIED|Clerk#000000718|0|the even asymptotes dazzle fluffily slyly regular asymptotes. final, unu|
+1827|106|O|210113.88|1996-06-22|4-NOT SPECIFIED|Clerk#000000369|0|luffily even requests haggle sly|
+1828|32|F|137369.50|1994-04-18|3-MEDIUM|Clerk#000000840|0|y quickly bold packag|
+1829|112|F|127532.20|1994-05-08|2-HIGH|Clerk#000000537|0| accounts wake above the furiously unusual requests. pending package|
+1830|133|F|85122.24|1995-02-23|1-URGENT|Clerk#000000045|0|according to the even,|
+1831|71|F|58032.77|1993-12-02|1-URGENT|Clerk#000000854|0| accounts. carefully even accounts boost furiously. regular ideas engage. |
+1856|106|F|189361.42|1992-03-20|4-NOT SPECIFIED|Clerk#000000952|0|. special pinto beans run acr|
+1857|133|F|102793.59|1993-01-13|2-HIGH|Clerk#000000083|0|hely final ideas slee|
+1858|143|O|30457.91|1997-12-13|1-URGENT|Clerk#000000389|0|thely. slyly final deposits sleep|
+1859|61|O|105094.09|1997-04-11|4-NOT SPECIFIED|Clerk#000000949|0| the foxes. bravely special excuses nag carefully special r|
+1860|10|O|9103.40|1996-04-04|3-MEDIUM|Clerk#000000556|0|osits. quickly bold deposits according to |
+1861|70|F|95063.41|1994-01-03|3-MEDIUM|Clerk#000000847|0|r the fluffily close sauternes. furio|
+1862|34|O|97981.06|1998-02-24|5-LOW|Clerk#000000348|0|ts snooze ironically abou|
+1863|74|F|96359.65|1993-09-23|4-NOT SPECIFIED|Clerk#000000658|0|old sentiments. careful, |
+1888|121|F|224724.11|1993-10-31|4-NOT SPECIFIED|Clerk#000000659|0|olites. pinto beans cajole. regular deposits affix. slyly regular|
+1889|25|O|96431.77|1997-03-16|1-URGENT|Clerk#000000854|0|p around the regular notornis. unusual deposits|
+1890|10|O|202364.58|1996-12-18|4-NOT SPECIFIED|Clerk#000000627|0|romise final, regular deposits. regular fox|
+1891|61|F|76848.96|1994-12-15|5-LOW|Clerk#000000495|0|unusual foxes sleep regular deposits. requests wake special pac|
+1892|25|F|133273.64|1994-03-26|5-LOW|Clerk#000000733|0|sts. slyly regular dependencies use slyly. ironic, spec|
+1893|125|O|116792.13|1997-10-30|2-HIGH|Clerk#000000111|0|olites. silent, special deposits eat slyly quickly express packages; hockey p|
+1894|76|F|44387.23|1992-03-30|1-URGENT|Clerk#000000626|0|e furiously. furiously even accounts are slyly final accounts. closely speci|
+1895|7|F|44429.81|1994-05-30|3-MEDIUM|Clerk#000000878|0|ress accounts. bold accounts cajole. slyly final pinto beans poach regul|
+1920|110|O|119605.91|1998-06-24|5-LOW|Clerk#000000018|0|hely; furiously regular excuses|
+1921|88|F|57584.12|1994-01-18|3-MEDIUM|Clerk#000000293|0|counts. slyly quiet requests along the ruthlessly regular accounts are |
+1922|56|O|11575.77|1996-07-13|3-MEDIUM|Clerk#000000984|0|side of the blithely final re|
+1923|136|O|171128.10|1997-07-07|1-URGENT|Clerk#000000471|0| express dolphins. |
+1924|76|O|169756.19|1996-09-07|4-NOT SPECIFIED|Clerk#000000823|0| of the ironic accounts. instructions near the final instr|
+1925|17|F|146382.71|1992-03-05|1-URGENT|Clerk#000000986|0|e slyly regular deposits. furiously |
+1926|94|O|100035.03|1996-01-31|2-HIGH|Clerk#000000568|0|cajole. even warhorses sleep carefully. |
+1927|140|O|23327.88|1995-09-30|3-MEDIUM|Clerk#000000616|0|riously special packages. permanent pearls wake furiously. even packages alo|
+1952|67|F|12896.25|1994-03-16|2-HIGH|Clerk#000000254|0| silent accounts boost |
+1953|149|F|57213.18|1993-11-30|3-MEDIUM|Clerk#000000891|0| fluffily along the quickly even packages. |
+1954|56|O|158853.63|1997-05-31|4-NOT SPECIFIED|Clerk#000000104|0| unusual excuses cajole according to the blithely regular theodolites.|
+1955|13|F|103085.13|1992-04-20|1-URGENT|Clerk#000000792|0|ly special ideas. sometimes final |
+1956|127|F|88704.26|1992-09-20|4-NOT SPECIFIED|Clerk#000000600|0|ironic ideas are silent ideas. furiously final deposits sleep slyly carefu|
+1957|31|O|77482.87|1998-07-21|2-HIGH|Clerk#000000639|0|nding excuses about the |
+1958|53|O|176294.34|1995-09-22|5-LOW|Clerk#000000343|0| haggle blithely. flu|
+1959|43|O|62277.18|1997-01-13|4-NOT SPECIFIED|Clerk#000000631|0| cajole about the blithely express requests. even excuses mold bl|
+1984|52|O|79230.47|1998-04-01|1-URGENT|Clerk#000000416|0| slyly special instructions. unusual foxes use packages. carefully regular req|
+1985|7|F|171522.54|1994-09-02|4-NOT SPECIFIED|Clerk#000000741|0|slyly slyly even pains. slyly reg|
+1986|149|F|34269.96|1994-05-05|2-HIGH|Clerk#000000609|0|across the theodolites. quick|
+1987|100|F|6406.29|1994-04-30|2-HIGH|Clerk#000000652|0|gular platelets alongside |
+1988|109|O|117132.72|1995-10-06|4-NOT SPECIFIED|Clerk#000000011|0|ly ironic dolphins serve quickly busy accounts. bu|
+1989|118|F|39263.28|1994-03-16|4-NOT SPECIFIED|Clerk#000000747|0|ely bold pinto beans ha|
+1990|119|F|48781.39|1994-12-16|2-HIGH|Clerk#000000114|0|e bold patterns. always regul|
+1991|19|F|139854.41|1992-09-07|4-NOT SPECIFIED|Clerk#000000854|0|ing accounts can haggle at the carefully final Tiresias-- pending, regular|
+2016|8|O|24347.36|1996-08-16|3-MEDIUM|Clerk#000000641|0|the carefully ironic foxes. requests nag bold, r|
+2017|101|O|70529.27|1998-05-13|3-MEDIUM|Clerk#000000427|0|nusual requests. blit|
+2018|19|P|25007.95|1995-04-05|4-NOT SPECIFIED|Clerk#000000920|0|gular accounts wake fur|
+2019|136|F|43789.14|1992-10-23|1-URGENT|Clerk#000000565|0| furiously bold packages. fluffily fi|
+2020|73|F|136162.13|1993-06-21|3-MEDIUM|Clerk#000000192|0|es. furiously regular packages above the furiously special theodolites are a|
+2021|70|O|27016.74|1995-07-15|1-URGENT|Clerk#000000155|0|ong the furiously regular requests. unusual deposits wake fluffily inside|
+2022|62|F|206742.11|1992-03-15|1-URGENT|Clerk#000000268|0| dependencies sleep fluffily even, ironic deposits. express, silen|
+2023|118|F|144123.37|1992-05-06|5-LOW|Clerk#000000137|0|ular courts engage according to the|
+2048|17|F|33401.77|1993-11-15|1-URGENT|Clerk#000000934|0|s cajole after the blithely final accounts. f|
+2049|31|O|153048.74|1995-12-07|2-HIGH|Clerk#000000859|0|ly regular requests thrash blithely about the fluffily even theodolites. r|
+2050|28|F|208517.98|1994-06-02|4-NOT SPECIFIED|Clerk#000000821|0|d accounts against the furiously regular packages use bli|
+2051|40|O|87988.34|1996-03-18|4-NOT SPECIFIED|Clerk#000000333|0|ctions sleep blithely. blithely regu|
+2052|91|F|141822.19|1992-04-13|2-HIGH|Clerk#000000767|0| requests sleep around the even, even courts. ironic theodolites affix furious|
+2053|142|F|125125.57|1995-02-07|1-URGENT|Clerk#000000717|0|ar requests: blithely sly accounts boost carefully across t|
+2054|41|F|144335.16|1992-06-08|4-NOT SPECIFIED|Clerk#000000103|0|l requests affix carefully about the furiously special|
+2055|97|F|57092.26|1993-09-04|1-URGENT|Clerk#000000067|0|. warhorses affix slyly blithely express instructions? fur|
+2080|95|F|45767.69|1993-06-18|5-LOW|Clerk#000000190|0|ironic, pending theodolites are carefully about the quickly regular theodolite|
+2081|121|O|145654.97|1997-07-05|2-HIGH|Clerk#000000136|0|ong the regular theo|
+2082|49|F|46753.63|1995-01-10|2-HIGH|Clerk#000000354|0|cial accounts. ironic, express dolphins nod slyly sometimes final reques|
+2083|101|F|31795.52|1993-07-14|3-MEDIUM|Clerk#000000361|0|al patterns. bold, final foxes nag bravely about the furiously express|
+2084|80|F|190652.53|1993-03-17|2-HIGH|Clerk#000000048|0|zle furiously final, careful packages. slyly ironic ideas amo|
+2085|49|F|45311.07|1993-11-21|3-MEDIUM|Clerk#000000818|0|ress, express ideas haggle|
+2086|142|F|188985.18|1994-10-19|1-URGENT|Clerk#000000046|0| permanently regular|
+2087|50|O|53581.41|1998-01-31|2-HIGH|Clerk#000000626|0|e always regular packages nod against the furiously spec|
+2112|64|O|17986.15|1997-02-05|2-HIGH|Clerk#000000351|0|against the slyly even id|
+2113|32|O|65678.21|1997-11-08|2-HIGH|Clerk#000000527|0|slyly regular instruct|
+2114|79|F|106446.02|1995-01-16|5-LOW|Clerk#000000751|0|r, unusual accounts haggle across the busy platelets. carefully |
+2115|106|O|134814.65|1998-05-23|4-NOT SPECIFIED|Clerk#000000101|0|odolites boost. carefully regular excuses cajole. quickly ironic pinto be|
+2116|23|F|60887.90|1994-08-26|1-URGENT|Clerk#000000197|0|efully after the asymptotes. furiously sp|
+2117|22|O|145713.03|1997-04-26|2-HIGH|Clerk#000000887|0|ely even dependencies. regular foxes use blithely.|
+2118|134|O|38974.67|1996-10-09|1-URGENT|Clerk#000000196|0|ial requests wake carefully special packages. f|
+2119|64|O|34632.57|1996-08-20|2-HIGH|Clerk#000000434|0|uickly pending escapades. fluffily ir|
+2144|136|F|119917.28|1994-03-29|3-MEDIUM|Clerk#000000546|0|t. carefully quick requests across the deposits wake regu|
+2145|134|F|18885.35|1992-10-03|1-URGENT|Clerk#000000886|0|sts would snooze blithely alongside of th|
+2146|118|F|179686.07|1992-09-14|4-NOT SPECIFIED|Clerk#000000476|0|ven packages. dependencies wake slyl|
+2147|100|F|91513.79|1992-09-06|4-NOT SPECIFIED|Clerk#000000424|0| haggle carefully furiously final foxes. pending escapades thrash. bold theod|
+2148|130|F|19612.03|1995-04-19|4-NOT SPECIFIED|Clerk#000000517|0|ross the furiously unusual theodolites. always expre|
+2149|101|F|105145.40|1993-03-13|5-LOW|Clerk#000000555|0|nusual accounts nag furiously special reques|
+2150|82|F|166961.06|1994-06-03|3-MEDIUM|Clerk#000000154|0|ect slyly against the even, final packages. quickly regular pinto beans wake c|
+2151|58|O|124608.69|1996-11-11|3-MEDIUM|Clerk#000000996|0|c requests. ironic platelets cajole across the quickly fluffy deposits.|
+2176|104|F|87248.17|1992-11-10|1-URGENT|Clerk#000000195|0|s haggle regularly accor|
+2177|136|O|183493.42|1997-01-20|3-MEDIUM|Clerk#000000161|0|ove the blithely unusual packages cajole carefully fluffily special request|
+2178|8|O|79594.68|1996-12-12|3-MEDIUM|Clerk#000000656|0|thely according to the instructions. furious|
+2179|41|O|77487.09|1996-09-07|2-HIGH|Clerk#000000935|0|ounts alongside of the furiously unusual braids cajol|
+2180|76|O|208481.57|1996-09-14|4-NOT SPECIFIED|Clerk#000000650|0|xpress, unusual pains. furiously ironic excu|
+2181|76|O|100954.64|1995-09-13|3-MEDIUM|Clerk#000000814|0|y against the ironic, even|
+2182|23|F|116003.11|1994-04-05|2-HIGH|Clerk#000000071|0|ccounts. quickly bold deposits across the excuses sl|
+2183|113|O|49841.12|1996-06-22|1-URGENT|Clerk#000000287|0| among the express, ironic packages. slyly ironic platelets integrat|
+2208|68|P|245388.06|1995-05-01|4-NOT SPECIFIED|Clerk#000000900|0|symptotes wake slyly blithely unusual packages.|
+2209|91|F|129086.93|1992-07-10|2-HIGH|Clerk#000000056|0|er above the slyly silent requests. furiously reg|
+2210|32|F|31689.46|1992-01-16|2-HIGH|Clerk#000000941|0| believe carefully quickly express pinto beans. deposi|
+2211|92|F|140031.23|1994-06-30|2-HIGH|Clerk#000000464|0|ffily bold courts e|
+2212|118|F|17231.05|1994-03-23|3-MEDIUM|Clerk#000000954|0|structions above the unusual requests use fur|
+2213|122|F|146136.10|1993-01-15|4-NOT SPECIFIED|Clerk#000000598|0|osits are carefully reg|
+2214|115|O|150345.63|1998-05-05|3-MEDIUM|Clerk#000000253|0|packages. fluffily even accounts haggle blithely. carefully ironic depen|
+2215|40|O|108239.46|1996-06-16|4-NOT SPECIFIED|Clerk#000000817|0|le final, final foxes. quickly regular gifts are carefully deposit|
+2240|56|F|174090.30|1992-03-06|4-NOT SPECIFIED|Clerk#000000622|0|accounts against the slyly express foxes are after the slyly regular |
+2241|103|F|165219.08|1993-05-11|1-URGENT|Clerk#000000081|0|y about the silent excuses. furiously ironic instructions along the sil|
+2242|82|O|15082.82|1997-07-20|4-NOT SPECIFIED|Clerk#000000360|0| pending multipliers. carefully express asymptotes use quickl|
+2243|49|O|10451.97|1995-06-10|2-HIGH|Clerk#000000813|0|ously regular deposits integrate s|
+2244|127|F|21207.08|1993-01-09|1-URGENT|Clerk#000001000|0|ckages. ironic, ironic accounts haggle blithely express excuses. |
+2245|58|F|150585.73|1993-04-28|3-MEDIUM|Clerk#000000528|0|ake carefully. braids haggle slyly quickly b|
+2246|113|O|85755.84|1996-05-27|4-NOT SPECIFIED|Clerk#000000739|0| final gifts sleep |
+2247|95|F|13491.31|1992-08-02|4-NOT SPECIFIED|Clerk#000000947|0|furiously regular packages. final brai|
+2272|139|F|127934.71|1993-04-13|2-HIGH|Clerk#000000449|0|s. bold, ironic pinto beans wake. silently specia|
+2273|136|O|142291.79|1996-12-14|5-LOW|Clerk#000000155|0|uickly express foxes haggle quickly against|
+2274|104|F|58273.89|1993-09-04|4-NOT SPECIFIED|Clerk#000000258|0|nstructions try to hag|
+2275|149|F|37398.90|1992-10-22|4-NOT SPECIFIED|Clerk#000000206|0| furiously furious platelets. slyly final packa|
+2276|43|O|141159.63|1996-04-29|4-NOT SPECIFIED|Clerk#000000821|0|ecial requests. fox|
+2277|89|F|79270.23|1995-01-02|4-NOT SPECIFIED|Clerk#000000385|0|accounts cajole. even i|
+2278|142|O|101878.46|1998-04-25|3-MEDIUM|Clerk#000000186|0|r pinto beans integrate after the carefully even deposits. blit|
+2279|80|F|142322.33|1993-02-23|3-MEDIUM|Clerk#000000898|0|de of the quickly unusual instructio|
+2304|46|F|93769.28|1994-01-07|4-NOT SPECIFIED|Clerk#000000415|0|onic platelets. ironic packages haggle. packages nag doggedly according to|
+2305|43|F|122964.66|1993-01-26|2-HIGH|Clerk#000000440|0|ove the furiously even acco|
+2306|28|O|244704.23|1995-07-26|2-HIGH|Clerk#000000975|0| wake furiously requests. permanent requests affix. final packages caj|
+2307|106|F|59417.76|1993-06-29|5-LOW|Clerk#000000952|0|furiously even asymptotes? carefully regular accounts|
+2308|25|F|58546.02|1992-10-25|4-NOT SPECIFIED|Clerk#000000609|0|ts. slyly final depo|
+2309|100|O|146933.07|1995-09-04|5-LOW|Clerk#000000803|0|he carefully pending packages. fluffily stealthy foxes engage carefully|
+2310|31|O|82928.12|1996-09-20|5-LOW|Clerk#000000917|0|wake carefully. unusual instructions nag ironic, regular excuse|
+2311|73|P|153233.93|1995-05-02|2-HIGH|Clerk#000000761|0|ly pending asymptotes-- furiously bold excus|
+2336|142|O|22294.51|1996-01-07|4-NOT SPECIFIED|Clerk#000000902|0|c, final excuses sleep furiously among the even theodolites. f|
+2337|142|O|45704.96|1997-06-18|4-NOT SPECIFIED|Clerk#000000754|0| quickly. final accounts haggle. carefully final acco|
+2338|140|O|28155.92|1997-09-15|2-HIGH|Clerk#000000951|0|riously final dugouts. final, ironic packages wake express, ironic id|
+2339|109|F|63470.78|1993-12-15|5-LOW|Clerk#000000847|0| against the regular |
+2340|65|O|30778.78|1996-01-12|1-URGENT|Clerk#000000964|0|ter the deposits sleep according to the slyly regular packages. carefully |
+2341|82|F|55950.21|1993-05-30|5-LOW|Clerk#000000443|0|sts-- blithely bold dolphins through the deposits nag blithely carefully re|
+2342|37|O|104038.78|1996-06-09|1-URGENT|Clerk#000000615|0|oost carefully across the regular accounts. blithely final d|
+2343|73|O|85381.00|1995-08-21|3-MEDIUM|Clerk#000000170|0|fluffily over the slyly special deposits. quickl|
+2368|13|F|101240.96|1993-08-20|1-URGENT|Clerk#000000830|0|t the bold instructions. carefully unusual |
+2369|110|O|73517.91|1996-12-24|2-HIGH|Clerk#000000752|0|iously even requests are dogged, express |
+2370|142|F|73924.21|1994-01-17|1-URGENT|Clerk#000000231|0|lyly final packages. quickly final deposits haggl|
+2371|19|O|193857.67|1998-01-07|1-URGENT|Clerk#000000028|0|ckages haggle at th|
+2372|31|O|104927.66|1997-11-21|5-LOW|Clerk#000000342|0|s: deposits haggle along the final ideas. careful|
+2373|28|F|55211.04|1994-03-12|4-NOT SPECIFIED|Clerk#000000306|0| even, special courts grow quickly. pending,|
+2374|4|F|115219.88|1993-10-29|4-NOT SPECIFIED|Clerk#000000081|0| blithely regular packages. blithely unusua|
+2375|5|O|106612.48|1996-11-20|3-MEDIUM|Clerk#000000197|0|unusual, pending theodolites cajole carefully |
+2400|37|O|92798.66|1998-07-25|5-LOW|Clerk#000000782|0|nusual courts nag against the carefully unusual pinto b|
+2401|148|O|88448.24|1997-07-29|4-NOT SPECIFIED|Clerk#000000531|0|ully unusual instructions boost carefully silently regular requests. |
+2402|67|O|70403.62|1996-09-06|4-NOT SPECIFIED|Clerk#000000162|0|slyly final sheaves sleep slyly. q|
+2403|55|O|111020.79|1998-04-11|3-MEDIUM|Clerk#000000820|0|furiously regular deposits use. furiously unusual accounts wake along the |
+2404|77|O|109077.69|1997-03-13|4-NOT SPECIFIED|Clerk#000000409|0|deposits breach furiously. ironic foxes haggle carefully bold packag|
+2405|73|O|115929.14|1996-12-23|3-MEDIUM|Clerk#000000535|0|ular, regular asympto|
+2406|7|O|182516.77|1996-10-28|5-LOW|Clerk#000000561|0|blithely regular accounts u|
+2407|55|O|112843.52|1998-06-19|2-HIGH|Clerk#000000068|0|uests affix slyly among the slyly regular depos|
+2432|103|O|62661.93|1996-07-13|1-URGENT|Clerk#000000115|0|re. slyly even deposits wake bra|
+2433|31|F|147071.86|1994-08-22|4-NOT SPECIFIED|Clerk#000000324|0|ess patterns are slyly. packages haggle carefu|
+2434|25|O|123956.25|1997-04-27|3-MEDIUM|Clerk#000000190|0|s. quickly ironic dolphins impress final deposits. blithel|
+2435|73|F|122490.66|1993-02-21|5-LOW|Clerk#000000112|0|es are carefully along the carefully final instructions. pe|
+2436|125|O|73990.08|1995-09-11|4-NOT SPECIFIED|Clerk#000000549|0|arefully. blithely bold deposits affix special accounts. final foxes nag. spe|
+2437|85|F|143411.69|1993-04-21|4-NOT SPECIFIED|Clerk#000000578|0|. theodolites wake slyly-- ironic, pending platelets above the carefully exp|
+2438|13|F|214494.39|1993-07-15|2-HIGH|Clerk#000000744|0|the final, regular warhorses. regularly |
+2439|55|O|41811.12|1997-03-15|2-HIGH|Clerk#000000819|0|lithely after the car|
+2464|145|O|30495.65|1997-11-23|5-LOW|Clerk#000000633|0|le about the instructions. courts wake carefully even|
+2465|34|O|180737.75|1995-06-24|1-URGENT|Clerk#000000078|0|al pinto beans. final, bold packages wake quickly|
+2466|19|F|161625.50|1994-03-06|1-URGENT|Clerk#000000424|0|c pinto beans. express deposits wake quickly. even, final courts nag. package|
+2467|35|O|7231.91|1995-07-16|4-NOT SPECIFIED|Clerk#000000914|0|pades sleep furiously. sometimes regular packages again|
+2468|112|O|160627.01|1997-06-09|4-NOT SPECIFIED|Clerk#000000260|0|ickly regular packages. slyly ruthless requests snooze quickly blithe|
+2469|124|O|192074.23|1996-11-26|5-LOW|Clerk#000000730|0| sleep closely regular instructions. furiously ironic instructi|
+2470|58|O|104966.33|1997-04-19|3-MEDIUM|Clerk#000000452|0|to the furiously final packages? pa|
+2471|89|O|34936.31|1998-03-12|4-NOT SPECIFIED|Clerk#000000860|0|carefully blithely regular pac|
+2496|136|F|140390.60|1994-01-09|2-HIGH|Clerk#000000142|0|slyly. pending instructions sleep. quic|
+2497|47|F|171326.48|1992-08-27|1-URGENT|Clerk#000000977|0|ily ironic pinto beans. furiously final platelets alongside of t|
+2498|97|F|45514.27|1993-11-08|5-LOW|Clerk#000000373|0|g the slyly special pinto beans. |
+2499|121|O|147243.86|1995-09-24|1-URGENT|Clerk#000000277|0|r the quickly bold foxes. bold instructi|
+2500|133|F|131122.82|1992-08-15|2-HIGH|Clerk#000000447|0|integrate slyly pending deposits. furiously ironic accounts across the s|
+2501|67|O|79380.51|1997-05-25|5-LOW|Clerk#000000144|0|ickly special theodolite|
+2502|70|F|33470.40|1993-05-28|4-NOT SPECIFIED|Clerk#000000914|0|lyly: carefully pending ideas affix again|
+2503|7|F|183671.08|1993-06-20|3-MEDIUM|Clerk#000000294|0|ly even packages was. ironic, regular deposits unwind furiously across the p|
+2528|55|F|92069.62|1994-11-20|1-URGENT|Clerk#000000789|0|ular dependencies? regular frays kindle according to the blith|
+2529|136|O|4104.30|1996-08-20|2-HIGH|Clerk#000000511|0|posits across the silent instructions wake blithely across |
+2530|128|F|58853.11|1994-03-21|3-MEDIUM|Clerk#000000291|0|ular instructions about the quic|
+2531|44|O|143212.85|1996-05-06|4-NOT SPECIFIED|Clerk#000000095|0|even accounts. furiously ironic excuses sleep fluffily. carefully silen|
+2532|94|O|116093.49|1995-10-11|2-HIGH|Clerk#000000498|0|the blithely pending accounts. regular, regular excuses boost aro|
+2533|50|O|168495.03|1997-03-24|1-URGENT|Clerk#000000594|0|ecial instructions. spec|
+2534|76|O|202784.54|1996-07-17|3-MEDIUM|Clerk#000000332|0|packages cajole ironic requests. furiously regular|
+2535|121|F|67018.30|1993-05-25|5-LOW|Clerk#000000296|0|phins cajole beneath the fluffily express asymptotes. c|
+2560|131|F|153426.79|1992-09-05|1-URGENT|Clerk#000000538|0|atelets; quickly sly requests|
+2561|58|O|137473.58|1997-11-14|1-URGENT|Clerk#000000861|0|ual requests. unusual deposits cajole furiously pending, regular platelets. |
+2562|10|F|136360.37|1992-08-01|1-URGENT|Clerk#000000467|0|elets. pending dolphins promise slyly. bo|
+2563|62|F|168952.10|1993-11-19|4-NOT SPECIFIED|Clerk#000000150|0|sly even packages after the furio|
+2564|77|F|3967.47|1994-09-09|2-HIGH|Clerk#000000718|0|usly regular pinto beans. orbits wake carefully. slyly e|
+2565|56|O|204438.57|1998-02-28|3-MEDIUM|Clerk#000000032|0|x-ray blithely along|
+2566|86|F|89992.48|1992-10-10|3-MEDIUM|Clerk#000000414|0|ructions boost bold ideas. idly ironic accounts use according to th|
+2567|70|O|263411.29|1998-02-27|2-HIGH|Clerk#000000031|0|detect. furiously ironic requests|
+2592|101|F|8225.96|1993-03-05|4-NOT SPECIFIED|Clerk#000000524|0|ts nag fluffily. quickly stealthy theodolite|
+2593|92|F|134726.09|1993-09-04|2-HIGH|Clerk#000000468|0|r the carefully final|
+2594|79|F|94866.39|1992-12-17|1-URGENT|Clerk#000000550|0|ests. theodolites above the blithely even accounts detect furio|
+2595|74|O|173130.20|1995-12-14|4-NOT SPECIFIED|Clerk#000000222|0|arefully ironic requests nag carefully ideas. |
+2596|43|O|74940.13|1996-08-17|1-URGENT|Clerk#000000242|0|requests. ironic, bold theodolites wak|
+2597|104|F|21964.66|1993-02-04|2-HIGH|Clerk#000000757|0|iously ruthless exc|
+2598|112|O|84871.50|1996-03-05|3-MEDIUM|Clerk#000000391|0| ironic notornis according to the blithely final requests should |
+2599|149|O|62807.13|1996-11-07|2-HIGH|Clerk#000000722|0|ts. slyly regular theodolites wake sil|
+2624|52|O|27148.63|1996-11-28|5-LOW|Clerk#000000930|0|ic, regular packages|
+2625|40|F|39382.74|1992-10-14|4-NOT SPECIFIED|Clerk#000000386|0| final deposits. blithely ironic ideas |
+2626|139|O|84314.51|1995-09-08|4-NOT SPECIFIED|Clerk#000000289|0|gside of the carefully special packages are furiously after the slyly express |
+2627|149|F|26798.65|1992-03-24|3-MEDIUM|Clerk#000000181|0|s. silent, ruthless requests|
+2628|56|F|165655.99|1993-10-22|5-LOW|Clerk#000000836|0|ajole across the blithely careful accounts. blithely silent deposits sl|
+2629|139|O|96458.03|1998-04-06|5-LOW|Clerk#000000680|0|uches dazzle carefully even, express excuses. ac|
+2630|85|F|127132.51|1992-10-24|5-LOW|Clerk#000000712|0|inal theodolites. ironic instructions s|
+2631|37|F|63103.32|1993-09-24|5-LOW|Clerk#000000833|0| quickly unusual deposits doubt around |
+2656|77|F|105492.37|1993-05-04|1-URGENT|Clerk#000000307|0|elets. slyly final accou|
+2657|25|O|148176.06|1995-10-17|2-HIGH|Clerk#000000160|0| foxes-- slyly final dependencies around the slyly final theodo|
+2658|14|O|163834.46|1995-09-23|3-MEDIUM|Clerk#000000400|0|bout the slyly regular accounts. ironic, |
+2659|83|F|79785.52|1993-12-18|4-NOT SPECIFIED|Clerk#000000758|0|cross the pending requests maintain |
+2660|127|O|16922.51|1995-08-05|5-LOW|Clerk#000000480|0|ly finally regular deposits. ironic theodolites cajole|
+2661|74|O|106036.84|1997-01-04|3-MEDIUM|Clerk#000000217|0|al, regular pinto beans. silently final deposits should have t|
+2662|37|O|87689.88|1996-08-21|3-MEDIUM|Clerk#000000589|0|bold pinto beans above the slyly final accounts affix furiously deposits. pac|
+2663|95|O|35131.80|1995-09-06|1-URGENT|Clerk#000000950|0|ar requests. furiously final dolphins along the fluffily spe|
+2688|98|F|181077.36|1992-01-24|2-HIGH|Clerk#000000720|0|have to nag according to the pending theodolites. sly|
+2689|103|F|41552.78|1992-04-09|4-NOT SPECIFIED|Clerk#000000698|0|press pains wake. furiously express theodolites alongsid|
+2690|94|O|224674.27|1996-03-31|3-MEDIUM|Clerk#000000760|0|ravely even theodolites |
+2691|7|F|30137.17|1992-04-30|5-LOW|Clerk#000000439|0|es at the regular deposits sleep slyly by the fluffy requests. eve|
+2692|62|O|24265.24|1997-12-02|3-MEDIUM|Clerk#000000878|0|es. regular asymptotes cajole above t|
+2693|19|O|66158.13|1996-09-04|1-URGENT|Clerk#000000370|0|ndle never. blithely regular packages nag carefully enticing platelets. ca|
+2694|121|O|102807.59|1996-03-14|5-LOW|Clerk#000000722|0| requests. bold deposits above the theodol|
+2695|58|O|138584.20|1996-08-20|1-URGENT|Clerk#000000697|0|ven deposits around the quickly regular packa|
+2720|31|F|161307.05|1993-06-08|1-URGENT|Clerk#000000948|0|quickly. special asymptotes are fluffily ironi|
+2721|79|O|59180.25|1996-01-27|2-HIGH|Clerk#000000401|0| ideas eat even, unusual ideas. theodolites are carefully|
+2722|35|F|50328.84|1994-04-09|5-LOW|Clerk#000000638|0|rding to the carefully quick deposits. bli|
+2723|61|O|104759.25|1995-10-06|5-LOW|Clerk#000000836|0|nts must have to cajo|
+2724|137|F|116069.66|1994-09-14|2-HIGH|Clerk#000000217|0| sleep blithely. blithely idle |
+2725|89|F|75144.68|1994-05-21|4-NOT SPECIFIED|Clerk#000000835|0|ular deposits. spec|
+2726|7|F|47753.00|1992-11-27|5-LOW|Clerk#000000470|0| blithely even dinos sleep care|
+2727|74|O|3089.42|1998-04-19|4-NOT SPECIFIED|Clerk#000000879|0|sual theodolites cajole enticingly above the furiously fin|
+2752|59|F|187932.30|1993-11-19|2-HIGH|Clerk#000000648|0| carefully regular foxes are quickly quickl|
+2753|16|F|159720.39|1993-11-30|2-HIGH|Clerk#000000380|0|ending instructions. unusual deposits|
+2754|145|F|25985.52|1994-04-03|2-HIGH|Clerk#000000960|0|cies detect slyly. |
+2755|118|F|101202.18|1992-02-07|4-NOT SPECIFIED|Clerk#000000177|0|ously according to the sly foxes. blithely regular pinto bean|
+2756|118|F|142323.38|1994-04-18|1-URGENT|Clerk#000000537|0|arefully special warho|
+2757|76|O|89792.48|1995-07-20|2-HIGH|Clerk#000000216|0| regular requests subl|
+2758|43|O|36671.88|1998-07-12|5-LOW|Clerk#000000863|0|s cajole according to the carefully special |
+2759|116|F|89731.10|1993-11-25|4-NOT SPECIFIED|Clerk#000000071|0|ts. regular, pending pinto beans sleep ab|
+2784|95|O|106635.21|1998-01-07|1-URGENT|Clerk#000000540|0|g deposits alongside of the silent requests s|
+2785|148|O|132854.79|1995-07-21|2-HIGH|Clerk#000000098|0|iously pending packages sleep according to the blithely unusual foxe|
+2786|79|F|178254.66|1992-03-22|2-HIGH|Clerk#000000976|0|al platelets cajole blithely ironic requests. ironic re|
+2787|103|O|3726.14|1995-09-30|1-URGENT|Clerk#000000906|0|he ironic, regular |
+2788|124|F|17172.66|1994-09-22|1-URGENT|Clerk#000000641|0|nts wake across the fluffily bold accoun|
+2789|37|O|219123.27|1998-03-14|2-HIGH|Clerk#000000972|0|gular patterns boost. carefully even re|
+2790|25|F|177458.97|1994-08-19|2-HIGH|Clerk#000000679|0| the carefully express deposits sleep slyly |
+2791|121|F|156697.55|1994-10-10|2-HIGH|Clerk#000000662|0|as. slyly ironic accounts play furiously bl|
+2816|58|F|42225.53|1994-09-20|2-HIGH|Clerk#000000289|0|kages at the final deposits cajole furious foxes. quickly |
+2817|40|F|71453.85|1994-04-19|3-MEDIUM|Clerk#000000982|0|ic foxes haggle upon the daringly even pinto beans. slyly|
+2818|49|F|120086.84|1994-12-12|3-MEDIUM|Clerk#000000413|0|eep furiously special ideas. express |
+2819|103|F|66927.16|1994-05-05|1-URGENT|Clerk#000000769|0|ngside of the blithely ironic dolphins. furio|
+2820|19|F|143813.39|1994-05-20|3-MEDIUM|Clerk#000000807|0|equests are furiously. carefu|
+2821|118|F|36592.48|1993-08-09|3-MEDIUM|Clerk#000000323|0|ng requests. even instructions are quickly express, silent instructi|
+2822|79|F|40142.15|1993-07-26|2-HIGH|Clerk#000000510|0|furiously against the accounts. unusual accounts aft|
+2823|79|O|171894.45|1995-09-09|2-HIGH|Clerk#000000567|0|encies. carefully fluffy accounts m|
+2848|70|F|116258.53|1992-03-10|1-URGENT|Clerk#000000256|0|ly fluffy foxes sleep furiously across the slyly regu|
+2849|46|O|180054.29|1996-04-30|2-HIGH|Clerk#000000659|0|al packages are after the quickly bold requests. carefully special |
+2850|100|O|122969.79|1996-10-02|2-HIGH|Clerk#000000392|0|, regular deposits. furiously pending packages hinder carefully carefully u|
+2851|145|O|7859.36|1997-09-07|5-LOW|Clerk#000000566|0|Tiresias wake quickly quickly even|
+2852|91|F|99050.81|1993-01-16|1-URGENT|Clerk#000000740|0|ruthless deposits against the final instructions use quickly al|
+2853|94|F|103641.15|1994-05-05|2-HIGH|Clerk#000000878|0|the carefully even packages.|
+2854|139|F|153568.02|1994-06-27|1-URGENT|Clerk#000000010|0| furiously ironic tithes use furiously |
+2855|49|F|48419.58|1993-04-04|4-NOT SPECIFIED|Clerk#000000973|0| silent, regular packages sleep |
+2880|8|F|145761.99|1992-03-15|2-HIGH|Clerk#000000756|0|ves maintain doggedly spec|
+2881|100|F|45695.84|1992-05-10|5-LOW|Clerk#000000864|0|uriously. slyly express requests according to the silent dol|
+2882|121|O|172872.37|1995-08-22|2-HIGH|Clerk#000000891|0|pending deposits. carefully eve|
+2883|121|F|170360.27|1995-01-23|5-LOW|Clerk#000000180|0|uses. carefully ironic accounts lose fluffil|
+2884|92|O|71683.84|1997-10-12|3-MEDIUM|Clerk#000000780|0|efully express instructions sleep against|
+2885|7|F|146896.72|1992-09-19|4-NOT SPECIFIED|Clerk#000000280|0|ly sometimes special excuses. final requests are |
+2886|109|F|94527.23|1994-11-13|4-NOT SPECIFIED|Clerk#000000619|0|uctions. ironic packages sle|
+2887|109|O|28571.39|1997-05-26|5-LOW|Clerk#000000566|0|slyly even pinto beans. slyly bold epitaphs cajole blithely above t|
+2912|94|F|27727.52|1992-03-12|5-LOW|Clerk#000000186|0|jole blithely above the quickly regular packages. carefully regular pinto bean|
+2913|43|O|130702.19|1997-07-12|3-MEDIUM|Clerk#000000118|0|mptotes doubt furiously slyly regu|
+2914|109|F|60867.14|1993-03-03|3-MEDIUM|Clerk#000000543|0|he slyly regular theodolites are furiously sile|
+2915|94|F|96015.13|1994-03-31|5-LOW|Clerk#000000410|0|ld packages. bold deposits boost blithely. ironic, unusual theodoli|
+2916|8|O|20182.22|1995-12-27|2-HIGH|Clerk#000000681|0|ithely blithe deposits sleep beyond the|
+2917|91|O|100714.13|1997-12-09|4-NOT SPECIFIED|Clerk#000000061|0| special dugouts among the special deposi|
+2918|118|O|21760.09|1996-09-08|3-MEDIUM|Clerk#000000439|0|ular deposits across th|
+2919|53|F|137223.14|1993-12-10|2-HIGH|Clerk#000000209|0|es. pearls wake quietly slyly ironic instructions--|
+2944|14|O|146581.14|1997-09-24|4-NOT SPECIFIED|Clerk#000000740|0|deas. permanently special foxes haggle carefully ab|
+2945|29|O|223507.72|1996-01-03|2-HIGH|Clerk#000000499|0|ons are carefully toward the permanent, bold pinto beans. regu|
+2946|125|O|102226.59|1996-02-05|5-LOW|Clerk#000000329|0|g instructions about the regular accounts sleep carefully along the pen|
+2947|70|P|43360.95|1995-04-26|1-URGENT|Clerk#000000464|0|ronic accounts. accounts run furiously d|
+2948|44|F|100758.71|1994-08-23|5-LOW|Clerk#000000701|0| deposits according to the blithely pending |
+2949|137|F|94231.71|1994-04-12|2-HIGH|Clerk#000000184|0|y ironic accounts use. quickly blithe accou|
+2950|136|O|183620.33|1997-07-06|1-URGENT|Clerk#000000833|0| dolphins around the furiously |
+2951|74|O|125509.17|1996-02-06|2-HIGH|Clerk#000000680|0|gular deposits above the finally regular ideas integrate idly stealthil|
+2976|29|F|145768.47|1993-12-10|4-NOT SPECIFIED|Clerk#000000159|0|. furiously ironic asymptotes haggle ruthlessly silently regular r|
+2977|73|O|25170.88|1996-08-27|3-MEDIUM|Clerk#000000252|0|quickly special platelets are furio|
+2978|44|P|139542.14|1995-05-03|1-URGENT|Clerk#000000135|0|d. even platelets are. ironic dependencies cajole slow, e|
+2979|133|O|116789.98|1996-03-23|3-MEDIUM|Clerk#000000820|0|even, ironic foxes sleep along|
+2980|4|O|187514.11|1996-09-14|3-MEDIUM|Clerk#000000661|0|y quick pinto beans wake. slyly re|
+2981|49|O|37776.79|1998-07-29|5-LOW|Clerk#000000299|0|hely among the express foxes. blithely stealthy requests cajole boldly. regu|
+2982|85|F|55582.94|1995-03-19|2-HIGH|Clerk#000000402|0|lyly. express theodolites affix slyly after the slyly speci|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/orders-part2.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/orders-part2.tbl
new file mode 100644
index 0000000..7e35045
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/orders-part2.tbl
@@ -0,0 +1,750 @@
+2983|62|F|58168.07|1992-01-07|1-URGENT|Clerk#000000278|0|r the even requests. accounts maintain. regular accounts|
+3008|40|O|156018.74|1995-11-08|3-MEDIUM|Clerk#000000701|0|ze quickly. blithely regular packages above the slyly bold foxes shall|
+3009|55|O|108424.94|1997-02-28|1-URGENT|Clerk#000000205|0|r ideas. carefully pe|
+3010|8|O|141647.08|1996-01-26|2-HIGH|Clerk#000000931|0| blithely final requests. special deposits are slyl|
+3011|91|F|46418.85|1992-01-14|5-LOW|Clerk#000000515|0|onic deposits kindle slyly. dependencies around the quickly iro|
+3012|32|F|91678.66|1993-05-05|1-URGENT|Clerk#000000414|0|ts after the regular pinto beans impress blithely s|
+3013|143|O|156407.40|1997-02-05|5-LOW|Clerk#000000591|0|the furiously pendin|
+3014|29|F|194159.59|1992-10-30|4-NOT SPECIFIED|Clerk#000000476|0|ep blithely according to the blith|
+3015|103|F|110826.83|1992-09-27|5-LOW|Clerk#000000013|0|ously regular deposits affix carefully. furiousl|
+3040|112|F|119201.64|1993-04-12|3-MEDIUM|Clerk#000000544|0|carefully special packages. blithe|
+3041|113|O|23039.46|1997-06-03|5-LOW|Clerk#000000092|0|s. unusual, pending deposits use carefully. thinly final|
+3042|20|F|104523.03|1994-11-21|3-MEDIUM|Clerk#000000573|0| the slyly ironic depo|
+3043|44|F|78221.69|1992-04-25|5-LOW|Clerk#000000137|0|cajole blithely furiously fina|
+3044|53|O|52433.54|1996-04-03|2-HIGH|Clerk#000000008|0|cajole final courts. ironic deposits about the quickly final re|
+3045|50|O|85822.67|1995-09-27|1-URGENT|Clerk#000000405|0| express courts sleep quickly special asymptotes. |
+3046|32|O|117817.52|1995-11-30|2-HIGH|Clerk#000000522|0|r deposits. platelets use furi|
+3047|25|O|37881.31|1997-03-21|1-URGENT|Clerk#000000962|0|as. slyly express deposits are dogged pearls. silent ide|
+3072|23|F|87475.82|1994-01-30|4-NOT SPECIFIED|Clerk#000000370|0|ely final deposits cajole carefully. ironic, re|
+3073|136|F|151419.50|1994-01-08|3-MEDIUM|Clerk#000000404|0|kly slyly bold accounts. express courts near the regular ideas sleep bli|
+3074|67|F|85861.93|1992-11-01|5-LOW|Clerk#000000546|0|yly even asymptotes shall have to haggle fluffily. deposits are|
+3075|127|F|37696.70|1994-05-07|3-MEDIUM|Clerk#000000433|0|ackages: carefully unusual reques|
+3076|92|F|93828.15|1993-07-23|2-HIGH|Clerk#000000099|0|busy foxes. deposits affix quickly ironic, pending pint|
+3077|121|O|99290.01|1997-08-06|2-HIGH|Clerk#000000228|0|kly. fluffily ironic requests use qui|
+3078|49|F|46310.83|1993-02-12|2-HIGH|Clerk#000000110|0|ounts are alongside of the blith|
+3079|100|O|148299.05|1997-09-12|5-LOW|Clerk#000000505|0|lly ironic accounts|
+3104|70|F|102693.61|1993-09-16|3-MEDIUM|Clerk#000000871|0|ges boost-- regular accounts are furiousl|
+3105|137|O|125396.80|1996-11-13|4-NOT SPECIFIED|Clerk#000000772|0|s. blithely final ins|
+3106|145|O|132494.97|1997-01-12|3-MEDIUM|Clerk#000000729|0|its use slyly final theodolites; regular dolphins hang above t|
+3107|26|O|107406.26|1997-08-21|1-URGENT|Clerk#000000669|0|ously even deposits acr|
+3108|85|F|63278.00|1993-08-05|1-URGENT|Clerk#000000574|0|s packages haggle furiously am|
+3109|124|F|216104.85|1993-07-24|5-LOW|Clerk#000000936|0|bold requests sleep quickly according to the slyly final|
+3110|88|F|115161.29|1994-12-17|2-HIGH|Clerk#000000564|0|round the fluffy instructions. carefully silent packages cajol|
+3111|133|O|154383.37|1995-08-25|5-LOW|Clerk#000000922|0|slyly regular theodolites. furious deposits cajole deposits. ironic theodoli|
+3136|23|F|145426.11|1994-08-10|4-NOT SPECIFIED|Clerk#000000891|0|tructions sleep slyly. pending di|
+3137|136|O|8958.65|1995-07-26|3-MEDIUM|Clerk#000000063|0|ymptotes wake carefully above t|
+3138|139|F|139579.18|1994-02-09|4-NOT SPECIFIED|Clerk#000000650|0|e fluffily final theodolites. even dependencies wake along the quickly ir|
+3139|17|F|40975.96|1992-01-02|3-MEDIUM|Clerk#000000855|0|ounts against the ruthlessly unusual dolphins|
+3140|145|F|54356.10|1992-04-09|1-URGENT|Clerk#000000670|0|carefully ironic deposits use furiously. blith|
+3141|26|O|115959.96|1995-11-10|1-URGENT|Clerk#000000475|0|es. furiously bold instructions after the carefully final p|
+3142|8|F|16030.15|1992-06-28|3-MEDIUM|Clerk#000000043|0|usual accounts about the carefully special requests sleep slyly quickly regul|
+3143|107|F|135647.68|1993-02-17|1-URGENT|Clerk#000000519|0| are final, ironic accounts. ironic |
+3168|136|F|69412.71|1992-01-30|5-LOW|Clerk#000000352|0|s sleep slyly? ironic, furious instructions detect. quickly final i|
+3169|19|F|126804.90|1993-12-21|3-MEDIUM|Clerk#000000252|0| even pinto beans are blithely special, special multip|
+3170|5|O|190142.17|1997-11-09|1-URGENT|Clerk#000000288|0|requests. furiously bold|
+3171|47|F|84405.78|1993-04-06|5-LOW|Clerk#000000940|0|ar deposits. idly r|
+3172|89|F|121360.83|1992-06-03|4-NOT SPECIFIED|Clerk#000000771|0|es. slyly ironic packages x-ra|
+3173|148|O|64892.73|1996-08-10|5-LOW|Clerk#000000516|0|ial requests lose along t|
+3174|127|O|92856.91|1995-11-15|5-LOW|Clerk#000000663|0|rts. silent, regular pinto beans are blithely regular packages. furiousl|
+3175|44|F|205282.63|1994-07-15|5-LOW|Clerk#000000629|0| across the slyly even realms use carefully ironic deposits: sl|
+3200|13|O|131103.31|1996-02-07|1-URGENT|Clerk#000000020|0| regular dependencies impress evenly even excuses. blithely |
+3201|97|F|90755.31|1993-07-02|4-NOT SPECIFIED|Clerk#000000738|0|. busy, express instruction|
+3202|88|F|50601.01|1992-12-24|5-LOW|Clerk#000000067|0|fluffily express requests affix carefully around th|
+3203|127|O|49357.72|1997-11-05|2-HIGH|Clerk#000000493|0|e furiously silent warhorses. slyly silent deposits wake bli|
+3204|10|F|41573.42|1992-12-26|1-URGENT|Clerk#000000693|0|ess somas cajole slyly. pending accounts cajole|
+3205|148|F|153637.79|1992-04-11|5-LOW|Clerk#000000803|0|e furiously. quickly regular dinos about the final pinto be|
+3206|122|O|64344.86|1996-08-09|1-URGENT|Clerk#000000755|0|ntegrate furiously final, express |
+3207|22|O|133038.59|1998-02-16|1-URGENT|Clerk#000000695|0|uriously accounts. fluffily i|
+3232|82|F|55619.01|1992-10-09|1-URGENT|Clerk#000000314|0|yly final accounts. packages agains|
+3233|140|F|54121.92|1994-10-24|5-LOW|Clerk#000000470|0|ly ironic epitaphs use stealthy, express deposits. quickly regular instruct|
+3234|14|O|147343.68|1996-04-05|4-NOT SPECIFIED|Clerk#000000367|0|ents according to the dependencies will sleep after the blithely even p|
+3235|46|O|104695.09|1995-11-15|5-LOW|Clerk#000000349|0| quickly pinto beans. ironi|
+3236|142|O|39470.39|1996-11-06|4-NOT SPECIFIED|Clerk#000000553|0|ithely slyly pending req|
+3237|19|F|10508.12|1992-06-03|1-URGENT|Clerk#000000606|0|inal requests. slyly even foxes detect about the furiously exp|
+3238|61|F|41375.69|1993-02-21|5-LOW|Clerk#000000818|0|lly express deposits are. furiously unusual ideas wake carefully somas. instr|
+3239|35|O|156802.80|1998-01-12|4-NOT SPECIFIED|Clerk#000000619|0| cajole carefully along the furiously pending deposits. |
+3264|94|O|162634.53|1996-11-02|5-LOW|Clerk#000000244|0|carefully. express, bold|
+3265|53|F|43315.15|1992-06-27|1-URGENT|Clerk#000000265|0|re quickly quickly pe|
+3266|4|P|68309.28|1995-03-17|5-LOW|Clerk#000000545|0|refully ironic instructions. slyly final pi|
+3267|112|O|33998.90|1997-01-07|5-LOW|Clerk#000000484|0| the packages. regular decoys about the bold dependencies grow fi|
+3268|142|F|36024.96|1994-06-25|5-LOW|Clerk#000000746|0|y brave requests unwind furiously accordin|
+3269|17|O|218697.85|1996-03-01|3-MEDIUM|Clerk#000000378|0|ts. accounts wake carefully. carefully dogged accounts wake slyly slyly i|
+3270|38|O|166669.86|1997-05-28|1-URGENT|Clerk#000000375|0|uffily pending courts ca|
+3271|34|F|86534.05|1992-01-01|1-URGENT|Clerk#000000421|0|s. furiously regular requests|
+3296|148|F|187553.35|1994-10-19|3-MEDIUM|Clerk#000000991|0|as! carefully final requests wake. furiously even|
+3297|139|F|9679.45|1992-11-03|2-HIGH|Clerk#000000220|0| after the theodolites cajole carefully according to the finally|
+3298|116|O|62716.67|1996-04-17|5-LOW|Clerk#000000241|0|even accounts boost |
+3299|91|F|42867.92|1993-12-26|3-MEDIUM|Clerk#000000853|0|bold deposits. special instructions sleep care|
+3300|118|O|27049.22|1995-07-15|5-LOW|Clerk#000000198|0|ses. carefully unusual instructions must have to detect about the blithel|
+3301|133|F|48497.09|1994-09-04|4-NOT SPECIFIED|Clerk#000000325|0|ular gifts impress enticingly carefully express deposits; instructions boo|
+3302|34|O|38330.42|1995-11-14|2-HIGH|Clerk#000000367|0|eep blithely ironic requests. quickly even courts haggle slyly|
+3303|145|O|97758.28|1997-12-14|4-NOT SPECIFIED|Clerk#000000661|0|nto beans sleep furiously above the carefully ironic |
+3328|7|F|139580.85|1992-11-19|5-LOW|Clerk#000000384|0|ake among the express accounts? carefully ironic packages cajole never.|
+3329|4|O|46107.70|1995-07-03|2-HIGH|Clerk#000000236|0|old deposits. special accounts haggle furiousl|
+3330|7|F|43255.19|1994-12-19|1-URGENT|Clerk#000000124|0|kages use. carefully regular deposits cajole carefully about |
+3331|91|F|65189.17|1993-05-21|2-HIGH|Clerk#000000901|0|uffily carefully sly accounts. blithely unu|
+3332|143|F|73739.06|1994-11-05|1-URGENT|Clerk#000000840|0|ans detect carefully furiously final deposits: regular accoun|
+3333|92|F|197973.22|1992-09-16|4-NOT SPECIFIED|Clerk#000000157|0|ctions boost slyly quickly even accounts. deposits along|
+3334|76|O|28930.68|1996-02-18|5-LOW|Clerk#000000532|0|ounts maintain carefully. furiously close request|
+3335|49|O|112603.34|1995-10-15|3-MEDIUM|Clerk#000000694|0| deposits poach. ironic ideas about the carefully ironi|
+3360|103|O|168750.48|1998-01-23|5-LOW|Clerk#000000254|0| the deposits. fluffily bold requests cajole regula|
+3361|49|F|75026.51|1992-08-23|4-NOT SPECIFIED|Clerk#000000577|0|unts detect furiously instructions. slow deposi|
+3362|140|O|183176.60|1995-07-29|5-LOW|Clerk#000000011|0|the quickly pending deposits. silent, ev|
+3363|52|O|91017.61|1995-09-23|2-HIGH|Clerk#000000615|0|posits. ironic, final deposits are furiously slyly pending |
+3364|46|O|108412.57|1997-06-21|1-URGENT|Clerk#000000280|0|y even foxes? blithely stea|
+3365|82|F|174634.12|1994-11-09|2-HIGH|Clerk#000000126|0|he slyly regular foxes nag about the accounts. fluffily |
+3366|52|O|13603.08|1997-05-18|1-URGENT|Clerk#000000160|0| pinto beans upon the quickly expres|
+3367|73|F|101339.68|1992-12-31|4-NOT SPECIFIED|Clerk#000000029|0|efully blithely ironic pinto beans. carefully close |
+3392|74|O|96057.42|1995-10-28|1-URGENT|Clerk#000000325|0|es thrash blithely depths. bold multipliers wake f|
+3393|98|O|183104.71|1995-07-04|2-HIGH|Clerk#000000076|0|even requests. excuses are carefully deposits. fluf|
+3394|149|O|162165.94|1996-05-05|4-NOT SPECIFIED|Clerk#000000105|0| blithely among the attainments. carefully final accounts nag blit|
+3395|149|F|141486.77|1994-10-30|4-NOT SPECIFIED|Clerk#000000682|0|ideas haggle beside the ev|
+3396|149|F|196443.16|1994-05-21|3-MEDIUM|Clerk#000000868|0|uffily regular platelet|
+3397|130|F|80084.61|1994-06-23|3-MEDIUM|Clerk#000000048|0|yly. final deposits wake f|
+3398|67|O|1147.42|1996-09-23|1-URGENT|Clerk#000000818|0|uthless, special courts atop the unusual accounts grow fur|
+3399|122|P|56938.16|1995-02-28|4-NOT SPECIFIED|Clerk#000000575|0|the carefully sly accounts. regular, pending theodolites wa|
+3424|103|O|42410.57|1996-08-21|1-URGENT|Clerk#000000190|0|ven requests are quickly pending accounts. blithely furious requests |
+3425|115|O|157040.57|1996-03-31|4-NOT SPECIFIED|Clerk#000000188|0|ions. deposits nag blithely alongside of the carefully f|
+3426|53|O|91929.93|1996-10-16|3-MEDIUM|Clerk#000000283|0|alongside of the slyly|
+3427|4|O|133451.14|1997-05-29|4-NOT SPECIFIED|Clerk#000000404|0|y final pinto beans snooze fluffily bold asymptot|
+3428|10|O|88047.04|1996-04-07|5-LOW|Clerk#000000953|0|lar excuses. slyly pending ideas detect p|
+3429|146|O|141902.54|1997-01-06|4-NOT SPECIFIED|Clerk#000000737|0|l deposits cajole furiously enticing deposits. blithe packages haggle careful|
+3430|113|F|161066.22|1994-12-12|4-NOT SPECIFIED|Clerk#000000664|0| regular attainments are at the final foxes. final packages along the blithe|
+3431|47|F|45536.27|1993-08-22|1-URGENT|Clerk#000000439|0| sleep. slyly busy Tiresias a|
+3456|46|F|32796.35|1993-06-01|5-LOW|Clerk#000000924|0|es promise slyly. ironicall|
+3457|25|P|174223.20|1995-04-27|4-NOT SPECIFIED|Clerk#000000849|0|ely thin asymptotes. deposits kindle. pending|
+3458|95|F|153069.14|1994-12-22|2-HIGH|Clerk#000000392|0|rges snooze. slyly unusua|
+3459|119|F|127134.05|1994-07-28|4-NOT SPECIFIED|Clerk#000000777|0|n instructions? carefully regular excuses are blithely. silent, ironi|
+3460|82|O|245976.74|1995-10-03|2-HIGH|Clerk#000000078|0|ans integrate carefu|
+3461|100|F|190960.69|1993-01-31|1-URGENT|Clerk#000000504|0|al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli|
+3462|133|O|63590.17|1997-05-17|3-MEDIUM|Clerk#000000657|0|uriously express asympto|
+3463|89|F|85255.56|1993-08-18|1-URGENT|Clerk#000000545|0|ding to the carefully ironic deposits|
+3488|148|F|92716.17|1995-01-08|3-MEDIUM|Clerk#000000694|0|cording to the carefully regular deposits. re|
+3489|109|F|62453.97|1993-07-29|3-MEDIUM|Clerk#000000307|0|s detect. carefully even platelets across the fur|
+3490|91|O|100106.96|1997-05-26|5-LOW|Clerk#000000703|0|gular ideas. furiously silent deposits across the unusual accounts boost i|
+3491|83|O|50287.06|1998-06-24|1-URGENT|Clerk#000000560|0|nic orbits believe carefully across the |
+3492|103|F|168721.45|1994-11-24|5-LOW|Clerk#000000066|0|packages along the regular foxes lose final dependencie|
+3493|82|F|41686.10|1993-08-24|2-HIGH|Clerk#000000887|0|lyly special accounts use blithely across the furiously sil|
+3494|49|F|136058.70|1993-04-04|5-LOW|Clerk#000000559|0|r instructions haggle. accounts cajole. carefully final requests at the |
+3495|31|O|58666.79|1996-02-26|2-HIGH|Clerk#000000441|0|nticing excuses are carefully|
+3520|125|O|151233.65|1997-08-04|1-URGENT|Clerk#000000023|0|hely. ideas nag; even, even fo|
+3521|7|F|142029.67|1992-10-26|5-LOW|Clerk#000000812|0|y even instructions cajole carefully above the bli|
+3522|26|F|151515.08|1994-09-26|5-LOW|Clerk#000000250|0|deposits-- slyly stealthy requests boost caref|
+3523|149|O|129657.08|1998-04-07|2-HIGH|Clerk#000000688|0|are on the carefully even depe|
+3524|94|F|22767.49|1992-05-03|2-HIGH|Clerk#000000607|0|efully unusual tithes among the foxes use blithely daringly bold deposits. re|
+3525|109|O|100749.60|1995-12-22|4-NOT SPECIFIED|Clerk#000000084|0|s nag among the blithely e|
+3526|56|F|53827.34|1995-03-16|5-LOW|Clerk#000000364|0|to the quickly special deposits print agai|
+3527|56|O|145232.09|1997-06-21|5-LOW|Clerk#000000874|0|regular ideas across the quickly bold theodo|
+3552|35|O|103656.44|1997-04-23|2-HIGH|Clerk#000000973|0| the ironic packages. furiously |
+3553|91|F|119838.14|1994-05-18|3-MEDIUM|Clerk#000000270|0|counts mold furiously. slyly i|
+3554|44|O|98335.61|1995-06-17|5-LOW|Clerk#000000931|0|hely ironic requests haggl|
+3555|46|O|134442.37|1996-07-07|5-LOW|Clerk#000000585|0|s nag carefully regular, even pinto be|
+3556|16|F|114681.55|1992-09-23|4-NOT SPECIFIED|Clerk#000000140|0|e. dependencies need to haggle alongs|
+3557|121|F|85477.89|1992-11-09|2-HIGH|Clerk#000000291|0|ithely courts. furi|
+3558|28|O|112912.00|1996-02-29|1-URGENT|Clerk#000000841|0|around the furiously even requests. quickl|
+3559|106|F|30722.49|1992-10-24|3-MEDIUM|Clerk#000000634|0|sly deposits. fluffily final ideas cajole careful|
+3584|13|O|80487.97|1997-08-11|1-URGENT|Clerk#000000760|0|fully bold packages. fluffily final braids haggle final, ironic dolphins. b|
+3585|139|F|159015.39|1994-11-23|2-HIGH|Clerk#000000988|0|regular asymptotes. bold pains above the carefully pending asymptot|
+3586|121|F|112845.04|1993-12-05|2-HIGH|Clerk#000000438|0|he quickly final courts. carefully regular requests nag unusua|
+3587|79|O|174798.97|1996-05-10|4-NOT SPECIFIED|Clerk#000000443|0|ular patterns detect |
+3588|119|F|207925.83|1995-03-19|4-NOT SPECIFIED|Clerk#000000316|0|ong the pains. evenly unusual |
+3589|31|F|39103.37|1994-05-26|2-HIGH|Clerk#000000023|0|ithe deposits nag furiously. furiously pending packages sleep f|
+3590|149|P|218482.70|1995-05-13|5-LOW|Clerk#000000986|0|lyly final deposits.|
+3591|136|F|98140.86|1993-12-08|3-MEDIUM|Clerk#000000144|0|ual foxes haggle! unusual request|
+3616|128|F|60933.29|1994-02-16|4-NOT SPECIFIED|Clerk#000000268|0|uickly about the quickly final requests. fluffily final packages wake evenly|
+3617|40|O|126205.42|1996-03-19|3-MEDIUM|Clerk#000000886|0|the carefully regular platelets ha|
+3618|10|O|136954.81|1997-12-13|3-MEDIUM|Clerk#000000894|0|. ideas run carefully. thin, pending |
+3619|149|O|222274.54|1996-11-20|2-HIGH|Clerk#000000211|0|uests mold after the blithely ironic excuses. slyly pending pa|
+3620|44|O|59291.75|1997-03-07|5-LOW|Clerk#000000124|0|le quickly against the epitaphs. requests sleep slyly according to the|
+3621|142|F|106150.05|1993-05-06|3-MEDIUM|Clerk#000000643|0|kly unusual deposits. qu|
+3622|91|O|109202.90|1995-11-27|5-LOW|Clerk#000000012|0|c deposits are fluffily about the blithely final theo|
+3623|4|O|175017.68|1996-12-26|1-URGENT|Clerk#000000184|0|- ironic excuses boost quickly in place |
+3648|125|F|180417.11|1993-06-17|5-LOW|Clerk#000000717|0|foxes. unusual deposits boost quickly. slyly regular asymptotes across t|
+3649|40|F|124470.32|1994-07-06|5-LOW|Clerk#000000349|0|taphs boost above the final p|
+3650|46|F|189547.57|1992-05-28|4-NOT SPECIFIED|Clerk#000000454|0|kages sleep fluffily slyly|
+3651|100|O|113191.45|1998-04-27|1-URGENT|Clerk#000000222|0|ly unusual deposits thrash quickly after the ideas.|
+3652|107|O|107732.23|1997-02-25|4-NOT SPECIFIED|Clerk#000000024|0|sly even requests after the |
+3653|40|F|142866.39|1994-03-27|1-URGENT|Clerk#000000402|0| pearls. bold accounts are along the ironic,|
+3654|7|F|222653.54|1992-06-03|5-LOW|Clerk#000000475|0|s cajole slyly carefully special theodolites. even deposits haggl|
+3655|49|F|74882.22|1992-10-06|1-URGENT|Clerk#000000815|0|er the carefully unusual deposits sleep quickly according to|
+3680|127|F|124402.59|1992-12-10|4-NOT SPECIFIED|Clerk#000000793|0|ular platelets. carefully regular packages cajole blithely al|
+3681|52|F|36889.65|1992-04-04|1-URGENT|Clerk#000000566|0|. ironic deposits against the ironic, regular frets use pending plat|
+3682|32|O|67525.43|1997-01-22|2-HIGH|Clerk#000000001|0|es haggle carefully. decoys nag |
+3683|88|F|99960.46|1993-03-04|2-HIGH|Clerk#000000248|0|ze across the express foxes. carefully special acco|
+3684|23|F|89509.91|1993-07-20|2-HIGH|Clerk#000000835|0|bold accounts affix along the carefully ironic requ|
+3685|16|F|154958.89|1992-01-17|3-MEDIUM|Clerk#000000954|0| sleep fluffily special ide|
+3686|40|O|82190.77|1998-07-07|2-HIGH|Clerk#000000175|0|s. furiously final pinto beans poach carefully among |
+3687|43|F|99851.38|1993-02-03|1-URGENT|Clerk#000000585|0|gular accounts. slyly regular instructions can are final ide|
+3712|64|F|127527.05|1992-01-02|2-HIGH|Clerk#000000032|0| promise according |
+3713|149|O|215342.63|1998-05-07|3-MEDIUM|Clerk#000000325|0|s haggle quickly. ironic, regular Tiresi|
+3714|40|O|84493.55|1998-05-01|3-MEDIUM|Clerk#000000595|0|nding accounts. ironic pinto beans wake slyly. furiously pendin|
+3715|65|O|64000.93|1996-03-18|1-URGENT|Clerk#000000463|0| always silent requests wake pinto beans. slyly pending foxes are aga|
+3716|43|O|146221.66|1997-08-19|4-NOT SPECIFIED|Clerk#000000748|0| pending ideas haggle. ironic,|
+3717|28|O|176525.53|1998-06-03|4-NOT SPECIFIED|Clerk#000000974|0|t the carefully even ideas use sp|
+3718|31|O|63195.54|1996-10-23|2-HIGH|Clerk#000000016|0|refully. furiously final packages use carefully slyly pending deposits! final,|
+3719|118|O|139902.71|1997-02-16|2-HIGH|Clerk#000000034|0|, enticing accounts are blithely among the daringly final asymptotes. furious|
+3744|65|F|33085.68|1992-01-10|3-MEDIUM|Clerk#000000765|0|osits sublate about the regular requests. fluffily unusual accou|
+3745|112|F|19405.73|1993-09-29|5-LOW|Clerk#000000181|0|ckages poach slyly against the foxes. slyly ironic instructi|
+3746|74|F|80018.54|1994-09-11|4-NOT SPECIFIED|Clerk#000000188|0|. express, special requests nag quic|
+3747|149|O|204355.65|1996-08-20|1-URGENT|Clerk#000000226|0|refully across the final theodolites. carefully bold accounts cajol|
+3748|53|O|83804.38|1998-02-28|1-URGENT|Clerk#000000156|0|slyly special packages|
+3749|38|P|87073.89|1995-02-24|3-MEDIUM|Clerk#000000639|0|y regular instructions haggle blithel|
+3750|97|P|177181.67|1995-04-30|3-MEDIUM|Clerk#000000885|0|y. express, even packages wake after the ide|
+3751|10|F|202917.72|1994-04-27|4-NOT SPECIFIED|Clerk#000000925|0|sheaves. express, unusual t|
+3776|85|F|150349.92|1992-11-20|2-HIGH|Clerk#000000698|0|efully even platelets slee|
+3777|28|F|82467.29|1994-04-08|3-MEDIUM|Clerk#000000941|0| regular, special dolphins cajole enticingly ca|
+3778|106|F|221036.31|1993-05-26|1-URGENT|Clerk#000000187|0| above the express requests. packages maintain fluffily according to|
+3779|74|O|31538.94|1997-01-05|4-NOT SPECIFIED|Clerk#000000670|0| against the deposits. quickly bold instructions x-ray. pending fox|
+3780|41|O|65385.42|1996-04-13|5-LOW|Clerk#000000967|0| around the brave, pendin|
+3781|139|O|133864.82|1996-06-20|1-URGENT|Clerk#000000394|0|yly after the ruthless packages. pinto beans use slyly: never ironic dependenc|
+3782|65|O|145096.17|1996-08-24|1-URGENT|Clerk#000000121|0|counts are. pending, regular asym|
+3783|44|F|155017.92|1993-12-06|4-NOT SPECIFIED|Clerk#000000614|0| along the pinto beans. special packages use. regular theo|
+3808|79|F|228054.01|1994-04-24|1-URGENT|Clerk#000000717|0|odolites. blithely ironic cour|
+3809|148|O|143070.70|1996-05-01|5-LOW|Clerk#000000646|0| regular excuses. even theodolites are fluffily according to t|
+3810|100|F|124675.27|1992-09-17|1-URGENT|Clerk#000000660|0|ters sleep across the carefully final |
+3811|80|O|154967.89|1998-04-16|3-MEDIUM|Clerk#000000290|0|sits wake slyly abo|
+3812|41|O|70502.52|1996-08-13|3-MEDIUM|Clerk#000000727|0|al, final requests cajole|
+3813|146|O|77247.05|1998-06-29|1-URGENT|Clerk#000000531|0|g the furiously regular instructions|
+3814|118|P|149451.88|1995-02-22|5-LOW|Clerk#000000669|0| the furiously pending theodo|
+3815|104|O|14275.01|1997-08-26|1-URGENT|Clerk#000000249|0|es snooze carefully stealth|
+3840|100|O|187156.38|1998-07-17|4-NOT SPECIFIED|Clerk#000000713|0|yly slow theodolites. enticingly |
+3841|58|F|129033.13|1994-10-05|4-NOT SPECIFIED|Clerk#000000018|0| bold requests sleep quickly ironic packages. sometimes regular deposits nag |
+3842|28|F|131447.03|1992-04-09|5-LOW|Clerk#000000418|0|silent ideas. final deposits use furiously. blithely express excuses cajole fu|
+3843|10|O|34035.17|1997-01-04|4-NOT SPECIFIED|Clerk#000000693|0|eodolites; slyly unusual accounts nag boldly |
+3844|79|F|6793.45|1994-12-29|1-URGENT|Clerk#000000686|0|r dolphins. slyly ironic theodolites ag|
+3845|89|F|134333.33|1992-04-26|1-URGENT|Clerk#000000404|0|es among the pending, regular accounts sleep blithely blithely even de|
+3846|49|O|123120.06|1998-02-05|2-HIGH|Clerk#000000877|0|y alongside of the slyl|
+3847|34|F|7014.31|1993-03-12|5-LOW|Clerk#000000338|0|uriously even deposits. furiously pe|
+3872|134|O|198538.68|1996-09-06|5-LOW|Clerk#000000943|0|counts boost slyly against the ironic platelets-- blithely p|
+3873|55|O|95291.79|1998-03-30|4-NOT SPECIFIED|Clerk#000000791|0|express deposits-- even ideas |
+3874|119|F|66455.34|1993-06-09|3-MEDIUM|Clerk#000000208|0|ular asymptotes sleep blithely ironic ideas. blithel|
+3875|118|O|74483.95|1997-09-10|1-URGENT|Clerk#000000587|0| solve among the fluffily even |
+3876|29|O|95126.32|1996-08-02|5-LOW|Clerk#000000708|0|into beans. blithely|
+3877|17|F|178492.01|1993-05-21|5-LOW|Clerk#000000652|0|foxes. thinly bold reques|
+3878|88|O|59989.66|1997-03-23|1-URGENT|Clerk#000000314|0|e carefully regular platelets. special, express dependencies slee|
+3879|142|O|80274.22|1995-11-23|1-URGENT|Clerk#000000231|0|sts along the quickly ironic sentiments cajole carefully according to t|
+3904|149|O|39338.44|1997-11-15|4-NOT SPECIFIED|Clerk#000000883|0|sits haggle furiously across the requests. theodolites ha|
+3905|22|F|56227.04|1993-12-21|4-NOT SPECIFIED|Clerk#000000573|0|usly even accounts lose quietly above the slyly express p|
+3906|46|F|145630.76|1992-05-28|3-MEDIUM|Clerk#000000867|0|ironic theodolites haggle blithely above the final re|
+3907|67|F|240457.56|1992-08-19|3-MEDIUM|Clerk#000000084|0|gular pinto beans sleep f|
+3908|43|F|57127.71|1993-03-09|3-MEDIUM|Clerk#000000490|0|ounts cajole. regularly|
+3909|22|O|82746.74|1998-07-27|1-URGENT|Clerk#000000980|0|nic, special theodolites sleep furiously! furiously |
+3910|64|O|47272.67|1996-08-26|3-MEDIUM|Clerk#000000270|0|ickly. furiously final packag|
+3911|10|P|35019.95|1995-03-17|4-NOT SPECIFIED|Clerk#000000818|0|he fluffily final forges haggle slyly according to the blithely|
+3936|32|O|168618.39|1996-11-07|2-HIGH|Clerk#000000200|0|iously express packages engage slyly fina|
+3937|94|O|187516.29|1997-11-30|4-NOT SPECIFIED|Clerk#000000189|0|ckages boost carefully blithely q|
+3938|31|F|46918.22|1993-03-03|1-URGENT|Clerk#000000199|0|. unusual, final foxes haggle|
+3939|70|O|8720.45|1996-01-11|5-LOW|Clerk#000000647|0|ly ruthlessly silent requests. blithely regular requests haggle blithely wh|
+3940|149|O|129012.84|1996-02-14|5-LOW|Clerk#000000363|0|e above the ideas. quickly even dependencies along the blithely ir|
+3941|136|O|95453.80|1996-08-29|2-HIGH|Clerk#000000503|0|gular theodolites integrate quickly |
+3942|76|F|38596.81|1993-06-28|4-NOT SPECIFIED|Clerk#000000608|0|eas cajole bold requests. idly silent instructions |
+3943|40|O|60314.97|1996-10-09|5-LOW|Clerk#000000482|0|se alongside of the final pinto beans. regular packages boost across the ca|
+3968|25|O|121704.45|1997-02-17|4-NOT SPECIFIED|Clerk#000000431|0| the slyly special accounts; |
+3969|52|O|169797.40|1997-05-14|2-HIGH|Clerk#000000731|0|uriously final dependencies slee|
+3970|76|F|163709.85|1992-03-27|3-MEDIUM|Clerk#000000190|0|luffily furiously regular deposits. blithely special requests cajole blithely|
+3971|104|O|47925.47|1996-06-28|5-LOW|Clerk#000000287|0|alongside of the instructions ought to are |
+3972|124|F|1861.19|1994-04-21|3-MEDIUM|Clerk#000000049|0|y regular requests haggle quickly. pending, express acco|
+3973|103|F|91541.48|1992-03-24|4-NOT SPECIFIED|Clerk#000000114|0|somas according to the quickly even instructions wake fu|
+3974|94|O|56779.06|1996-03-05|4-NOT SPECIFIED|Clerk#000000938|0|deposits are furiously beneath the bl|
+3975|118|O|37804.43|1995-04-11|3-MEDIUM|Clerk#000000016|0|ts. regular, regular Tiresias play furiously. ironi|
+4000|70|F|84053.93|1992-01-04|5-LOW|Clerk#000000339|0|le carefully closely even pinto beans. regular, ironic foxes against the|
+4001|115|O|95929.46|1997-05-15|3-MEDIUM|Clerk#000000878|0|detect. asymptotes sleep furio|
+4002|104|O|76518.11|1997-04-08|5-LOW|Clerk#000000097|0| regular braids are. furiously even patterns agains|
+4003|112|F|17603.01|1993-01-27|1-URGENT|Clerk#000000177|0| blithe theodolites are slyly. slyly silent accounts toward|
+4004|70|F|220715.14|1993-05-07|3-MEDIUM|Clerk#000000273|0|accounts among the blithely regular sentiments |
+4005|140|O|129062.13|1996-11-20|2-HIGH|Clerk#000000341|0|ily according to the slyly iron|
+4006|35|F|70557.05|1995-01-04|3-MEDIUM|Clerk#000000765|0|ly ironic packages integrate. regular requests alongside of |
+4007|8|F|116193.97|1993-06-18|2-HIGH|Clerk#000000623|0|ecial packages. slyly regular accounts integrate |
+4032|10|O|62497.51|1998-02-26|3-MEDIUM|Clerk#000000686|0|iresias sleep slyly regular ideas. quickly unusual|
+4033|83|F|57740.74|1993-06-02|5-LOW|Clerk#000000181|0|ously bold instructions haggle furiously above the fluf|
+4034|94|F|186912.51|1993-11-14|4-NOT SPECIFIED|Clerk#000000548|0|ts x-ray. express requests affix fluffily regular theodolites. pending, fina|
+4035|118|F|22840.21|1992-02-19|5-LOW|Clerk#000000097|0|he ironic deposits sleep blith|
+4036|47|O|82563.10|1997-04-26|3-MEDIUM|Clerk#000000398|0|ly express deposits nag slyly. ironic, final asymptotes boost bra|
+4037|121|F|36389.43|1993-03-24|2-HIGH|Clerk#000000384|0|t carefully above the unusual the|
+4038|94|O|155045.39|1996-01-06|1-URGENT|Clerk#000000272|0|re slyly. silent requests wake quickly. regular packages play quickly |
+4039|29|O|143753.01|1997-11-16|1-URGENT|Clerk#000000358|0|ly ironic deposits. ironic reques|
+4064|130|O|148500.71|1996-10-10|4-NOT SPECIFIED|Clerk#000000598|0|ccounts. furiously unusual theodolites wake carefully about|
+4065|80|F|156345.64|1994-06-09|1-URGENT|Clerk#000000131|0|even foxes! slyly final deposits agai|
+4066|32|O|176911.21|1997-01-27|4-NOT SPECIFIED|Clerk#000000286|0|yly ironic dinos. quickly regular accounts haggle. requests wa|
+4067|16|F|136517.34|1992-10-07|2-HIGH|Clerk#000000027|0|tes boost furiously quick asymptotes. final deposits of the dolphins solv|
+4068|125|O|71852.67|1996-09-18|3-MEDIUM|Clerk#000000203|0|lly even accounts wake furiously across the unusual platelets. unusu|
+4069|73|F|198816.13|1992-05-13|3-MEDIUM|Clerk#000000359|0|deposits: slyly bold ideas detect furiously. f|
+4070|29|O|98275.37|1995-06-12|2-HIGH|Clerk#000000713|0|xpress ideas poach ab|
+4071|148|O|67789.42|1996-09-15|4-NOT SPECIFIED|Clerk#000000486|0|nal deposits. pending deposits d|
+4096|139|F|81089.61|1992-07-03|4-NOT SPECIFIED|Clerk#000000706|0|sits. quickly thin deposits x-ray blith|
+4097|10|O|134308.04|1996-05-24|1-URGENT|Clerk#000000475|0|ickly under the even accounts. even packages after the furiously express|
+4098|23|O|48478.54|1996-11-05|4-NOT SPECIFIED|Clerk#000000491|0|otes. quickly final requests after the stealthily ironic pinto bean|
+4099|17|F|207364.80|1992-08-21|1-URGENT|Clerk#000000379|0|r platelets. slyly regular requests cajole carefully against the|
+4100|4|O|3892.77|1996-03-12|3-MEDIUM|Clerk#000000429|0|posits. carefully unusual packages use pending deposits. regular she|
+4101|142|F|21640.10|1993-11-22|4-NOT SPECIFIED|Clerk#000000704|0|y around the express, careful epitaphs. accounts use fluffily. quickly p|
+4102|22|O|128786.57|1996-03-17|1-URGENT|Clerk#000000675|0|nding dependencies was slyly about the bl|
+4103|106|F|38164.23|1992-07-03|5-LOW|Clerk#000000679|0|fully ironic dependencies.|
+4128|139|O|5472.17|1995-10-07|4-NOT SPECIFIED|Clerk#000000635|0|ctions. dependencies from the slyly regular accounts nag slyly fu|
+4129|32|F|67226.28|1993-06-26|3-MEDIUM|Clerk#000000541|0|nwind. quickly final theodolites use packages. accounts|
+4130|104|O|47823.04|1996-03-10|5-LOW|Clerk#000000609|0|omise alongside of the carefully final foxes. blithel|
+4131|44|O|145971.60|1998-01-30|1-URGENT|Clerk#000000612|0| above the foxes hang |
+4132|19|P|65601.08|1995-05-29|4-NOT SPECIFIED|Clerk#000000158|0|ld asymptotes solve alongside of the express, final packages. fluffily fi|
+4133|101|F|31693.88|1992-08-07|4-NOT SPECIFIED|Clerk#000000268|0|al, express foxes. quickly pending deposits might cajole alongsi|
+4134|97|F|125191.12|1995-01-12|1-URGENT|Clerk#000000171|0|fully even deposits. regular de|
+4135|37|O|99577.55|1997-03-10|3-MEDIUM|Clerk#000000627|0|ly quietly even ideas. deposits haggle blithely|
+4160|55|O|82493.07|1996-08-20|5-LOW|Clerk#000000283|0|the carefully special accounts. furiously regular dugouts alongs|
+4161|118|F|198995.21|1993-08-21|5-LOW|Clerk#000000047|0|nts. fluffily regular foxes above the quickly daring reques|
+4162|22|F|72359.55|1992-02-10|5-LOW|Clerk#000000179|0|r packages are slyly accounts. furiously special foxes detect carefully re|
+4163|64|F|11493.80|1992-12-21|2-HIGH|Clerk#000000268|0| integrate furiously slyly regular depende|
+4164|94|O|8709.16|1998-07-03|2-HIGH|Clerk#000000720|0| regularly busy theodolites boost furiously quickly bold packages. express, s|
+4165|4|O|11405.40|1997-07-25|3-MEDIUM|Clerk#000000621|0|special foxes affix never blithely ironic pinto beans; blithely |
+4166|43|F|100671.06|1993-02-28|5-LOW|Clerk#000000757|0|quickly sly forges impress. careful foxes across the blithely even a|
+4167|28|O|62108.45|1998-06-17|1-URGENT|Clerk#000000917|0|kly furiously even deposits. unu|
+4192|146|O|197192.95|1998-04-19|1-URGENT|Clerk#000000369|0|equests above the slyly regular pinto beans unwi|
+4193|4|F|143191.54|1994-01-09|2-HIGH|Clerk#000000201|0|ng accounts haggle quickly. packages use fluffily ironic excu|
+4194|106|F|62972.29|1994-10-16|3-MEDIUM|Clerk#000000385|0| instructions are quickly even pinto beans. courts boost furiously regular, ev|
+4195|104|F|54478.95|1993-05-29|4-NOT SPECIFIED|Clerk#000000777|0| pinto beans cajole furiously theodolites-- slyly regular deposits doub|
+4196|106|O|201455.98|1998-05-15|3-MEDIUM|Clerk#000000532|0|affix carefully. quickly final requests |
+4197|92|O|217709.03|1996-08-13|4-NOT SPECIFIED|Clerk#000000264|0| pinto beans according|
+4198|143|O|105789.01|1997-06-16|3-MEDIUM|Clerk#000000583|0|g the special packages haggle pen|
+4199|5|F|30494.62|1992-02-13|1-URGENT|Clerk#000000309|0|e blithely. special deposits haggle slyly final foxes. carefully even|
+4224|70|O|150655.44|1997-07-14|1-URGENT|Clerk#000000034|0|jole quickly final dolphins. slyly pending foxes wake furiously bold pl|
+4225|128|O|72533.07|1997-06-03|3-MEDIUM|Clerk#000000992|0|r the platelets nag among the special deposits. ironic, ironic re|
+4226|92|F|29827.44|1993-03-09|5-LOW|Clerk#000000203|0|phins wake slyly regular packages. deposits haggle slowl|
+4227|133|F|92261.08|1995-02-24|1-URGENT|Clerk#000000063|0|ng the requests; ideas haggle fluffily. slyly unusual ideas c|
+4228|110|O|22072.16|1997-03-28|5-LOW|Clerk#000000309|0|pecial requests aft|
+4229|14|O|75145.87|1998-03-03|1-URGENT|Clerk#000000301|0|p furiously: final excuses hagg|
+4230|140|F|219709.60|1992-03-04|1-URGENT|Clerk#000000364|0|lly ironic deposits integrate carefully about the fu|
+4231|86|O|111403.66|1997-11-20|4-NOT SPECIFIED|Clerk#000000630|0|ly final accounts cajole furiously accounts. bravely ironic platelets am|
+4256|118|F|23067.48|1992-04-05|4-NOT SPECIFIED|Clerk#000000043|0|y alongside of the fluffily iro|
+4257|17|P|41723.86|1995-03-25|3-MEDIUM|Clerk#000000682|0|r ideas cajole along the blithely regular gifts.|
+4258|92|O|133829.35|1996-10-27|4-NOT SPECIFIED|Clerk#000000364|0|efully final platelets around the blit|
+4259|104|O|12918.70|1997-10-09|5-LOW|Clerk#000000781|0|es snooze slyly against the furiously unusual ideas. furious|
+4260|142|F|18566.14|1992-05-16|4-NOT SPECIFIED|Clerk#000000919|0|e among the fluffily bold accounts.|
+4261|118|F|83665.20|1992-10-03|1-URGENT|Clerk#000000662|0| about the even, pending packages. slyly bold deposits boost|
+4262|88|O|176278.57|1996-08-04|3-MEDIUM|Clerk#000000239|0| of the furious accounts. furiously regular accounts w|
+4263|4|O|158885.83|1998-03-16|1-URGENT|Clerk#000000265|0|sly ruthless deposits. final packages are instructions. fu|
+4288|34|F|75030.81|1992-12-04|4-NOT SPECIFIED|Clerk#000000823|0|usly carefully even theodolites: slyly express pac|
+4289|125|F|20752.62|1993-10-07|3-MEDIUM|Clerk#000000912|0|e carefully close instructions. slyly special reques|
+4290|41|F|26128.99|1995-01-15|3-MEDIUM|Clerk#000000688|0| slyly quickly bold requests. final deposits haggle pending ideas! som|
+4291|89|F|71822.86|1993-11-29|3-MEDIUM|Clerk#000000655|0| sleep fluffily between the bold packages. bold|
+4292|25|F|145906.24|1992-01-09|3-MEDIUM|Clerk#000000794|0| ruthlessly. slyly bo|
+4293|103|O|198322.91|1996-08-20|2-HIGH|Clerk#000000750|0|ly packages. regular packages nag according to t|
+4294|49|F|232194.74|1992-08-15|3-MEDIUM|Clerk#000000407|0|ng pinto beans breach. slyly express requests bo|
+4295|5|O|77754.62|1996-02-10|3-MEDIUM|Clerk#000000023|0|e boldly bold dependencies|
+4320|115|O|67049.37|1996-12-08|4-NOT SPECIFIED|Clerk#000000223|0|ages haggle after the slowly bold se|
+4321|16|F|118896.95|1994-07-18|3-MEDIUM|Clerk#000000041|0|ending deposits are carefully carefully regular packa|
+4322|142|O|149671.92|1998-03-13|3-MEDIUM|Clerk#000000433|0|totes nag across the fluffily special instructions. quickly silent hockey |
+4323|104|F|27598.17|1994-01-23|2-HIGH|Clerk#000000282|0|lve after the slyly regular multipliers. even, regular excus|
+4324|73|O|178249.05|1995-07-17|1-URGENT|Clerk#000000800|0|ccounts. slyly stealthy requests shall have t|
+4325|130|O|20214.49|1996-07-18|2-HIGH|Clerk#000000591|0|y around the always ev|
+4326|29|O|39048.94|1996-10-27|4-NOT SPECIFIED|Clerk#000000869|0|packages. carefully express deposit|
+4327|146|P|126235.35|1995-03-16|2-HIGH|Clerk#000000571|0|yly pending braids. final requests abo|
+4352|14|O|18653.09|1997-11-26|2-HIGH|Clerk#000000620|0|ly final platelets integrate carefully even requ|
+4353|73|O|21815.30|1997-12-12|2-HIGH|Clerk#000000790|0|uickly even ideas cajole|
+4354|145|F|179827.12|1994-09-30|4-NOT SPECIFIED|Clerk#000000046|0|pending notornis. requests serve |
+4355|4|O|186370.23|1996-11-16|1-URGENT|Clerk#000000362|0|ndencies use furiously across the regular |
+4356|97|F|39828.51|1994-04-11|5-LOW|Clerk#000000956|0| asymptotes sleep blithely. asymptotes sleep. blithely regul|
+4357|47|O|67045.94|1997-10-23|4-NOT SPECIFIED|Clerk#000000031|0|ages nag between the|
+4358|25|O|46298.53|1997-08-12|1-URGENT|Clerk#000000692|0|according to the fluffily special asymptotes |
+4359|16|F|107824.40|1993-03-03|1-URGENT|Clerk#000000393|0|sts. special, unusual deposits across the ironic theodo|
+4384|25|F|52562.16|1992-07-13|1-URGENT|Clerk#000000192|0|onic platelets. furiously regular asymptotes according to the special pac|
+4385|122|O|39190.62|1996-08-06|2-HIGH|Clerk#000000597|0|ully final requests. ironic, even dolphins above the regular |
+4386|61|O|134413.58|1998-02-06|5-LOW|Clerk#000000070|0| dolphins. silent, idle pinto beans |
+4387|110|O|116740.67|1995-10-23|1-URGENT|Clerk#000000025|0|ter the regular pinto beans. special, final gifts above the requests wi|
+4388|10|O|69668.22|1996-03-28|2-HIGH|Clerk#000000715|0|ts wake against the carefully final accounts. sly|
+4389|55|F|120324.82|1994-05-05|3-MEDIUM|Clerk#000000403|0|wly express excuses after the permanently even instructions are|
+4390|7|P|140608.69|1995-05-23|1-URGENT|Clerk#000000691|0|inal pinto beans. exp|
+4391|38|F|48284.06|1992-02-18|2-HIGH|Clerk#000000880|0|regular accounts. even depo|
+4416|149|F|76067.10|1992-06-30|5-LOW|Clerk#000000391|0| deposits. ideas cajole express theodolites: |
+4417|67|O|60868.39|1998-07-09|1-URGENT|Clerk#000000365|0|ideas are alongside of the blithely final reque|
+4418|61|F|47099.71|1993-03-25|3-MEDIUM|Clerk#000000731|0|pecial pinto beans. close foxes affix iron|
+4419|104|O|94030.43|1996-06-12|4-NOT SPECIFIED|Clerk#000000410|0|ages wake furiously slyly thin theodolit|
+4420|109|F|6088.41|1994-06-18|1-URGENT|Clerk#000000706|0|lly bold deposits along the bold, pending foxes detect blithely after the acco|
+4421|10|O|258779.02|1997-04-04|3-MEDIUM|Clerk#000000246|0|t the pending warhorses. express waters a|
+4422|70|P|107140.22|1995-05-22|3-MEDIUM|Clerk#000000938|0|ly bold accounts sleep special, regular foxes. doggedly regular in|
+4423|64|F|4913.06|1995-02-17|5-LOW|Clerk#000000888|0|excuses are ruthless|
+4448|70|O|127191.47|1998-05-21|2-HIGH|Clerk#000000428|0|. deposits haggle around the silent packages; slyly unusual packages|
+4449|10|O|48206.14|1998-02-08|5-LOW|Clerk#000000035|0|ourts are carefully even deposits. pending |
+4450|106|O|110194.31|1997-07-15|1-URGENT|Clerk#000000867|0|quests boost. furiously even realms are blithely bold requests. bl|
+4451|4|F|92851.80|1994-10-01|1-URGENT|Clerk#000000181|0|. carefully final foxes along the quickly express T|
+4452|13|F|64838.66|1994-06-21|5-LOW|Clerk#000000985|0|oxes are slyly. express, ironic pinto beans wake after the quickly pending re|
+4453|65|O|137030.40|1997-04-01|3-MEDIUM|Clerk#000000603|0|ages could have to nag slyly furiously even asymptotes! slowly regular |
+4454|142|F|159578.94|1994-02-02|5-LOW|Clerk#000000411|0|uriously regular pint|
+4455|19|F|102534.63|1993-10-11|3-MEDIUM|Clerk#000000924|0|even requests. bravely regular foxes according to the carefully unusual |
+4480|85|F|28658.26|1994-03-31|4-NOT SPECIFIED|Clerk#000000534|0|press, bold deposits boost blit|
+4481|148|O|77705.40|1996-03-30|5-LOW|Clerk#000000443|0|press sheaves cajole furio|
+4482|82|P|63535.56|1995-05-15|4-NOT SPECIFIED|Clerk#000000534|0|ravely bold accounts. furiously ironic instructions affix quickly. pend|
+4483|52|F|126597.21|1992-03-07|3-MEDIUM|Clerk#000000615|0|its. blithely idle accounts run; theodolites wake carefully around the fi|
+4484|131|O|237947.61|1996-12-24|1-URGENT|Clerk#000000392|0|ct across the pinto beans. quickly pending excuses engage furiously.|
+4485|53|F|182432.17|1994-11-13|3-MEDIUM|Clerk#000000038|0|es wake slyly even packages. blithely brave requests nag above the regul|
+4486|37|O|135613.18|1998-03-03|2-HIGH|Clerk#000000656|0|ffily according to the carefully pending acc|
+4487|46|F|109469.90|1993-02-23|3-MEDIUM|Clerk#000000017|0|s up the never pending excuses wake furiously special pinto beans. furiously i|
+4512|70|O|148682.82|1995-10-25|5-LOW|Clerk#000000393|0|ending instructions maintain fu|
+4513|85|O|119820.38|1996-03-15|5-LOW|Clerk#000000154|0|ests. final, final ideas|
+4514|97|F|143899.85|1994-04-30|3-MEDIUM|Clerk#000000074|0|deposits according to the carefull|
+4515|140|F|161745.44|1992-03-17|1-URGENT|Clerk#000000191|0|quests among the accounts sleep boldly about the regular f|
+4516|130|F|35949.14|1994-03-29|3-MEDIUM|Clerk#000000739|0|ing packages sleep slyly regular attainments|
+4517|113|O|47614.08|1998-03-07|4-NOT SPECIFIED|Clerk#000000231|0|uriously final deposits doze furiously furiously reg|
+4518|125|O|25861.74|1997-05-01|3-MEDIUM|Clerk#000000187|0|luffily against the spec|
+4519|136|F|68885.66|1993-03-30|4-NOT SPECIFIED|Clerk#000000938|0|ccording to the final |
+4544|112|O|151148.81|1997-08-07|3-MEDIUM|Clerk#000000435|0|g dependencies dazzle slyly ironic somas. carefu|
+4545|59|F|143276.28|1993-01-17|4-NOT SPECIFIED|Clerk#000000303|0|ep. requests use sly|
+4546|43|O|39906.87|1995-07-29|5-LOW|Clerk#000000373|0|ns sleep. regular, regular instructions maintai|
+4547|109|F|52114.01|1993-08-23|3-MEDIUM|Clerk#000000519|0|uctions thrash platelets. slyly final foxes wake slyly against th|
+4548|127|O|139915.23|1996-06-28|5-LOW|Clerk#000000798|0| in place of the blithely express sentiments haggle slyly r|
+4549|64|O|43889.17|1998-03-05|4-NOT SPECIFIED|Clerk#000000965|0|ully even deposits dazzle. fluffily pending ideas against the requests|
+4550|118|F|27461.48|1994-12-29|2-HIGH|Clerk#000000748|0|s haggle carefully acco|
+4551|109|O|82824.14|1996-02-09|2-HIGH|Clerk#000000462|0|ts. slyly quick theodolite|
+4576|139|O|56936.10|1996-08-14|5-LOW|Clerk#000000798|0|e pending deposits. |
+4577|79|O|104259.88|1998-05-02|5-LOW|Clerk#000000409|0|ly. unusual platelets are alw|
+4578|91|F|95761.93|1992-09-13|5-LOW|Clerk#000000121|0| to the furiously ironic instructions? furiou|
+4579|106|O|85927.85|1995-12-01|2-HIGH|Clerk#000000951|0|its wake quickly blithely specia|
+4580|82|F|118464.65|1993-11-15|4-NOT SPECIFIED|Clerk#000000086|0|rs wake blithely regular requests. fluffily ev|
+4581|79|F|89592.11|1992-09-04|4-NOT SPECIFIED|Clerk#000000687|0|ges. carefully pending accounts use furiously abo|
+4582|19|O|18247.86|1996-07-04|1-URGENT|Clerk#000000638|0|g the furiously regular pac|
+4583|22|F|206495.43|1994-09-25|3-MEDIUM|Clerk#000000240|0|equests. slyly even platelets was qui|
+4608|80|F|157767.86|1994-06-17|1-URGENT|Clerk#000000259|0|y even instructions detect slyly asymptotes. blithely final packa|
+4609|133|O|70462.84|1996-12-05|3-MEDIUM|Clerk#000000239|0|hang slyly slyly expre|
+4610|26|F|135934.60|1993-06-18|5-LOW|Clerk#000000616|0|e carefully express pinto|
+4611|29|F|166506.22|1993-01-10|2-HIGH|Clerk#000000152|0|. furiously regular instructions haggle dolphins. even instructions det|
+4612|61|F|82598.87|1993-09-20|3-MEDIUM|Clerk#000000397|0|bove the deposits. even deposits dazzle. slyly express packages haggle sl|
+4613|133|O|212339.55|1998-03-05|3-MEDIUM|Clerk#000000541|0|furiously blithely pending dependen|
+4614|61|O|151801.06|1996-04-22|1-URGENT|Clerk#000000974|0| sauternes wake thinly special accounts. fur|
+4615|29|F|10500.27|1993-08-27|3-MEDIUM|Clerk#000000982|0|jole after the fluffily pending foxes. packages affix carefully acco|
+4640|97|O|81138.17|1996-01-01|5-LOW|Clerk#000000902|0|requests. deposits do detect above the blithely iron|
+4641|134|F|98485.21|1993-01-20|4-NOT SPECIFIED|Clerk#000000755|0|ronic, final requests integrate slyly: specia|
+4642|148|F|117537.87|1995-02-27|1-URGENT|Clerk#000000295|0|cial requests wake carefully around the regular, unusual ideas. furi|
+4643|67|O|52414.19|1995-06-30|2-HIGH|Clerk#000000292|0|ously regular packages. unusual, special platel|
+4644|94|O|85901.70|1998-01-17|5-LOW|Clerk#000000961|0|requests. fluffily even ideas bo|
+4645|44|F|231012.22|1994-09-20|1-URGENT|Clerk#000000764|0|fully even instructions. final gifts sublate quickly final requests. bl|
+4646|83|O|124637.19|1996-06-18|1-URGENT|Clerk#000000036|0|n place of the blithely qu|
+4647|28|F|110958.36|1994-05-14|3-MEDIUM|Clerk#000000626|0|out the deposits. slyly final pinto beans haggle idly. slyly s|
+4672|79|O|199593.71|1995-11-07|1-URGENT|Clerk#000000475|0|lyly final dependencies caj|
+4673|82|O|58094.75|1996-08-13|4-NOT SPECIFIED|Clerk#000000914|0|c deposits are slyly. bravely ironic deposits cajole carefully after the |
+4674|37|F|115411.37|1994-04-19|1-URGENT|Clerk#000000122|0|careful hockey players. carefully pending deposits caj|
+4675|86|F|68817.08|1993-11-25|4-NOT SPECIFIED|Clerk#000000741|0|al deposits haggle slyly final|
+4676|14|O|182025.95|1995-09-01|2-HIGH|Clerk#000000407|0|s. slyly bold accounts sleep furiously special|
+4677|40|O|25661.87|1998-02-21|3-MEDIUM|Clerk#000000245|0|ly pending deposits after the carefully regular foxes sleep blithely after t|
+4678|88|O|131752.07|1998-08-02|4-NOT SPECIFIED|Clerk#000000175|0|side of the bold platelets detect slyly blithely ironic e|
+4679|88|F|7211.59|1993-01-20|2-HIGH|Clerk#000000905|0|ely regular accounts affix slyly. final dolphins are. furiously final de|
+4704|2|O|63873.14|1996-08-16|4-NOT SPECIFIED|Clerk#000000256|0|lithely final requests about the fluffily regular |
+4705|98|F|173340.09|1992-03-22|4-NOT SPECIFIED|Clerk#000000522|0| special instructions poa|
+4706|25|F|101709.52|1992-12-29|4-NOT SPECIFIED|Clerk#000000722|0| packages above the never regular packages nag packages. deposits c|
+4707|91|F|61052.10|1995-02-27|2-HIGH|Clerk#000000943|0|ully enticing accounts behind the regular|
+4708|85|F|56998.36|1994-10-01|1-URGENT|Clerk#000000383|0|ly thinly even accounts. unusu|
+4709|26|O|49903.57|1996-01-08|3-MEDIUM|Clerk#000000785|0|he furiously even deposits! ironic theodolites haggle blithely. r|
+4710|100|F|88966.68|1994-12-08|4-NOT SPECIFIED|Clerk#000000734|0|the final, regular foxes. carefully ironic pattern|
+4711|142|O|129546.56|1998-05-06|1-URGENT|Clerk#000000818|0|mptotes. unusual packages wake furiously qui|
+4736|139|O|67572.73|1995-11-20|2-HIGH|Clerk#000000563|0|blithely regular courts affix into the carefully ironic deposits. slyly exp|
+4737|79|F|62014.51|1993-03-11|4-NOT SPECIFIED|Clerk#000000275|0|ents use slyly among the unusual, ironic pearls. furiously pending |
+4738|5|F|149466.62|1992-04-08|2-HIGH|Clerk#000000150|0|deposits. thin acco|
+4739|148|F|68255.82|1993-02-21|5-LOW|Clerk#000000872|0|ing to the pending attainments: pending, express account|
+4740|68|O|42579.40|1996-07-05|2-HIGH|Clerk#000000420|0| dependencies haggle about the|
+4741|127|F|180692.90|1992-07-07|4-NOT SPECIFIED|Clerk#000000983|0|ly bold deposits are slyly about the r|
+4742|64|P|155356.80|1995-03-23|3-MEDIUM|Clerk#000000058|0|n packages. quickly regular ideas cajole blithely|
+4743|97|F|65702.39|1993-03-31|5-LOW|Clerk#000000048|0|pinto beans above the bold, even idea|
+4768|136|F|4820.55|1993-11-22|2-HIGH|Clerk#000000875|0|ctions snooze idly beneath the quick waters. fluffily u|
+4769|121|P|136765.03|1995-04-14|4-NOT SPECIFIED|Clerk#000000116|0|pon the asymptotes. idle, final account|
+4770|59|O|72150.68|1995-06-20|2-HIGH|Clerk#000000461|0|cial instructions believe carefully. |
+4771|95|F|49625.21|1992-12-14|1-URGENT|Clerk#000000571|0|lly express deposits serve furiously along the f|
+4772|28|F|64102.93|1994-09-14|1-URGENT|Clerk#000000708|0|es sleep. regular requests haggle furiously slyly |
+4773|122|O|196080.26|1995-12-23|1-URGENT|Clerk#000000327|0|ptotes was slyly along the|
+4774|52|F|124380.73|1993-04-20|3-MEDIUM|Clerk#000000299|0|eposits use blithely bold deposits. carefully regular gifts about the fin|
+4775|128|O|112444.42|1995-08-13|4-NOT SPECIFIED|Clerk#000000609|0|s integrate slyly slyly final instructions. carefully bold pack|
+4800|37|F|91795.13|1992-01-06|5-LOW|Clerk#000000625|0|ggle furiously along the pending pinto beans. deposits use: final foxe|
+4801|88|O|108353.08|1996-01-25|1-URGENT|Clerk#000000553|0|r the final sentiments. pending theodolites sleep doggedly across t|
+4802|130|O|5978.65|1997-01-23|3-MEDIUM|Clerk#000000400|0| ironic, thin packages wake furiously ironic, ironic deposits. the|
+4803|124|O|158776.68|1996-02-08|5-LOW|Clerk#000000892|0|lly unusual courts are ironic|
+4804|37|F|111547.31|1992-01-28|2-HIGH|Clerk#000000614|0|ly final accounts. blithely unusual theodolite|
+4805|16|F|172102.96|1992-04-25|4-NOT SPECIFIED|Clerk#000000514|0|even accounts wake furiously slyly final accounts; blithel|
+4806|7|F|35390.15|1993-04-21|5-LOW|Clerk#000000625|0|ave accounts. furiously pending wa|
+4807|53|O|138902.23|1997-01-09|3-MEDIUM|Clerk#000000310|0|kly. slyly special accounts|
+4832|34|O|84954.79|1997-12-04|3-MEDIUM|Clerk#000000548|0|final accounts sleep among the blithe|
+4833|133|O|84800.44|1996-05-12|3-MEDIUM|Clerk#000000256|0|r deposits against the slyly final excuses slee|
+4834|19|O|124539.00|1996-09-12|2-HIGH|Clerk#000000284|0|lar accounts. furiously ironic accounts haggle slyly |
+4835|146|F|70857.51|1994-10-25|1-URGENT|Clerk#000000250|0|s integrate furiously blithely expr|
+4836|65|O|78711.40|1996-12-18|1-URGENT|Clerk#000000691|0|c packages cajole carefully through the accounts. careful|
+4837|130|O|68519.84|1998-04-24|4-NOT SPECIFIED|Clerk#000000517|0|n accounts are regular, bold accounts. even instructions use request|
+4838|44|F|61811.33|1992-08-02|1-URGENT|Clerk#000000569|0|ffily bold sentiments. carefully close dolphins cajole across the |
+4839|25|F|71241.63|1994-05-10|1-URGENT|Clerk#000000925|0| even somas. slyly express ideas lose carefully. blithely unusu|
+4864|88|F|149614.34|1992-11-11|5-LOW|Clerk#000000423|0|ests nag within the quickly ironic asymptotes. ironic|
+4865|85|O|162113.46|1997-06-07|3-MEDIUM|Clerk#000000418|0|sits boost stealthily above the bl|
+4866|53|O|25767.07|1997-08-07|2-HIGH|Clerk#000000663|0|kages. unusual packages nag fluffily. qui|
+4867|10|F|9741.03|1992-05-21|1-URGENT|Clerk#000000891|0|ss the slyly regular dependencies. fluffily regular deposits within the car|
+4868|76|O|159005.35|1997-03-02|5-LOW|Clerk#000000729|0|regular asymptotes. regular packages sublate carefully al|
+4869|58|F|175422.13|1994-09-26|5-LOW|Clerk#000000802|0|boost! ironic packages un|
+4870|103|F|94534.07|1994-08-06|3-MEDIUM|Clerk#000000911|0|nto beans about the blithely regular d|
+4871|46|O|129636.99|1995-06-12|1-URGENT|Clerk#000000531|0|ven, special instructions across t|
+4896|85|F|93206.35|1992-08-22|1-URGENT|Clerk#000000622|0|sly pending deposits. final accounts boost above the sly, even|
+4897|80|F|115688.85|1992-09-17|5-LOW|Clerk#000000184|0|s. bold pinto beans sleep. evenly final accounts daz|
+4898|14|F|40572.64|1994-07-11|4-NOT SPECIFIED|Clerk#000000841|0|final patterns. special theodolites haggle ruthlessly at the blithely spec|
+4899|61|F|12291.83|1993-10-18|4-NOT SPECIFIED|Clerk#000000348|0| instructions. furiously even packages are furiously speci|
+4900|137|F|221320.76|1992-06-30|4-NOT SPECIFIED|Clerk#000000878|0|sleep quickly unusual |
+4901|79|O|146298.28|1997-12-31|4-NOT SPECIFIED|Clerk#000000980|0|inal dependencies cajole furiously. carefully express accounts na|
+4902|139|O|26011.20|1998-07-04|3-MEDIUM|Clerk#000000874|0| the slyly express dolphins. |
+4903|92|F|34363.63|1992-03-22|4-NOT SPECIFIED|Clerk#000000907|0|yly. multipliers within the fo|
+4928|4|F|59931.42|1993-10-04|4-NOT SPECIFIED|Clerk#000000952|0|slyly brave instructions after the ironic excuses haggle ruthlessly about|
+4929|149|O|135187.33|1996-02-29|3-MEDIUM|Clerk#000000109|0|uests. furiously special ideas poach. pending |
+4930|149|F|176867.34|1994-05-06|5-LOW|Clerk#000000593|0| haggle slyly quietly final theodolites. packages are furious|
+4931|50|F|115759.13|1994-11-17|1-URGENT|Clerk#000000356|0|leep. slyly express dolphins nag slyly. furiously regular s|
+4932|122|F|42927.07|1993-08-10|1-URGENT|Clerk#000000830|0|onic foxes. enticingly reg|
+4933|94|O|42945.82|1995-07-14|3-MEDIUM|Clerk#000000848|0|y special sauternes integr|
+4934|40|O|180478.16|1997-02-17|1-URGENT|Clerk#000000372|0|nes cajole; carefully special accounts haggle. special pinto beans nag |
+4935|40|F|162088.30|1993-05-25|4-NOT SPECIFIED|Clerk#000000601|0|c foxes. fluffily pendin|
+4960|124|F|153259.41|1995-02-26|5-LOW|Clerk#000000229|0|uriously even excuses. fluffily regular instructions along the furiously ironi|
+4961|58|O|89224.24|1998-04-06|3-MEDIUM|Clerk#000000731|0| braids. furiously even theodolites |
+4962|104|F|44781.32|1993-07-28|3-MEDIUM|Clerk#000000008|0| breach never ironic |
+4963|34|O|54175.35|1996-11-07|3-MEDIUM|Clerk#000000754|0|ully unusual epitaphs nod s|
+4964|101|O|204163.10|1997-07-28|4-NOT SPECIFIED|Clerk#000000144|0|ithely final theodolites. blithely regu|
+4965|52|F|110626.82|1993-10-21|5-LOW|Clerk#000000638|0|dependencies poach packages. sometim|
+4966|70|O|59186.02|1996-09-07|2-HIGH|Clerk#000000243|0|accounts. blithely ironic courts wake boldly furiously express |
+4967|98|O|103814.27|1997-02-17|3-MEDIUM|Clerk#000000397|0|e theodolites; furiously b|
+4992|62|F|203904.80|1992-05-10|1-URGENT|Clerk#000000166|0|telets nag carefully am|
+4993|13|F|145730.19|1994-08-04|4-NOT SPECIFIED|Clerk#000000258|0|ing instructions nag furiously. un|
+4994|43|O|216071.76|1996-06-29|4-NOT SPECIFIED|Clerk#000000868|0|oxes wake above the asymptotes. bold requests sleep br|
+4995|40|O|189651.76|1996-01-06|4-NOT SPECIFIED|Clerk#000000748|0|s. even deposits boost along the express, even theodolites. stealthily ir|
+4996|133|F|100750.67|1992-09-14|3-MEDIUM|Clerk#000000433|0|foxes. carefully special packages haggle quickly fluffi|
+4997|47|O|122611.05|1998-03-18|5-LOW|Clerk#000000040|0|egrate final pinto beans. fluffily special notornis use blith|
+4998|32|F|129096.80|1992-01-11|4-NOT SPECIFIED|Clerk#000000054|0|alongside of the quickly final requests hang always|
+4999|85|F|98643.17|1993-06-26|2-HIGH|Clerk#000000504|0| dolphins cajole blithely above the sly |
+5024|124|O|116127.69|1996-10-25|3-MEDIUM|Clerk#000000659|0|r foxes. regular excuses are about the quickly regular theodolites. regular, |
+5025|121|O|20099.43|1997-02-03|5-LOW|Clerk#000000805|0|ackages are slyly about the quickly |
+5026|28|O|13197.78|1997-09-06|1-URGENT|Clerk#000000955|0|y final requests us|
+5027|148|O|181346.56|1997-08-30|2-HIGH|Clerk#000000751|0|e-- final, pending requests along t|
+5028|13|F|30755.69|1992-04-17|2-HIGH|Clerk#000000180|0|ickly blithely express deposits. b|
+5029|11|F|19811.69|1992-11-14|3-MEDIUM|Clerk#000000469|0|. regular accounts haggle slyly. regul|
+5030|106|O|71781.23|1998-05-25|4-NOT SPECIFIED|Clerk#000000564|0| wake slyly furiously thin requests. ironic pinto beans ha|
+5031|139|F|91438.59|1994-12-02|3-MEDIUM|Clerk#000000788|0|lar instructions haggle blithely pending foxes? sometimes final excuses h|
+5056|52|O|62258.18|1997-02-15|5-LOW|Clerk#000000828|0|lithely above the express ideas. blithely final deposits are fluffily spec|
+5057|64|O|76164.41|1997-08-03|1-URGENT|Clerk#000000955|0|r ironic requests of the carefully ironic dependencies wake slyly a|
+5058|119|O|17031.01|1998-03-23|1-URGENT|Clerk#000000367|0| the pending packages wake after the quickly speci|
+5059|43|F|67173.82|1993-11-10|2-HIGH|Clerk#000000058|0|latelets. final, regular accounts cajole furiously ironic pinto beans? do|
+5060|112|F|65218.47|1992-07-07|4-NOT SPECIFIED|Clerk#000000333|0|e according to the excuses. express theodo|
+5061|101|F|52190.52|1993-08-14|1-URGENT|Clerk#000000009|0|e packages use fluffily according to the carefully ironic deposits. bol|
+5062|61|F|109247.00|1992-10-08|3-MEDIUM|Clerk#000000012|0|ithely. blithely bold theodolites affix. blithely final deposits haggle ac|
+5063|23|O|98753.57|1997-05-17|2-HIGH|Clerk#000000745|0|lyly after the pending foxes. express theodolites breach across t|
+5088|130|F|101616.44|1993-01-06|5-LOW|Clerk#000000930|0|ole slyly since the quickly ironic br|
+5089|130|F|109246.54|1992-07-29|1-URGENT|Clerk#000000677|0|cial platelets. quiet, final ideas cajole carefully. unusu|
+5090|89|O|132838.49|1997-03-09|1-URGENT|Clerk#000000953|0|ress accounts affix silently carefully quick accounts. carefully f|
+5091|148|O|47852.06|1998-05-21|3-MEDIUM|Clerk#000000311|0|egular decoys mold carefully fluffily unus|
+5092|22|O|195834.96|1995-10-30|5-LOW|Clerk#000000194|0|are blithely along the pin|
+5093|79|F|190693.92|1993-09-03|3-MEDIUM|Clerk#000000802|0|ully ironic theodolites sleep above the furiously ruthless instructions. bli|
+5094|106|F|74892.08|1993-03-29|4-NOT SPECIFIED|Clerk#000000406|0|uickly pending deposits haggle quickly ide|
+5095|97|F|184583.99|1992-04-22|2-HIGH|Clerk#000000964|0|accounts are carefully! slyly even packages wake slyly a|
+5120|16|O|28007.73|1996-06-05|1-URGENT|Clerk#000000332|0|against the slyly express requests. furiousl|
+5121|133|F|150334.57|1992-05-11|4-NOT SPECIFIED|Clerk#000000736|0|gular requests. furiously final pearls against the permanent, thin courts s|
+5122|70|O|79863.84|1996-02-10|5-LOW|Clerk#000000780|0|blithely. slyly ironic deposits nag. excuses s|
+5123|10|O|11850.45|1998-02-10|1-URGENT|Clerk#000000776|0|ic requests. furiously ironic packages grow above the express, ironic inst|
+5124|25|O|159170.80|1997-04-04|4-NOT SPECIFIED|Clerk#000000749|0|kly even courts. bold packages solve. |
+5125|28|O|38065.28|1998-02-07|5-LOW|Clerk#000000834|0|ructions. dolphins wake slowly carefully unusual |
+5126|112|F|92123.32|1992-10-12|4-NOT SPECIFIED|Clerk#000000270|0|s. unusual deposits |
+5127|73|O|48024.99|1997-01-15|5-LOW|Clerk#000000829|0|fully express pinto beans. slyly final accounts along the ironic dugouts use s|
+5152|44|O|60568.34|1997-01-04|3-MEDIUM|Clerk#000000963|0| for the blithely reg|
+5153|113|O|193832.28|1995-08-26|1-URGENT|Clerk#000000954|0| the furiously ironic foxes. express packages shall cajole carefully across|
+5154|8|O|28070.86|1997-04-13|3-MEDIUM|Clerk#000000316|0|inal requests. slyly regular deposits nag. even deposits haggle agains|
+5155|77|F|70183.29|1994-06-12|2-HIGH|Clerk#000000108|0|y pending deposits are ag|
+5156|125|O|59439.44|1996-11-04|5-LOW|Clerk#000000117|0|ngside of the multipliers solve slyly requests. regu|
+5157|142|O|167056.34|1997-07-06|4-NOT SPECIFIED|Clerk#000000689|0|closely above the unusual deposits. furiously|
+5158|76|O|240284.95|1997-01-21|1-URGENT|Clerk#000000541|0| regular foxes. even foxes wake blithely |
+5159|106|O|147543.26|1996-09-25|1-URGENT|Clerk#000000303|0|tegrate slyly around the slyly sly sauternes. final pa|
+5184|85|O|209155.48|1998-07-20|5-LOW|Clerk#000000250|0|nding accounts detect final, even|
+5185|148|O|206179.68|1997-07-25|3-MEDIUM|Clerk#000000195|0| regular ideas about the even ex|
+5186|52|O|208892.63|1996-08-03|1-URGENT|Clerk#000000332|0|pecial platelets. slyly final ac|
+5187|55|O|46380.69|1997-07-16|3-MEDIUM|Clerk#000000682|0|ckly according to t|
+5188|140|P|66268.86|1995-03-02|4-NOT SPECIFIED|Clerk#000000029|0|counts. finally ironic requests ab|
+5189|71|F|184172.31|1993-11-26|5-LOW|Clerk#000000940|0|e after the pending accounts. asymptotes boost. re|
+5190|58|F|89684.31|1992-04-26|5-LOW|Clerk#000000888|0|equests. slyly unusual|
+5191|77|F|119910.04|1994-12-11|4-NOT SPECIFIED|Clerk#000000318|0|ing, regular deposits alongside of the deposits boost fluffily quickly ev|
+5216|59|O|16763.95|1997-08-14|3-MEDIUM|Clerk#000000418|0|des boost across the platelets. slyly busy theodolit|
+5217|35|O|135745.58|1995-10-13|2-HIGH|Clerk#000000873|0|ons might wake quickly according to th|
+5218|82|F|73882.37|1992-07-30|4-NOT SPECIFIED|Clerk#000000683|0|y ruthless packages according to the bold, ironic package|
+5219|88|O|21267.72|1997-02-27|1-URGENT|Clerk#000000510|0|aggle always. foxes above the ironic deposits |
+5220|10|F|24844.39|1992-07-30|2-HIGH|Clerk#000000051|0| final packages. ideas detect slyly around|
+5221|13|O|71968.10|1995-06-09|4-NOT SPECIFIED|Clerk#000000324|0|lar accounts above the sl|
+5222|80|F|1051.15|1994-05-27|4-NOT SPECIFIED|Clerk#000000613|0|along the bold ideas. furiously final foxes snoo|
+5223|149|F|105561.21|1994-06-30|1-URGENT|Clerk#000000745|0|e. theodolites serve blithely unusual, final foxes. carefully pending packag|
+5248|70|P|86958.28|1995-04-15|2-HIGH|Clerk#000000737|0|theodolites cajole according to the silent packages. quickly ironic packages a|
+5249|103|F|123586.03|1994-09-06|3-MEDIUM|Clerk#000000019|0|refully bold accounts |
+5250|97|O|29673.73|1995-07-16|2-HIGH|Clerk#000000307|0|. carefully final instructions sleep among the finally regular dependen|
+5251|34|O|34004.48|1995-04-12|3-MEDIUM|Clerk#000000687|0| ironic dugouts detect. reque|
+5252|91|O|173145.37|1996-02-17|1-URGENT|Clerk#000000724|0| ironic accounts among the silent asym|
+5253|148|P|108361.46|1995-04-11|2-HIGH|Clerk#000000275|0|egular requests! blithely regular deposits alongside of t|
+5254|112|F|196989.09|1992-07-26|4-NOT SPECIFIED|Clerk#000000527|0|he express, even ideas cajole blithely special requests|
+5255|64|O|75074.07|1996-07-12|5-LOW|Clerk#000000591|0|ly slow forges. express foxes haggle. regular, even asymp|
+5280|34|O|68052.70|1997-12-03|3-MEDIUM|Clerk#000000604|0|riously ironic instructions. ironic ideas according to the accounts boost fur|
+5281|124|O|179418.31|1995-11-02|2-HIGH|Clerk#000000158|0|ackages haggle slyly a|
+5282|50|O|94446.69|1998-01-30|1-URGENT|Clerk#000000030|0|rding to the unusual, bold accounts. regular instructions|
+5283|131|F|18594.66|1994-06-04|3-MEDIUM|Clerk#000000579|0|ests. even, final ideas alongside of t|
+5284|61|O|40548.99|1995-07-09|4-NOT SPECIFIED|Clerk#000000155|0| careful dependencies use sly|
+5285|70|F|99377.51|1994-01-18|2-HIGH|Clerk#000000976|0|p across the furiously ironic deposits.|
+5286|116|O|79646.89|1997-09-26|5-LOW|Clerk#000000606|0|structions are furiously quickly ironic asymptotes. quickly iro|
+5287|25|F|30045.95|1993-12-22|5-LOW|Clerk#000000406|0|regular packages. bold instructions sleep always. carefully final p|
+5312|65|F|66697.95|1995-02-24|2-HIGH|Clerk#000000690|0|ter the even, bold foxe|
+5313|13|O|159870.44|1997-06-17|4-NOT SPECIFIED|Clerk#000000896|0|le. final courts haggle furiously according to the |
+5314|34|O|26999.83|1995-06-02|2-HIGH|Clerk#000000617|0|ions across the quickly special d|
+5315|139|F|55554.97|1992-10-29|4-NOT SPECIFIED|Clerk#000000035|0| furiously. quickly unusual packages use. sly|
+5316|100|F|62316.61|1994-01-31|1-URGENT|Clerk#000000734|0| requests haggle across the regular, pending deposits. furiously regular requ|
+5317|37|F|228002.51|1994-09-09|5-LOW|Clerk#000000687|0|jole quickly at the slyly pend|
+5318|59|F|106935.19|1993-04-04|2-HIGH|Clerk#000000663|0|efully regular dolphins. even ideas nag fluffily furiously even packa|
+5319|98|O|68619.29|1996-01-21|1-URGENT|Clerk#000000237|0|lent requests. quickly pe|
+5344|109|O|88216.32|1998-06-21|3-MEDIUM|Clerk#000000569|0|s. ironic excuses cajole across the|
+5345|31|O|111924.56|1997-08-24|1-URGENT|Clerk#000000057|0|r the slyly silent packages. pending, even pinto b|
+5346|37|F|149536.20|1993-12-26|2-HIGH|Clerk#000000220|0|gly close packages against the even, regular escapades boost evenly accordi|
+5347|49|F|173024.71|1995-02-22|3-MEDIUM|Clerk#000000180|0|onic, regular deposits. packag|
+5348|53|O|119164.96|1997-11-08|5-LOW|Clerk#000000497|0|totes. accounts after the furiously|
+5349|67|O|38038.84|1996-09-01|1-URGENT|Clerk#000000960|0|le along the carefully bold dolphins. carefully special packa|
+5350|76|F|113417.03|1993-10-10|5-LOW|Clerk#000000604|0|ccounts after the carefully pending requests believe |
+5351|122|O|76799.25|1998-05-11|1-URGENT|Clerk#000000443|0|to beans sleep furiously after the carefully even|
+5376|149|F|98422.83|1994-07-04|5-LOW|Clerk#000000392|0|. quickly ironic deposits integrate along|
+5377|64|O|117728.37|1997-04-24|2-HIGH|Clerk#000000917|0|ons nag blithely furiously regula|
+5378|43|F|101899.93|1992-10-25|1-URGENT|Clerk#000000520|0|n ideas. regular accounts haggle. ironic ideas use along the bold ideas. blith|
+5379|89|O|47010.15|1995-08-08|2-HIGH|Clerk#000000503|0|he unusual accounts. carefully special instructi|
+5380|148|O|123014.83|1997-10-12|1-URGENT|Clerk#000000481|0|le slyly about the slyly final dolphins. fu|
+5381|32|F|223995.46|1993-01-29|5-LOW|Clerk#000000531|0|arefully bold packages are slyly furiously ironic foxes. fluffil|
+5382|35|F|138423.03|1992-01-13|5-LOW|Clerk#000000809|0|lent deposits are according to the reg|
+5383|31|O|11474.95|1995-05-26|5-LOW|Clerk#000000409|0|ly bold requests hang furiously furiously unusual accounts. evenly unusu|
+5408|23|F|123477.05|1992-07-21|5-LOW|Clerk#000000735|0|egular requests according to the|
+5409|13|F|145040.38|1992-01-09|5-LOW|Clerk#000000171|0|eans. regular accounts are regul|
+5410|22|O|139104.17|1998-07-28|4-NOT SPECIFIED|Clerk#000000117|0|final deposits: pending excuses boost. ironic theodolites cajole furi|
+5411|61|O|62541.27|1997-05-16|3-MEDIUM|Clerk#000000800|0|equests cajole slyly furious|
+5412|142|O|109979.71|1998-01-20|2-HIGH|Clerk#000000151|0|ets boost furiously regular accounts. regular foxes above th|
+5413|94|O|224382.57|1997-10-17|1-URGENT|Clerk#000000066|0|e even excuses. always final depen|
+5414|100|F|167017.39|1993-03-25|4-NOT SPECIFIED|Clerk#000000242|0|lent dependencies? carefully express requests sleep furiously ac|
+5415|23|F|176864.83|1992-08-05|3-MEDIUM|Clerk#000000998|0|ly even ideas nag blithely above the final instructions|
+5440|130|O|3223.17|1997-01-12|1-URGENT|Clerk#000000154|0|posits boost regularly ironic packages. regular, ironic deposits wak|
+5441|41|F|131891.05|1994-07-21|4-NOT SPECIFIED|Clerk#000000257|0|after the furiously ironic |
+5442|43|O|139332.94|1998-01-13|4-NOT SPECIFIED|Clerk#000000954|0|ully. quickly express accounts against the|
+5443|131|O|124950.79|1996-10-10|4-NOT SPECIFIED|Clerk#000000492|0|al foxes could detect. blithely stealthy asymptotes kind|
+5444|130|P|172908.01|1995-03-18|1-URGENT|Clerk#000000677|0| asymptotes. asymptotes cajole quickly quickly bo|
+5445|115|F|114990.63|1993-07-26|5-LOW|Clerk#000000623|0|s. even, special requests cajole furiously even, |
+5446|7|F|29920.80|1994-06-21|5-LOW|Clerk#000000304|0| furiously final pac|
+5447|13|O|29029.84|1996-03-16|3-MEDIUM|Clerk#000000597|0|uternes around the furiously bold accounts wake after |
+5472|70|F|221636.83|1993-04-11|5-LOW|Clerk#000000552|0|counts. deposits about the slyly dogged pinto beans cajole slyly|
+5473|65|F|63041.33|1992-03-25|4-NOT SPECIFIED|Clerk#000000306|0|te the quickly stealthy ideas. even, regular deposits above|
+5474|55|F|131079.52|1992-06-01|4-NOT SPECIFIED|Clerk#000000487|0|gle blithely enticing ideas. final, exp|
+5475|139|O|10645.48|1996-07-07|5-LOW|Clerk#000000856|0|es shall boost slyly. furiously even deposits lose. instruc|
+5476|91|O|26906.38|1997-11-06|1-URGENT|Clerk#000000189|0|furiously final ideas. furiously bold dependencies sleep care|
+5477|107|O|130125.64|1997-12-30|5-LOW|Clerk#000000689|0|ckages. ironic deposits caj|
+5478|116|O|97502.23|1996-05-17|1-URGENT|Clerk#000000272|0|ckages. quickly pending deposits thrash furiously: bl|
+5479|70|F|70553.45|1993-12-22|3-MEDIUM|Clerk#000000335|0|ng asymptotes. pinto beans sleep care|
+5504|19|F|41492.25|1993-01-06|2-HIGH|Clerk#000000221|0|y pending packages. furiousl|
+5505|95|O|147329.51|1997-10-04|5-LOW|Clerk#000000719|0| final, regular packages according to the slyly ironic accounts nag ironica|
+5506|91|F|8413.31|1993-11-08|1-URGENT|Clerk#000000292|0|nusual theodolites. sly|
+5507|2|O|140363.70|1998-05-28|5-LOW|Clerk#000000692|0|the carefully ironic instructions are quickly iro|
+5508|56|O|3808.05|1996-06-21|1-URGENT|Clerk#000000128|0|y express packages cajole furiously. slyly unusual requests |
+5509|80|F|135335.96|1994-04-08|5-LOW|Clerk#000000164|0|usual deposits use packages. furiously final requests wake slyly about th|
+5510|37|F|126948.81|1993-01-08|3-MEDIUM|Clerk#000000819|0| nag slyly. carefully eve|
+5511|79|F|151089.96|1994-11-29|1-URGENT|Clerk#000000438|0|ng instructions integrate fluffily among the fluffily silent accounts. bli|
+5536|116|O|108196.56|1998-03-16|4-NOT SPECIFIED|Clerk#000000076|0| carefully final dolphins. ironic, ironic deposits lose. bold, |
+5537|118|O|102207.20|1996-10-03|3-MEDIUM|Clerk#000000742|0|ng to the daring, final |
+5538|139|F|90981.28|1993-12-25|1-URGENT|Clerk#000000992|0|ttainments. slyly final ideas are about the furiously silent excuses.|
+5539|119|F|39397.60|1994-07-31|5-LOW|Clerk#000000675|0|structions. slyly regular patterns solve above the carefully expres|
+5540|130|O|90707.58|1996-10-12|4-NOT SPECIFIED|Clerk#000000120|0|y ironic packages cajole blithely|
+5541|143|O|37526.68|1997-09-30|3-MEDIUM|Clerk#000000217|0|encies among the silent accounts sleep slyly quickly pending deposits|
+5542|49|O|6402.41|1996-04-20|4-NOT SPECIFIED|Clerk#000000100|0|riously among the regularly regular pac|
+5543|115|F|118201.53|1993-09-25|3-MEDIUM|Clerk#000000644|0|ckly regular epitaphs. carefully bold accounts haggle furiously|
+5568|31|O|105421.09|1995-06-07|3-MEDIUM|Clerk#000000491|0| nag. fluffily pending de|
+5569|109|F|126113.32|1993-04-30|4-NOT SPECIFIED|Clerk#000000759|0|e regular dependencies. furiously unusual ideas b|
+5570|112|O|78567.55|1996-08-12|2-HIGH|Clerk#000000795|0|eans. ironic, even requests doze |
+5571|103|F|79248.35|1992-12-19|4-NOT SPECIFIED|Clerk#000000184|0|ts cajole furiously carefully regular sheaves. un|
+5572|8|F|182966.39|1994-07-17|2-HIGH|Clerk#000000163|0|e fluffily express deposits cajole slyly across th|
+5573|37|O|158479.37|1996-08-15|3-MEDIUM|Clerk#000000055|0|lites. slyly final pinto beans about the carefully regul|
+5574|28|F|129803.03|1992-03-10|4-NOT SPECIFIED|Clerk#000000002|0|n deposits. special, regular t|
+5575|103|O|51839.94|1995-07-24|5-LOW|Clerk#000000948|0|uriously express frays breach|
+5600|95|O|53649.35|1997-02-08|4-NOT SPECIFIED|Clerk#000000019|0|lly regular deposits. car|
+5601|11|F|118570.79|1992-01-06|2-HIGH|Clerk#000000827|0|gular deposits wake platelets? blithe|
+5602|130|O|67979.49|1997-07-30|3-MEDIUM|Clerk#000000395|0|onic asymptotes haggl|
+5603|71|F|145100.47|1992-06-20|4-NOT SPECIFIED|Clerk#000000535|0| asymptotes. fluffily ironic instructions are. pending pinto bean|
+5604|46|O|98987.51|1998-04-14|4-NOT SPECIFIED|Clerk#000000123|0|ously across the blithely ironic pinto beans. sile|
+5605|35|O|172899.84|1996-08-22|2-HIGH|Clerk#000000538|0|sleep carefully final packages. dependencies wake slyly. theodol|
+5606|149|O|219959.08|1996-11-12|5-LOW|Clerk#000000688|0|uriously express pinto beans. packages sh|
+5607|92|F|24660.06|1992-01-01|4-NOT SPECIFIED|Clerk#000000137|0|c requests promise quickly fluffily ironic deposits. caref|
+5632|79|O|89503.11|1996-02-05|1-URGENT|Clerk#000000508|0|ons. blithely pending pinto beans thrash. furiously busy theodoli|
+5633|79|O|207119.83|1998-05-31|3-MEDIUM|Clerk#000000841|0|cial deposits wake final, final|
+5634|68|O|99494.67|1996-07-31|3-MEDIUM|Clerk#000000915|0|out the accounts. carefully ironic ideas are slyly. sheaves could h|
+5635|70|F|192217.86|1992-08-16|3-MEDIUM|Clerk#000000734|0|nal platelets sleep daringly. idle, final accounts about |
+5636|122|F|143350.75|1995-02-16|3-MEDIUM|Clerk#000000916|0|. boldly even Tiresias sleep. blithely ironic packages among the ca|
+5637|103|O|128776.90|1996-06-17|3-MEDIUM|Clerk#000000183|0|nic dolphins are regular packages. ironic pinto beans hagg|
+5638|109|F|79197.77|1994-01-17|1-URGENT|Clerk#000000355|0|enly bold deposits eat. special realms play against the regular, speci|
+5639|145|F|9669.46|1994-06-02|3-MEDIUM|Clerk#000000005|0|ending packages use after the blithely regular accounts. regular package|
+5664|119|O|186215.81|1998-07-23|2-HIGH|Clerk#000000789|0|the quickly ironic dolp|
+5665|100|F|129821.09|1993-06-28|4-NOT SPECIFIED|Clerk#000000513|0| carefully special instructions. ironic pinto beans nag slyly blithe|
+5666|14|F|121663.68|1994-02-02|2-HIGH|Clerk#000000396|0|mptotes. quickly final instructions are |
+5667|44|O|37301.25|1995-08-10|1-URGENT|Clerk#000000358|0|s print upon the quickly ironic packa|
+5668|109|F|13679.32|1995-03-22|4-NOT SPECIFIED|Clerk#000000047|0|p slyly slyly express accoun|
+5669|74|O|113156.30|1996-05-06|1-URGENT|Clerk#000000336|0|ng packages nag fluffily furio|
+5670|7|F|101429.61|1993-04-21|5-LOW|Clerk#000000922|0|he carefully final packages. deposits are slyly among the requests. |
+5671|43|O|176647.54|1998-02-06|2-HIGH|Clerk#000000838|0|k dependencies. slyly |
+5696|142|P|198723.30|1995-05-04|1-URGENT|Clerk#000000447|0|e quickly unusual pack|
+5697|55|F|99177.69|1992-10-05|1-URGENT|Clerk#000000112|0|pendencies impress furiously. bold, final requests solve ab|
+5698|95|F|154936.43|1994-05-21|3-MEDIUM|Clerk#000000455|0|he furiously silent accounts haggle blithely against the carefully unusual|
+5699|142|F|226314.91|1992-07-30|5-LOW|Clerk#000000311|0|o beans. ironic asymptotes boost. blithe, final courts integrate|
+5700|143|O|79901.18|1997-12-25|1-URGENT|Clerk#000000618|0|ly pending dolphins sleep carefully slyly pending i|
+5701|43|O|16689.19|1997-02-07|5-LOW|Clerk#000000798|0| blithely final pinto beans. blit|
+5702|97|F|153024.28|1993-09-07|4-NOT SPECIFIED|Clerk#000000743|0|ironic accounts. final accounts wake express deposits. final pac|
+5703|121|F|1816.28|1993-05-16|3-MEDIUM|Clerk#000000647|0|ly special instructions. slyly even reque|
+5728|80|F|85397.04|1994-12-11|4-NOT SPECIFIED|Clerk#000000426|0|furiously express pin|
+5729|44|F|88080.33|1994-10-10|2-HIGH|Clerk#000000843|0|uffily sly accounts about|
+5730|11|O|10934.84|1997-12-18|1-URGENT|Clerk#000000181|0|l platelets. ironic pinto beans wake slyly. quickly b|
+5731|8|O|57823.37|1997-05-17|5-LOW|Clerk#000000841|0| silent excuses among the express accounts wake |
+5732|37|O|28330.42|1997-08-03|1-URGENT|Clerk#000000910|0|he quickly bold asymptotes: final platelets wake quickly. blithely final pinto|
+5733|101|F|38545.97|1993-03-17|2-HIGH|Clerk#000000873|0|osits. pending accounts boost quickly. furiously permanent acco|
+5734|94|O|45860.94|1997-10-12|3-MEDIUM|Clerk#000000084|0|efully even braids detect blithely alo|
+5735|40|F|39358.51|1994-12-11|3-MEDIUM|Clerk#000000600|0| bold realms cajole slyly fu|
+5760|25|F|59404.77|1994-05-25|4-NOT SPECIFIED|Clerk#000000498|0|s among the blithely regular frays haggle ironically bold theodolites. al|
+5761|16|O|130345.90|1998-07-06|3-MEDIUM|Clerk#000000208|0|s asymptotes cajole boldly. regular, |
+5762|49|O|165019.32|1997-02-14|1-URGENT|Clerk#000000901|0|ly bold packages: slyly ironic deposits sleep quietly foxes. express a|
+5763|8|O|140838.11|1998-06-26|4-NOT SPECIFIED|Clerk#000000633|0|according to the furiously regular pinto beans. even accounts wake fu|
+5764|131|F|53212.95|1993-10-03|4-NOT SPECIFIED|Clerk#000000363|0| furiously regular deposits haggle fluffily around th|
+5765|52|F|249900.42|1994-12-15|5-LOW|Clerk#000000959|0|longside of the quickly final packages. instructions so|
+5766|49|F|47940.51|1993-09-27|5-LOW|Clerk#000000753|0|. quickly final packages print slyly. fu|
+5767|118|F|135643.87|1992-04-29|2-HIGH|Clerk#000000225|0|ts wake fluffily above the r|
+5792|26|F|158991.89|1993-04-04|2-HIGH|Clerk#000000731|0|packages. doggedly bold deposits integrate furiously across the|
+5793|37|O|119887.47|1997-07-13|2-HIGH|Clerk#000000294|0|thely. fluffily even instructi|
+5794|8|F|122823.78|1993-04-05|5-LOW|Clerk#000000855|0|t accounts kindle about the gifts. as|
+5795|37|F|35514.45|1992-05-05|2-HIGH|Clerk#000000581|0| even instructions x-ray ironic req|
+5796|149|O|23280.61|1996-01-23|3-MEDIUM|Clerk#000000326|0|eodolites. slyly ironic pinto beans at the silent, special request|
+5797|122|O|15313.61|1997-10-15|4-NOT SPECIFIED|Clerk#000000381|0|ng! packages against the blithely b|
+5798|106|O|125011.92|1998-03-30|5-LOW|Clerk#000000343|0|lent accounts affix quickly! platelets run slyly slyly final packages. f|
+5799|26|O|71381.21|1995-08-03|1-URGENT|Clerk#000000238|0| unusual deposits sleep blithely along the carefully even requests. care|
+5824|56|O|169107.85|1996-12-03|2-HIGH|Clerk#000000171|0|unusual packages. even ideas along the even requests are along th|
+5825|61|F|23020.62|1995-02-21|5-LOW|Clerk#000000494|0|regular packages use bravely.|
+5826|22|O|21119.86|1998-06-13|1-URGENT|Clerk#000000087|0|even, regular dependenc|
+5827|31|O|137297.71|1998-07-23|3-MEDIUM|Clerk#000000660|0|hely furiously blithe dolphins. slyly |
+5828|127|F|62172.34|1994-03-06|5-LOW|Clerk#000000377|0|ages boost never during the final packa|
+5829|125|O|183734.56|1997-01-11|1-URGENT|Clerk#000000196|0|gular accounts. bold accounts are blithely furiously ironic r|
+5830|85|F|28223.57|1993-03-25|3-MEDIUM|Clerk#000000233|0|lites haggle. ironic, ironic instructions maintain blit|
+5831|139|O|113505.19|1996-11-17|5-LOW|Clerk#000000585|0|s final, final pinto beans. unusual depos|
+5856|37|F|71460.49|1994-11-06|2-HIGH|Clerk#000000634|0|special excuses. slyly final theodolites cajole blithely furiou|
+5857|124|O|158345.31|1997-11-06|4-NOT SPECIFIED|Clerk#000000267|0|gage blithely. quickly special ac|
+5858|64|F|181320.50|1992-07-14|4-NOT SPECIFIED|Clerk#000000580|0|lyly pending dugouts believe through the ironic deposits. silent s|
+5859|5|O|210643.96|1997-04-23|1-URGENT|Clerk#000000993|0|requests boost. asymptotes across the deposits solve slyly furiously pendin|
+5860|13|F|9495.28|1992-02-20|4-NOT SPECIFIED|Clerk#000000079|0| beans. bold, special foxes sleep about the ir|
+5861|139|O|41450.19|1997-04-10|3-MEDIUM|Clerk#000000094|0|rthogs cajole slyly. express packages sleep blithely final |
+5862|64|O|30550.90|1997-02-20|1-URGENT|Clerk#000000039|0|leep beneath the quickly busy excuses. ironic theodolit|
+5863|65|F|67941.54|1993-11-22|3-MEDIUM|Clerk#000000774|0|ets about the slyly pending ideas sleep according to the blithely |
+5888|46|O|67167.19|1996-09-28|3-MEDIUM|Clerk#000000748|0|quickly against the furiously final requests. evenly fi|
+5889|22|O|15417.57|1995-05-23|5-LOW|Clerk#000000690|0|ites wake across the slyly ironic|
+5890|49|F|41162.24|1992-11-04|2-HIGH|Clerk#000000013|0|packages. final, final reques|
+5891|46|F|41760.00|1992-12-29|3-MEDIUM|Clerk#000000302|0|ounts haggle furiously abo|
+5892|101|P|92340.77|1995-05-09|5-LOW|Clerk#000000639|0| pending instruction|
+5893|2|F|44777.63|1992-07-08|4-NOT SPECIFIED|Clerk#000000560|0|final sentiments. instructions boost above the never speci|
+5894|71|F|70377.31|1994-08-13|2-HIGH|Clerk#000000776|0|regular deposits wake|
+5895|64|O|201419.83|1997-01-01|4-NOT SPECIFIED|Clerk#000000747|0| ironic, unusual requests cajole blithely special, special deposits. s|
+5920|119|F|142767.26|1994-11-20|2-HIGH|Clerk#000000081|0|ns: even ideas cajole slyly among the packages. never ironic patterns|
+5921|58|F|152940.00|1994-04-07|5-LOW|Clerk#000000125|0|kly special requests breach.|
+5922|143|O|142494.99|1996-11-14|5-LOW|Clerk#000000625|0| ironic instructions haggle furiously blithely regular accounts: even platele|
+5923|101|O|157968.27|1997-05-27|2-HIGH|Clerk#000000304|0|o beans haggle slyly above the regular, even dependencies|
+5924|31|O|106823.97|1995-10-10|3-MEDIUM|Clerk#000000433|0|arefully after the pains. blithely ironic pinto |
+5925|146|O|242588.87|1995-11-13|5-LOW|Clerk#000000602|0|ourts. boldly regular foxes might sleep. slyly express tithes against |
+5926|76|F|105770.53|1994-05-20|5-LOW|Clerk#000000071|0| carefully after the furiously even re|
+5927|116|O|84983.90|1997-08-28|4-NOT SPECIFIED|Clerk#000000972|0|endencies according to the slyly ironic foxes detect furiously about the furio|
+5952|148|O|128624.99|1997-04-14|3-MEDIUM|Clerk#000000950|0| regular, final pla|
+5953|7|F|95312.81|1992-03-28|1-URGENT|Clerk#000000049|0|ages are furiously. slowly bold requests|
+5954|28|F|167262.34|1992-12-03|1-URGENT|Clerk#000000968|0|requests along the blith|
+5955|94|P|67944.38|1995-03-27|5-LOW|Clerk#000000340|0|deas integrate. fluffily regular pa|
+5956|22|O|118036.54|1998-05-18|1-URGENT|Clerk#000000587|0|le even, express platelets.|
+5957|89|F|230949.45|1993-12-27|2-HIGH|Clerk#000000020|0| dependencies are slyly. bold accounts according to the carefully regular r|
+5958|115|O|145060.41|1995-09-16|3-MEDIUM|Clerk#000000787|0|e final requests detect alongside of the qu|
+5959|23|F|195515.26|1992-05-15|3-MEDIUM|Clerk#000000913|0|into beans use ironic, unusual foxes. carefully regular excuses boost caref|
+5984|70|F|83413.30|1994-06-18|5-LOW|Clerk#000000023|0|ickly final pains haggle along the furiously ironic pinto bea|
+5985|143|F|3942.73|1995-01-12|3-MEDIUM|Clerk#000000417|0|as nag fluffily slyly permanent accounts. regular depo|
+5986|115|F|92187.80|1992-04-22|2-HIGH|Clerk#000000674|0|iously unusual notornis are |
+5987|64|O|98956.82|1996-08-03|1-URGENT|Clerk#000000464|0| ideas. quietly final accounts haggle blithely pending escapade|
+5988|31|F|41655.51|1993-11-22|4-NOT SPECIFIED|Clerk#000000867|0|fully express accounts. final pi|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/orders.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/orders.tbl
new file mode 100644
index 0000000..1ebd663
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/orders.tbl
@@ -0,0 +1,1500 @@
+1|37|O|131251.81|1996-01-02|5-LOW|Clerk#000000951|0|nstructions sleep furiously among |
+2|79|O|40183.29|1996-12-01|1-URGENT|Clerk#000000880|0| foxes. pending accounts at the pending, silent asymptot|
+3|124|F|160882.76|1993-10-14|5-LOW|Clerk#000000955|0|sly final accounts boost. carefully regular ideas cajole carefully. depos|
+4|137|O|31084.79|1995-10-11|5-LOW|Clerk#000000124|0|sits. slyly regular warthogs cajole. regular, regular theodolites acro|
+5|46|F|86615.25|1994-07-30|5-LOW|Clerk#000000925|0|quickly. bold deposits sleep slyly. packages use slyly|
+6|56|F|36468.55|1992-02-21|4-NOT SPECIFIED|Clerk#000000058|0|ggle. special, final requests are against the furiously specia|
+7|40|O|171488.73|1996-01-10|2-HIGH|Clerk#000000470|0|ly special requests |
+32|131|O|116923.00|1995-07-16|2-HIGH|Clerk#000000616|0|ise blithely bold, regular requests. quickly unusual dep|
+33|67|F|99798.76|1993-10-27|3-MEDIUM|Clerk#000000409|0|uriously. furiously final request|
+34|62|O|41670.02|1998-07-21|3-MEDIUM|Clerk#000000223|0|ly final packages. fluffily final deposits wake blithely ideas. spe|
+35|128|O|148789.52|1995-10-23|4-NOT SPECIFIED|Clerk#000000259|0|zzle. carefully enticing deposits nag furio|
+36|116|O|38988.98|1995-11-03|1-URGENT|Clerk#000000358|0| quick packages are blithely. slyly silent accounts wake qu|
+37|88|F|113701.89|1992-06-03|3-MEDIUM|Clerk#000000456|0|kly regular pinto beans. carefully unusual waters cajole never|
+38|125|O|46366.56|1996-08-21|4-NOT SPECIFIED|Clerk#000000604|0|haggle blithely. furiously express ideas haggle blithely furiously regular re|
+39|82|O|219707.84|1996-09-20|3-MEDIUM|Clerk#000000659|0|ole express, ironic requests: ir|
+64|34|F|20065.73|1994-07-16|3-MEDIUM|Clerk#000000661|0|wake fluffily. sometimes ironic pinto beans about the dolphin|
+65|17|P|65883.92|1995-03-18|1-URGENT|Clerk#000000632|0|ular requests are blithely pending orbits-- even requests against the deposit|
+66|130|F|79258.24|1994-01-20|5-LOW|Clerk#000000743|0|y pending requests integrate|
+67|58|O|116227.05|1996-12-19|4-NOT SPECIFIED|Clerk#000000547|0|symptotes haggle slyly around the furiously iron|
+68|29|O|215135.72|1998-04-18|3-MEDIUM|Clerk#000000440|0| pinto beans sleep carefully. blithely ironic deposits haggle furiously acro|
+69|85|F|162176.23|1994-06-04|4-NOT SPECIFIED|Clerk#000000330|0| depths atop the slyly thin deposits detect among the furiously silent accou|
+70|65|F|84651.80|1993-12-18|5-LOW|Clerk#000000322|0| carefully ironic request|
+71|4|O|178821.73|1998-01-24|4-NOT SPECIFIED|Clerk#000000271|0| express deposits along the blithely regul|
+96|109|F|55090.67|1994-04-17|2-HIGH|Clerk#000000395|0|oost furiously. pinto|
+97|22|F|68908.31|1993-01-29|3-MEDIUM|Clerk#000000547|0|hang blithely along the regular accounts. furiously even ideas after the|
+98|106|F|51004.44|1994-09-25|1-URGENT|Clerk#000000448|0|c asymptotes. quickly regular packages should have to nag re|
+99|89|F|92326.79|1994-03-13|4-NOT SPECIFIED|Clerk#000000973|0|e carefully ironic packages. pending|
+100|148|O|141311.01|1998-02-28|4-NOT SPECIFIED|Clerk#000000577|0|heodolites detect slyly alongside of the ent|
+101|28|O|95591.40|1996-03-17|3-MEDIUM|Clerk#000000419|0|ding accounts above the slyly final asymptote|
+102|1|O|113954.89|1997-05-09|2-HIGH|Clerk#000000596|0| slyly according to the asymptotes. carefully final packages integrate furious|
+103|31|O|95563.95|1996-06-20|4-NOT SPECIFIED|Clerk#000000090|0|ges. carefully unusual instructions haggle quickly regular f|
+128|74|F|36333.34|1992-06-15|1-URGENT|Clerk#000000385|0|ns integrate fluffily. ironic asymptotes after the regular excuses nag around |
+129|73|F|188124.55|1992-11-19|5-LOW|Clerk#000000859|0|ing tithes. carefully pending deposits boost about the silently express |
+130|37|F|115717.37|1992-05-08|2-HIGH|Clerk#000000036|0|le slyly unusual, regular packages? express deposits det|
+131|94|F|96596.81|1994-06-08|3-MEDIUM|Clerk#000000625|0|after the fluffily special foxes integrate s|
+132|28|F|118802.62|1993-06-11|3-MEDIUM|Clerk#000000488|0|sits are daringly accounts. carefully regular foxes sleep slyly about the|
+133|44|O|80437.72|1997-11-29|1-URGENT|Clerk#000000738|0|usly final asymptotes |
+134|7|F|154260.84|1992-05-01|4-NOT SPECIFIED|Clerk#000000711|0|lar theodolites boos|
+135|61|O|174569.88|1995-10-21|4-NOT SPECIFIED|Clerk#000000804|0|l platelets use according t|
+160|83|O|86076.86|1996-12-19|4-NOT SPECIFIED|Clerk#000000342|0|thely special sauternes wake slyly of t|
+161|17|F|19056.99|1994-08-31|2-HIGH|Clerk#000000322|0|carefully! special instructions sin|
+162|16|O|2158.13|1995-05-08|3-MEDIUM|Clerk#000000378|0|nts hinder fluffily ironic instructions. express, express excuses |
+163|88|O|125170.86|1997-09-05|3-MEDIUM|Clerk#000000379|0|y final packages. final foxes since the quickly even|
+164|1|F|202660.52|1992-10-21|5-LOW|Clerk#000000209|0|cajole ironic courts. slyly final ideas are slyly. blithely final Tiresias sub|
+165|28|F|141824.23|1993-01-30|4-NOT SPECIFIED|Clerk#000000292|0|across the blithely regular accounts. bold|
+166|109|O|93335.60|1995-09-12|2-HIGH|Clerk#000000440|0|lets. ironic, bold asymptotes kindle|
+167|121|F|52982.23|1993-01-04|4-NOT SPECIFIED|Clerk#000000731|0|s nag furiously bold excuses. fluffily iron|
+192|83|O|133002.55|1997-11-25|5-LOW|Clerk#000000483|0|y unusual platelets among the final instructions integrate rut|
+193|80|F|48053.18|1993-08-08|1-URGENT|Clerk#000000025|0|the furiously final pin|
+194|62|F|114097.63|1992-04-05|3-MEDIUM|Clerk#000000352|0|egular requests haggle slyly regular, regular pinto beans. asymptote|
+195|136|F|120053.52|1993-12-28|3-MEDIUM|Clerk#000000216|0|old forges are furiously sheaves. slyly fi|
+196|65|F|33248.04|1993-03-17|2-HIGH|Clerk#000000988|0|beans boost at the foxes. silent foxes|
+197|34|P|100290.07|1995-04-07|2-HIGH|Clerk#000000969|0|solve quickly about the even braids. carefully express deposits affix care|
+198|112|O|125792.83|1998-01-02|4-NOT SPECIFIED|Clerk#000000331|0|its. carefully ironic requests sleep. furiously express fox|
+199|53|O|80592.44|1996-03-07|2-HIGH|Clerk#000000489|0|g theodolites. special packag|
+224|4|F|155680.60|1994-06-18|4-NOT SPECIFIED|Clerk#000000642|0|r the quickly thin courts. carefully|
+225|34|P|165890.47|1995-05-25|1-URGENT|Clerk#000000177|0|s. blithely ironic accounts wake quickly fluffily special acc|
+226|128|F|180119.22|1993-03-10|2-HIGH|Clerk#000000756|0|s are carefully at the blithely ironic acc|
+227|10|O|46076.46|1995-11-10|5-LOW|Clerk#000000919|0| express instructions. slyly regul|
+228|46|F|2638.98|1993-02-25|1-URGENT|Clerk#000000562|0|es was slyly among the regular foxes. blithely regular dependenci|
+229|112|F|142290.77|1993-12-29|1-URGENT|Clerk#000000628|0|he fluffily even instructions. furiously i|
+230|103|F|107231.60|1993-10-27|1-URGENT|Clerk#000000520|0|odolites. carefully quick requ|
+231|91|F|141554.06|1994-09-29|2-HIGH|Clerk#000000446|0| packages haggle slyly after the carefully ironic instruct|
+256|125|F|106315.25|1993-10-19|4-NOT SPECIFIED|Clerk#000000834|0|he fluffily final ideas might are final accounts. carefully f|
+257|124|O|7102.74|1998-03-28|3-MEDIUM|Clerk#000000680|0|ts against the sly warhorses cajole slyly accounts|
+258|43|F|186669.10|1993-12-29|1-URGENT|Clerk#000000167|0|dencies. blithely quick packages cajole. ruthlessly final accounts|
+259|44|F|75661.70|1993-09-29|4-NOT SPECIFIED|Clerk#000000601|0|ages doubt blithely against the final foxes. carefully express deposits dazzle|
+260|106|O|179292.14|1996-12-10|3-MEDIUM|Clerk#000000960|0|lently regular pinto beans sleep after the slyly e|
+261|47|F|201003.12|1993-06-29|3-MEDIUM|Clerk#000000310|0|ully fluffily brave instructions. furiousl|
+262|31|O|108443.84|1995-11-25|4-NOT SPECIFIED|Clerk#000000551|0|l packages. blithely final pinto beans use carefu|
+263|118|F|79782.56|1994-05-17|2-HIGH|Clerk#000000088|0| pending instructions. blithely un|
+288|8|O|163794.53|1997-02-21|1-URGENT|Clerk#000000109|0|uriously final requests. even, final ideas det|
+289|104|O|131092.67|1997-02-10|3-MEDIUM|Clerk#000000103|0|sily. slyly special excuse|
+290|118|F|62814.89|1994-01-01|4-NOT SPECIFIED|Clerk#000000735|0|efully dogged deposits. furiou|
+291|142|F|66817.05|1994-03-13|1-URGENT|Clerk#000000923|0|dolites. carefully regular pinto beans cajol|
+292|23|F|30783.05|1992-01-13|2-HIGH|Clerk#000000193|0|g pinto beans will have to sleep f|
+293|31|F|37248.78|1992-10-02|2-HIGH|Clerk#000000629|0|re bold, ironic deposits. platelets c|
+294|52|F|30059.47|1993-07-16|3-MEDIUM|Clerk#000000499|0|kly according to the frays. final dolphins affix quickly |
+295|19|F|89345.99|1994-09-29|2-HIGH|Clerk#000000155|0| unusual pinto beans play. regular ideas haggle|
+320|1|O|39835.54|1997-11-21|2-HIGH|Clerk#000000573|0|ar foxes nag blithely|
+321|124|F|62251.15|1993-03-21|3-MEDIUM|Clerk#000000289|0|equests run. blithely final dependencies after the deposits wake caref|
+322|134|F|127068.89|1992-03-19|1-URGENT|Clerk#000000158|0|fully across the slyly bold packages. packages against the quickly regular i|
+323|40|F|79683.42|1994-03-26|1-URGENT|Clerk#000000959|0|arefully pending foxes sleep blithely. slyly express accoun|
+324|106|F|26868.85|1992-03-20|1-URGENT|Clerk#000000352|0| about the ironic, regular deposits run blithely against the excuses|
+325|41|F|71543.41|1993-10-17|5-LOW|Clerk#000000844|0|ly sometimes pending pa|
+326|76|O|229165.17|1995-06-04|2-HIGH|Clerk#000000466|0| requests. furiously ironic asymptotes mold carefully alongside of the blit|
+327|145|P|24468.16|1995-04-17|5-LOW|Clerk#000000992|0|ng the slyly final courts. slyly even escapades eat |
+352|107|F|16003.86|1994-03-08|2-HIGH|Clerk#000000932|0|ke slyly bold pinto beans. blithely regular accounts against the spe|
+353|2|F|179984.42|1993-12-31|5-LOW|Clerk#000000449|0| quiet ideas sleep. even instructions cajole slyly. silently spe|
+354|139|O|157062.70|1996-03-14|2-HIGH|Clerk#000000511|0|ly regular ideas wake across the slyly silent ideas. final deposits eat b|
+355|71|F|69447.25|1994-06-14|5-LOW|Clerk#000000532|0|s. sometimes regular requests cajole. regular, pending accounts a|
+356|148|F|162786.67|1994-06-30|4-NOT SPECIFIED|Clerk#000000944|0|as wake along the bold accounts. even, |
+357|61|O|98723.11|1996-10-09|2-HIGH|Clerk#000000301|0|e blithely about the express, final accounts. quickl|
+358|4|F|226806.66|1993-09-20|2-HIGH|Clerk#000000392|0|l, silent instructions are slyly. silently even de|
+359|79|F|142891.22|1994-12-19|3-MEDIUM|Clerk#000000934|0|n dolphins. special courts above the carefully ironic requests use|
+384|115|F|122785.82|1992-03-03|5-LOW|Clerk#000000206|0|, even accounts use furiously packages. slyly ironic pla|
+385|34|O|50724.06|1996-03-22|5-LOW|Clerk#000000600|0|hless accounts unwind bold pain|
+386|61|F|90380.40|1995-01-25|2-HIGH|Clerk#000000648|0| haggle quickly. stealthily bold asymptotes haggle among the furiously even re|
+387|4|O|130647.18|1997-01-26|4-NOT SPECIFIED|Clerk#000000768|0| are carefully among the quickly even deposits. furiously silent req|
+388|46|F|120533.46|1992-12-16|4-NOT SPECIFIED|Clerk#000000356|0|ar foxes above the furiously ironic deposits nag slyly final reque|
+389|127|F|1984.14|1994-02-17|2-HIGH|Clerk#000000062|0|ing to the regular asymptotes. final, pending foxes about the blithely sil|
+390|103|O|168562.27|1998-04-07|5-LOW|Clerk#000000404|0|xpress asymptotes use among the regular, final pinto b|
+391|112|F|13282.23|1994-11-17|2-HIGH|Clerk#000000256|0|orges thrash fluffil|
+416|41|F|71362.50|1993-09-27|5-LOW|Clerk#000000294|0| the accounts. fluffily bold depo|
+417|55|F|91982.29|1994-02-06|3-MEDIUM|Clerk#000000468|0|ironic, even packages. thinly unusual accounts sleep along the slyly unusual |
+418|95|P|33124.96|1995-04-13|4-NOT SPECIFIED|Clerk#000000643|0|. furiously ironic instruc|
+419|118|O|111597.96|1996-10-01|3-MEDIUM|Clerk#000000376|0|osits. blithely pending theodolites boost carefully|
+420|91|O|198039.23|1995-10-31|4-NOT SPECIFIED|Clerk#000000756|0|leep carefully final excuses. fluffily pending requests unwind carefully above|
+421|40|F|1084.38|1992-02-22|5-LOW|Clerk#000000405|0|egular, even packages according to the final, un|
+422|74|O|106045.89|1997-05-31|4-NOT SPECIFIED|Clerk#000000049|0|aggle carefully across the accounts. regular accounts eat fluffi|
+423|104|O|26981.31|1996-06-01|1-URGENT|Clerk#000000674|0|quests. deposits cajole quickly. furiously bold accounts haggle q|
+448|149|O|114978.03|1995-08-21|3-MEDIUM|Clerk#000000597|0| regular, express foxes use blithely. quic|
+449|97|O|41605.63|1995-07-20|2-HIGH|Clerk#000000841|0|. furiously regular theodolites affix blithely |
+450|49|P|153386.61|1995-03-05|4-NOT SPECIFIED|Clerk#000000293|0|d theodolites. boldly bold foxes since the pack|
+451|100|O|104664.40|1998-05-25|5-LOW|Clerk#000000048|0|nic pinto beans. theodolites poach carefully; |
+452|61|O|2007.48|1997-10-14|1-URGENT|Clerk#000000498|0|t, unusual instructions above the blithely bold pint|
+453|46|O|216826.73|1997-05-26|5-LOW|Clerk#000000504|0|ss foxes. furiously regular ideas sleep according to t|
+454|49|O|23198.24|1995-12-27|5-LOW|Clerk#000000890|0|dolites sleep carefully blithely regular deposits. quickly regul|
+455|13|O|138010.76|1996-12-04|1-URGENT|Clerk#000000796|0| about the final platelets. dependen|
+480|73|F|20530.97|1993-05-08|5-LOW|Clerk#000000004|0|ealthy pinto beans. fluffily regular requests along the special sheaves wake |
+481|31|F|117827.18|1992-10-08|2-HIGH|Clerk#000000230|0|ly final ideas. packages haggle fluffily|
+482|127|O|136634.34|1996-03-26|1-URGENT|Clerk#000000295|0|ts. deposits wake: final acco|
+483|35|O|39793.05|1995-07-11|2-HIGH|Clerk#000000025|0|cross the carefully final e|
+484|55|O|219920.62|1997-01-03|3-MEDIUM|Clerk#000000545|0|grouches use. furiously bold accounts maintain. bold, regular deposits|
+485|101|O|110432.76|1997-03-26|2-HIGH|Clerk#000000105|0| regular ideas nag thinly furiously s|
+486|52|O|185968.15|1996-03-11|4-NOT SPECIFIED|Clerk#000000803|0|riously dolphins. fluffily ironic requ|
+487|109|F|48502.79|1992-08-18|1-URGENT|Clerk#000000086|0|ithely unusual courts eat accordi|
+512|64|P|124661.48|1995-05-20|5-LOW|Clerk#000000814|0|ding requests. carefully express theodolites was quickly. furious|
+513|61|O|63703.92|1995-05-01|2-HIGH|Clerk#000000522|0|regular packages. pinto beans cajole carefully against the even|
+514|76|O|104585.77|1996-04-04|2-HIGH|Clerk#000000094|0| cajole furiously. slyly final excuses cajole. slyly special instructions |
+515|142|F|153720.22|1993-08-29|4-NOT SPECIFIED|Clerk#000000700|0|eposits are furiously furiously silent pinto beans. pending pack|
+516|44|O|10677.86|1998-04-21|2-HIGH|Clerk#000000305|0|lar, unusual platelets are carefully. even courts sleep bold, final pinto bea|
+517|10|O|82197.79|1997-04-07|5-LOW|Clerk#000000359|0|slyly pending deposits cajole quickly packages. furiou|
+518|145|O|223537.09|1998-02-08|2-HIGH|Clerk#000000768|0| the carefully bold accounts. quickly regular excuses are|
+519|64|O|95731.50|1997-10-31|1-URGENT|Clerk#000000985|0|ains doze furiously against the f|
+544|94|F|47627.89|1993-02-17|2-HIGH|Clerk#000000145|0|the special, final accounts. dogged dolphins|
+545|64|O|23476.12|1995-11-07|2-HIGH|Clerk#000000537|0|as. blithely final hockey players about th|
+546|145|O|14790.37|1996-11-01|2-HIGH|Clerk#000000041|0|osits sleep. slyly special dolphins about the q|
+547|100|O|96855.29|1996-06-22|3-MEDIUM|Clerk#000000976|0|ing accounts eat. carefully regular packa|
+548|124|F|99088.75|1994-09-21|1-URGENT|Clerk#000000435|0|arefully express instru|
+549|110|F|141679.41|1992-07-13|1-URGENT|Clerk#000000196|0|ideas alongside of |
+550|25|O|33123.28|1995-08-02|1-URGENT|Clerk#000000204|0|t requests. blithely |
+551|91|O|46355.83|1995-05-30|1-URGENT|Clerk#000000179|0|xpress accounts boost quic|
+576|31|O|18307.45|1997-05-13|3-MEDIUM|Clerk#000000955|0|l requests affix regular requests. final account|
+577|56|F|34768.68|1994-12-19|5-LOW|Clerk#000000154|0| deposits engage stealthil|
+578|94|O|70392.02|1997-01-10|5-LOW|Clerk#000000281|0|e blithely even packages. slyly pending platelets bes|
+579|68|O|120828.12|1998-03-11|2-HIGH|Clerk#000000862|0| regular instructions. blithely even p|
+580|61|O|88219.12|1997-07-05|2-HIGH|Clerk#000000314|0|tegrate fluffily regular accou|
+581|70|O|126066.00|1997-02-23|4-NOT SPECIFIED|Clerk#000000239|0| requests. even requests use slyly. blithely ironic |
+582|50|O|129004.81|1997-10-21|1-URGENT|Clerk#000000378|0|n pinto beans print a|
+583|49|O|127817.38|1997-03-19|3-MEDIUM|Clerk#000000792|0|efully express requests. a|
+608|26|O|62567.99|1996-02-28|3-MEDIUM|Clerk#000000995|0|nic waters wake slyly slyly expre|
+609|127|F|21088.59|1994-06-01|3-MEDIUM|Clerk#000000348|0|- ironic gifts believe furiously ca|
+610|52|O|175142.28|1995-08-02|1-URGENT|Clerk#000000610|0|totes. ironic, unusual packag|
+611|106|F|73907.63|1993-01-27|1-URGENT|Clerk#000000401|0|ounts detect furiously ac|
+612|82|F|145695.42|1992-10-21|3-MEDIUM|Clerk#000000759|0|boost quickly quickly final excuses. final foxes use bravely afte|
+613|139|O|33396.35|1995-06-18|2-HIGH|Clerk#000000172|0|ts hinder among the deposits. fluffily ironic depos|
+614|134|F|218116.21|1992-12-01|2-HIGH|Clerk#000000388|0| deposits! even, daring theodol|
+615|67|F|32890.89|1992-05-09|5-LOW|Clerk#000000388|0|t to promise asymptotes. packages haggle alongside of the fluffil|
+640|97|F|145495.62|1993-01-23|2-HIGH|Clerk#000000433|0|r, unusual accounts boost carefully final ideas. slyly silent theod|
+641|133|F|120626.49|1993-08-30|5-LOW|Clerk#000000175|0|ents cajole furiously about the quickly silent pac|
+642|40|F|22994.51|1993-12-16|3-MEDIUM|Clerk#000000357|0| among the requests wake slyly alongside of th|
+643|58|P|180396.95|1995-03-25|2-HIGH|Clerk#000000354|0|g dependencies. regular accounts |
+644|8|F|201268.06|1992-05-01|1-URGENT|Clerk#000000550|0| blithely unusual platelets haggle ironic, special excuses. excuses unwi|
+645|115|F|234763.73|1994-12-03|2-HIGH|Clerk#000000090|0|quickly daring theodolites across the regu|
+646|52|F|142070.65|1994-11-22|2-HIGH|Clerk#000000203|0|carefully even foxes. fina|
+647|143|O|56449.23|1997-08-07|1-URGENT|Clerk#000000270|0|egular pearls. carefully express asymptotes are. even account|
+672|109|F|89877.09|1994-04-14|5-LOW|Clerk#000000106|0|egular requests are furiously according to |
+673|80|F|21137.08|1994-03-10|1-URGENT|Clerk#000000448|0| special pinto beans use quickly furiously even depende|
+674|34|F|27204.60|1992-08-29|5-LOW|Clerk#000000448|0|ully special deposits. furiously final warhorses affix carefully. fluffily f|
+675|13|O|125188.72|1997-07-31|2-HIGH|Clerk#000000168|0|ffily between the careful|
+676|38|O|163966.67|1996-12-13|2-HIGH|Clerk#000000248|0|the final deposits. special, pending|
+677|124|F|147915.68|1993-11-24|3-MEDIUM|Clerk#000000824|0|uriously special pinto beans cajole carefully. fi|
+678|131|F|135761.05|1993-02-27|5-LOW|Clerk#000000530|0|. blithely final somas about the|
+679|49|O|8945.03|1995-12-15|2-HIGH|Clerk#000000853|0|tealthy, final pinto beans haggle slyly. pending platelets about the special, |
+704|85|O|56210.26|1996-11-21|3-MEDIUM|Clerk#000000682|0|blithely pending platelets wake alongside of the final, iron|
+705|43|O|83773.49|1997-02-13|4-NOT SPECIFIED|Clerk#000000294|0|ithely regular dependencies. express, even packages sleep slyly pending t|
+706|148|O|23973.60|1995-09-09|1-URGENT|Clerk#000000448|0|g the packages. deposits caj|
+707|118|F|58218.35|1994-11-20|3-MEDIUM|Clerk#000000199|0| ideas about the silent, bold deposits nag dolphins|
+708|32|O|100445.59|1998-07-03|3-MEDIUM|Clerk#000000101|0|lphins cajole about t|
+709|37|O|72055.87|1998-04-21|1-URGENT|Clerk#000000461|0|ons alongside of the carefully bold pinto bea|
+710|133|F|208974.42|1993-01-02|5-LOW|Clerk#000000026|0| regular, regular requests boost. fluffily re|
+711|64|F|92484.70|1993-09-23|4-NOT SPECIFIED|Clerk#000000856|0|its. fluffily regular gifts are furi|
+736|47|O|130204.17|1998-06-21|5-LOW|Clerk#000000881|0|refully of the final pi|
+737|121|F|12984.85|1992-04-26|5-LOW|Clerk#000000233|0|ake blithely express, ironic theodolites. blithely special accounts wa|
+738|22|F|114145.18|1993-03-02|4-NOT SPECIFIED|Clerk#000000669|0|ly even foxes. furiously regular accounts cajole ca|
+739|1|O|159171.69|1998-05-31|5-LOW|Clerk#000000900|0| against the slyly ironic packages nag slyly ironic|
+740|44|O|83490.99|1995-07-16|3-MEDIUM|Clerk#000000583|0|courts haggle furiously across the final, regul|
+741|106|O|47985.98|1998-07-07|2-HIGH|Clerk#000000295|0|ic instructions. slyly express instructions solv|
+742|103|F|207632.55|1994-12-23|5-LOW|Clerk#000000543|0|equests? slyly ironic dolphins boost carefully above the blithely|
+743|79|O|23614.89|1996-10-04|4-NOT SPECIFIED|Clerk#000000933|0|eans. furiously ironic deposits sleep carefully carefully qui|
+768|98|O|220636.82|1996-08-20|3-MEDIUM|Clerk#000000411|0|jole slyly ironic packages. slyly even idea|
+769|80|F|43092.76|1993-06-02|3-MEDIUM|Clerk#000000172|0|ggle furiously. ironic packages haggle slyly. bold platelets affix s|
+770|32|O|64271.75|1998-05-23|5-LOW|Clerk#000000572|0|heodolites. furiously special pinto beans cajole pac|
+771|46|O|105302.05|1995-06-17|1-URGENT|Clerk#000000105|0|s. furiously final instructions across the deposit|
+772|97|F|128234.96|1993-04-17|2-HIGH|Clerk#000000430|0|s boost blithely fluffily idle ideas? fluffily even pin|
+773|133|F|146862.27|1993-09-26|3-MEDIUM|Clerk#000000307|0|tions are quickly accounts. accounts use bold, even pinto beans. gifts ag|
+774|80|O|145857.60|1995-12-04|1-URGENT|Clerk#000000883|0|tealthily even depths|
+775|134|F|59455.61|1995-03-18|1-URGENT|Clerk#000000191|0|kly express requests. fluffily silent accounts poach furiously|
+800|56|O|87892.38|1998-07-14|2-HIGH|Clerk#000000213|0|y alongside of the pending packages? final platelets nag fluffily carefu|
+801|118|F|127717.72|1992-02-18|1-URGENT|Clerk#000000186|0|iously from the furiously enticing reques|
+802|137|F|156381.95|1995-01-05|1-URGENT|Clerk#000000516|0|posits. ironic, pending requests cajole. even theodol|
+803|16|O|27629.66|1997-04-29|5-LOW|Clerk#000000260|0|ic instructions. even deposits haggle furiously at the deposits-- regular de|
+804|50|F|94400.43|1993-03-12|3-MEDIUM|Clerk#000000931|0|s. blithely final foxes are about the packag|
+805|127|O|90042.41|1995-07-05|4-NOT SPECIFIED|Clerk#000000856|0|y according to the fluffily |
+806|131|O|26839.16|1996-06-20|2-HIGH|Clerk#000000240|0| the ironic packages wake carefully fina|
+807|145|F|222392.53|1993-11-24|3-MEDIUM|Clerk#000000012|0|refully special tithes. blithely regular accoun|
+832|29|F|68494.08|1992-04-19|5-LOW|Clerk#000000495|0|xes. bravely regular packages sleep up the furiously bold accou|
+833|56|F|49033.69|1994-02-13|3-MEDIUM|Clerk#000000437|0|ts haggle quickly across the slyl|
+834|43|F|46459.92|1994-05-23|3-MEDIUM|Clerk#000000805|0| sleep. quickly even foxes are boldly. slyly express requests use slyly|
+835|65|O|62430.67|1995-10-08|4-NOT SPECIFIED|Clerk#000000416|0|s about the carefully special foxes haggle quickly about the|
+836|70|O|72843.48|1996-11-25|4-NOT SPECIFIED|Clerk#000000729|0|ely bold excuses sleep regular ideas. furiously unusual ideas wake furiou|
+837|116|F|60918.41|1994-06-15|4-NOT SPECIFIED|Clerk#000000563|0|kages sleep slyly above the ironic, final orbits|
+838|17|O|82918.36|1998-01-29|5-LOW|Clerk#000000213|0| slyly around the slyly even|
+839|28|O|70182.63|1995-08-08|1-URGENT|Clerk#000000951|0|the carefully even platelets. furiously unusual fo|
+864|139|O|74710.74|1997-08-17|1-URGENT|Clerk#000000036|0|ly after the slyly regular deposits. express, regular asymptotes nag ca|
+865|4|F|70430.54|1993-05-04|3-MEDIUM|Clerk#000000337|0|. special packages wake after the carefully final accounts. express pinto be|
+866|40|F|4766.19|1992-11-28|3-MEDIUM|Clerk#000000718|0|ins after the even, even accounts nod blithel|
+867|26|F|7471.75|1993-11-16|3-MEDIUM|Clerk#000000877|0|pades nag quickly final, |
+868|104|F|127345.45|1992-06-09|4-NOT SPECIFIED|Clerk#000000782|0|onic theodolites print carefully. blithely dogge|
+869|136|O|58932.19|1997-01-12|2-HIGH|Clerk#000000245|0|ar sheaves are slowly. slyly even attainments boost theodolites. furiously|
+870|34|F|40492.37|1993-06-20|4-NOT SPECIFIED|Clerk#000000123|0|blithely ironic ideas nod. sly, r|
+871|16|O|172861.58|1995-11-15|5-LOW|Clerk#000000882|0|oss the ironic theodolites.|
+896|2|F|169847.63|1993-03-09|1-URGENT|Clerk#000000187|0|inal packages eat blithely according to the warhorses. furiously quiet de|
+897|49|P|57697.44|1995-03-20|1-URGENT|Clerk#000000316|0| wake quickly against |
+898|55|F|101020.75|1993-06-03|2-HIGH|Clerk#000000611|0|. unusual pinto beans haggle quickly across |
+899|109|O|125562.09|1998-04-08|5-LOW|Clerk#000000575|0|rts engage carefully final theodolites.|
+900|46|F|120073.51|1994-10-01|4-NOT SPECIFIED|Clerk#000000060|0| fluffily express deposits nag furiousl|
+901|13|O|81826.12|1998-07-21|4-NOT SPECIFIED|Clerk#000000929|0|lyly even foxes are furious, silent requests. requests about the quickly |
+902|10|F|37348.62|1994-07-27|4-NOT SPECIFIED|Clerk#000000811|0|yly final requests over the furiously regula|
+903|11|O|109351.87|1995-07-07|4-NOT SPECIFIED|Clerk#000000793|0|e slyly about the final pl|
+928|67|F|228136.49|1995-03-02|5-LOW|Clerk#000000450|0|ithely express pinto beans. |
+929|83|F|109301.02|1992-10-02|2-HIGH|Clerk#000000160|0|its. furiously even foxes affix carefully finally silent accounts. express req|
+930|131|F|199102.23|1994-12-17|1-URGENT|Clerk#000000004|0| accounts nag slyly. ironic, ironic accounts wake blithel|
+931|103|F|117909.23|1992-12-07|1-URGENT|Clerk#000000881|0|ss packages haggle furiously express, regular deposits. even, e|
+932|41|O|40234.50|1997-05-16|2-HIGH|Clerk#000000218|0|ly express instructions boost furiously reg|
+933|97|F|71349.30|1992-08-05|4-NOT SPECIFIED|Clerk#000000752|0|ial courts wake permanently against the furiously regular ideas. unusual |
+934|52|O|17213.59|1996-07-03|1-URGENT|Clerk#000000229|0|ts integrate carefully. sly, regular deposits af|
+935|50|O|97733.87|1997-09-24|5-LOW|Clerk#000000180|0|iously final deposits cajole. blithely even packages |
+960|35|F|63537.13|1994-09-21|3-MEDIUM|Clerk#000000120|0|regular accounts. requests|
+961|56|P|158893.16|1995-06-04|4-NOT SPECIFIED|Clerk#000000720|0|ons nag furiously among the quickl|
+962|37|F|98258.73|1994-05-06|5-LOW|Clerk#000000463|0|ments nag deposits. fluffily ironic a|
+963|26|F|53287.25|1994-05-26|3-MEDIUM|Clerk#000000497|0|uses haggle carefully. slyly even dependencies after the packages ha|
+964|76|O|131146.47|1995-05-20|3-MEDIUM|Clerk#000000657|0|print blithely ironic, careful theodolit|
+965|70|P|41758.44|1995-05-15|5-LOW|Clerk#000000218|0|iously special packages. slyly pending requests are carefully |
+966|14|O|120516.93|1998-04-30|2-HIGH|Clerk#000000239|0|special deposits. furious|
+967|110|F|179287.95|1992-06-21|3-MEDIUM|Clerk#000000167|0|excuses engage quickly bold dep|
+992|55|O|133665.12|1997-11-11|3-MEDIUM|Clerk#000000875|0|ts. regular pinto beans thrash carefully sl|
+993|80|O|198238.65|1995-09-10|3-MEDIUM|Clerk#000000894|0|quickly express accounts among the furiously bol|
+994|2|F|41433.48|1994-04-20|5-LOW|Clerk#000000497|0|ole. slyly bold excuses nag caref|
+995|116|P|135157.92|1995-05-31|3-MEDIUM|Clerk#000000439|0|deas. blithely final deposits play. express accounts wake blithely caref|
+996|71|O|47447.63|1997-12-29|1-URGENT|Clerk#000000497|0|arefully final packages into the slyly final requests affix blit|
+997|109|O|27561.82|1997-05-19|2-HIGH|Clerk#000000651|0|ly express depths. furiously final requests haggle furiously. carefu|
+998|32|F|65269.38|1994-11-26|4-NOT SPECIFIED|Clerk#000000956|0|ronic dolphins. ironic, bold ideas haggle furiously furious|
+999|61|F|145249.13|1993-09-05|5-LOW|Clerk#000000464|0|pitaphs sleep. regular accounts use. f|
+1024|4|O|176084.63|1997-12-23|5-LOW|Clerk#000000903|0| blithely. even, express theodolites cajole slyly across|
+1025|103|F|82034.03|1995-05-05|2-HIGH|Clerk#000000376|0|ross the slyly final pa|
+1026|73|O|36464.76|1997-06-04|5-LOW|Clerk#000000223|0|s wake blithely. special acco|
+1027|128|F|112770.89|1992-06-03|3-MEDIUM|Clerk#000000241|0|equests cajole. slyly final pinto bean|
+1028|70|F|153864.67|1994-01-01|2-HIGH|Clerk#000000131|0|ts are. final, silent deposits are among the fl|
+1029|130|F|47440.91|1994-06-21|2-HIGH|Clerk#000000700|0|quests sleep. slyly even foxes wake quickly final theodolites. clo|
+1030|134|F|16346.94|1994-06-15|5-LOW|Clerk#000000422|0|ully ironic accounts sleep carefully. requests are carefully alongside of the |
+1031|4|F|128024.71|1994-09-01|3-MEDIUM|Clerk#000000448|0|s; ironic theodolites along the carefully ex|
+1056|28|F|38446.39|1995-02-11|1-URGENT|Clerk#000000125|0|t, even deposits hang about the slyly special i|
+1057|76|F|108107.42|1992-02-20|1-URGENT|Clerk#000000124|0|cuses dazzle carefully careful, ironic pinto beans. carefully even theod|
+1058|53|F|89359.11|1993-04-26|3-MEDIUM|Clerk#000000373|0|kly pending courts haggle. blithely regular sheaves integrate carefully fi|
+1059|127|F|198360.22|1994-02-27|1-URGENT|Clerk#000000104|0|en accounts. carefully bold packages cajole daringly special depende|
+1060|140|F|121994.04|1993-02-21|3-MEDIUM|Clerk#000000989|0|l platelets sleep quickly slyly special requests. furiously |
+1061|103|O|166947.75|1998-05-15|5-LOW|Clerk#000000576|0|uests sleep at the packages. fur|
+1062|106|O|39805.04|1997-01-15|1-URGENT|Clerk#000000152|0|eposits use blithely |
+1063|37|F|41392.31|1994-04-02|2-HIGH|Clerk#000000024|0|deposits nag quickly regular deposits. quickl|
+1088|148|F|47120.41|1992-05-21|5-LOW|Clerk#000000347|0|counts are blithely. platelets print. carefully |
+1089|49|O|103192.74|1996-05-04|4-NOT SPECIFIED|Clerk#000000226|0|ns haggle ruthlessly. even requests are quickly abov|
+1090|19|O|32929.30|1997-11-15|2-HIGH|Clerk#000000300|0| furiously regular platelets haggle along the slyly unusual foxes! |
+1091|83|O|35795.22|1996-08-27|1-URGENT|Clerk#000000549|0| even pinto beans haggle quickly alongside of the eve|
+1092|124|P|85552.21|1995-03-04|3-MEDIUM|Clerk#000000006|0|re quickly along the blithe|
+1093|101|O|79189.58|1997-07-31|4-NOT SPECIFIED|Clerk#000000159|0| after the carefully ironic requests. carefully ironic packages wake fluffil|
+1094|145|O|9006.25|1997-12-24|3-MEDIUM|Clerk#000000570|0|beans affix furiously about the pending, even deposits. finally pendi|
+1095|145|O|178491.24|1995-08-22|3-MEDIUM|Clerk#000000709|0|sly bold requests cajole carefully according to|
+1120|140|O|107958.62|1997-11-07|3-MEDIUM|Clerk#000000319|0|lly special requests. slyly pending platelets are quickly pending requ|
+1121|29|O|241837.88|1997-01-13|3-MEDIUM|Clerk#000000541|0|r escapades. deposits above the fluffily bold requests hag|
+1122|121|O|179747.47|1997-01-10|1-URGENT|Clerk#000000083|0|uffily carefully final theodolites. furiously express packages affix|
+1123|73|O|93259.93|1996-08-03|3-MEDIUM|Clerk#000000929|0|uriously pending requests. slyly regular instruction|
+1124|80|O|141858.97|1998-07-30|5-LOW|Clerk#000000326|0|regular pinto beans along the fluffily silent packages|
+1125|25|F|80438.38|1994-10-27|2-HIGH|Clerk#000000510|0|ithely final requests. i|
+1126|145|O|59982.31|1998-01-28|4-NOT SPECIFIED|Clerk#000000928|0|d slyly regular ideas: special ideas believe slyly. slyly ironic sheaves w|
+1127|58|O|103320.91|1995-09-19|4-NOT SPECIFIED|Clerk#000000397|0|usly silent, regular pinto beans. blithely express requests boos|
+1152|49|F|51775.54|1994-08-14|4-NOT SPECIFIED|Clerk#000000496|0|equests. deposits ab|
+1153|121|O|220727.97|1996-04-18|5-LOW|Clerk#000000059|0| across the pending deposi|
+1154|37|F|192417.85|1992-02-15|1-URGENT|Clerk#000000268|0|old asymptotes are special requests. blithely even deposits sleep furiously|
+1155|149|O|126902.81|1997-10-06|2-HIGH|Clerk#000000164|0|c deposits haggle among the ironic, even requests. carefully ironic sheaves n|
+1156|133|O|217682.81|1996-10-19|1-URGENT|Clerk#000000200|0| blithely ironic dolphins. furiously pendi|
+1157|97|O|85394.06|1998-01-14|4-NOT SPECIFIED|Clerk#000000207|0|out the regular excuses boost carefully against the furio|
+1158|142|O|31075.51|1996-06-30|2-HIGH|Clerk#000000549|0|integrate slyly furiously ironic deposit|
+1159|70|F|55553.68|1992-09-18|3-MEDIUM|Clerk#000000992|0|ts may sleep. requests according to the|
+1184|89|O|39700.29|1997-10-26|5-LOW|Clerk#000000777|0|iously even packages haggle fluffily care|
+1185|74|F|47033.21|1992-08-24|5-LOW|Clerk#000000344|0| even escapades are. package|
+1186|59|O|82026.18|1996-08-15|4-NOT SPECIFIED|Clerk#000000798|0|ingly regular pinto beans: instructi|
+1187|134|F|85948.02|1992-11-20|3-MEDIUM|Clerk#000000047|0|s after the furiously final deposits boost slyly under the|
+1188|20|O|54655.07|1996-04-11|2-HIGH|Clerk#000000256|0|ully ironic deposits. slyl|
+1189|46|F|71017.99|1994-04-09|1-URGENT|Clerk#000000243|0|f the even accounts. courts print blithely ironic accounts. sile|
+1190|13|O|31043.39|1997-03-16|5-LOW|Clerk#000000575|0|ccounts above the foxes integrate carefully after the |
+1191|112|O|28623.04|1995-11-07|3-MEDIUM|Clerk#000000011|0|uests nag furiously. carefully even requests|
+1216|122|F|68056.57|1992-12-07|5-LOW|Clerk#000000918|0|nal foxes around the e|
+1217|7|F|40982.08|1992-04-26|4-NOT SPECIFIED|Clerk#000000538|0| foxes nag quickly. ironic excuses nod. blithely pending|
+1218|10|F|99834.47|1994-06-20|4-NOT SPECIFIED|Clerk#000000994|0|s cajole. special, silent deposits about the theo|
+1219|28|O|10163.56|1995-10-05|3-MEDIUM|Clerk#000000800|0|od carefully. slyly final dependencies across the even fray|
+1220|49|O|122157.14|1996-08-29|1-URGENT|Clerk#000000712|0|inal theodolites wake. fluffily ironic asymptotes cajol|
+1221|14|F|117397.16|1992-04-19|4-NOT SPECIFIED|Clerk#000000852|0| detect against the silent, even deposits. carefully ironic|
+1222|10|F|47623.94|1993-02-05|3-MEDIUM|Clerk#000000811|0|theodolites use quickly even accounts. carefully final asympto|
+1223|10|O|26714.67|1996-05-25|4-NOT SPECIFIED|Clerk#000000238|0|posits was blithely fr|
+1248|49|F|210713.88|1992-01-02|1-URGENT|Clerk#000000890|0|t the carefully regular dugouts. s|
+1249|149|F|45889.09|1994-01-05|1-URGENT|Clerk#000000095|0|al ideas sleep above the pending pin|
+1250|37|F|12907.62|1992-09-29|4-NOT SPECIFIED|Clerk#000000652|0|ts after the fluffily pending instructions use slyly about the s|
+1251|38|O|109536.55|1997-10-30|1-URGENT|Clerk#000000276|0|, brave sauternes. deposits boost fluffily.|
+1252|149|O|93403.05|1997-08-04|5-LOW|Clerk#000000348|0|ng the slyly regular excuses. special courts nag furiously blithely e|
+1253|115|F|92730.74|1993-01-26|1-URGENT|Clerk#000000775|0| requests sleep furiously even foxes. ruthless packag|
+1254|70|O|94649.25|1995-12-22|1-URGENT|Clerk#000000607|0| pinto beans. carefully regular request|
+1255|122|F|62518.31|1994-05-30|4-NOT SPECIFIED|Clerk#000000798|0|ct slyly regular accounts. quick|
+1280|97|F|91664.85|1993-01-11|5-LOW|Clerk#000000160|0|posits thrash quickly after the theodolites. furiously iro|
+1281|62|F|165454.51|1994-12-11|1-URGENT|Clerk#000000430|0|counts. carefully pending accounts eat |
+1282|116|F|61297.42|1992-02-29|4-NOT SPECIFIED|Clerk#000000168|0|he quickly special packages. furiously final re|
+1283|118|O|202623.92|1996-08-30|4-NOT SPECIFIED|Clerk#000000260|0| pinto beans boost slyly ac|
+1284|134|O|106122.38|1996-01-07|2-HIGH|Clerk#000000492|0|s. blithely silent deposits s|
+1285|11|F|139124.72|1992-06-01|1-URGENT|Clerk#000000423|0|cial deposits cajole after the ironic requests. p|
+1286|109|F|207291.83|1993-05-14|4-NOT SPECIFIED|Clerk#000000939|0| deposits use carefully from the excuses. slyly bold p|
+1287|19|F|131432.42|1994-07-05|2-HIGH|Clerk#000000288|0|ly ironic dolphins integrate furiously among the final packages. st|
+1312|112|F|58111.00|1994-05-19|3-MEDIUM|Clerk#000000538|0|n, express accounts across the ironic|
+1313|148|F|46598.65|1994-09-13|1-URGENT|Clerk#000000774|0|ld accounts. regular deposits cajole. ironically pending theodolites use car|
+1314|143|F|56207.66|1994-05-13|3-MEDIUM|Clerk#000000485|0|ickly blithe packages nod ideas. furiously bold braids boost around the car|
+1315|22|O|121935.23|1998-03-22|5-LOW|Clerk#000000840|0|final theodolites alongside of the carefu|
+1316|16|F|163746.47|1993-12-03|1-URGENT|Clerk#000000857|0|ully bold theodolites? pending, bold pin|
+1317|100|P|139714.71|1995-05-19|2-HIGH|Clerk#000000373|0|sts. furiously special deposits lose fur|
+1318|128|O|81663.65|1998-06-27|3-MEDIUM|Clerk#000000581|0|s hang bold requests. pending, re|
+1319|32|O|31103.83|1996-09-27|2-HIGH|Clerk#000000257|0|y across the ruthlessly ironic accounts. unusu|
+1344|17|F|43809.37|1992-04-16|5-LOW|Clerk#000000178|0|omise close, silent requests. pending theodolites boost pending |
+1345|95|F|111207.93|1992-10-28|5-LOW|Clerk#000000447|0| regular tithes. quickly fluffy de|
+1346|76|F|171975.62|1992-06-18|2-HIGH|Clerk#000000374|0|ges sleep quickly-- even pint|
+1347|41|O|173444.60|1997-06-20|5-LOW|Clerk#000000977|0|he furiously even foxes use carefully express req|
+1348|19|O|94135.77|1998-04-18|5-LOW|Clerk#000000206|0|tly. quickly even deposi|
+1349|64|O|46376.09|1997-10-26|1-URGENT|Clerk#000000543|0|yly! blithely special theodolites cajole. unusual, reg|
+1350|52|F|49305.98|1993-08-24|1-URGENT|Clerk#000000635|0|iously about the blithely special a|
+1351|106|O|24637.96|1998-04-20|1-URGENT|Clerk#000000012|0| cajole. regular, special re|
+1376|47|O|23984.88|1997-05-04|4-NOT SPECIFIED|Clerk#000000730|0|der furiously final, final frets. carefull|
+1377|20|O|108334.30|1998-04-24|4-NOT SPECIFIED|Clerk#000000625|0|lly across the blithely express accounts. ironic excuses promise carefully de|
+1378|20|O|118495.12|1996-03-09|4-NOT SPECIFIED|Clerk#000000705|0| furiously even tithes cajole slyly among the quick|
+1379|65|O|84627.76|1998-05-25|5-LOW|Clerk#000000861|0|y deposits are caref|
+1380|137|O|94969.41|1996-07-07|3-MEDIUM|Clerk#000000969|0|inal deposits wake slyly daringly even requests. bold, even foxe|
+1381|127|O|58212.22|1998-05-25|3-MEDIUM|Clerk#000000107|0|even requests breach after the bold, ironic instructions. slyly even|
+1382|133|F|173522.71|1993-08-17|5-LOW|Clerk#000000241|0|fully final packages sl|
+1383|121|F|34797.72|1993-04-27|2-HIGH|Clerk#000000785|0|ts. express requests sleep blithel|
+1408|55|O|183965.61|1997-12-26|4-NOT SPECIFIED|Clerk#000000942|0|t the quickly final asymptotes. unusual|
+1409|143|F|72440.52|1992-12-31|4-NOT SPECIFIED|Clerk#000000065|0|ructions. furiously unusual excuses are regular, unusual theodolites. fin|
+1410|113|O|114879.19|1997-04-12|5-LOW|Clerk#000000123|0|iously along the bravely regular dolphins. pinto beans cajole furiously sp|
+1411|95|F|164462.61|1994-12-21|2-HIGH|Clerk#000000566|0|s. furiously special excuses across the pending pinto beans haggle sp|
+1412|53|F|78676.54|1993-03-13|4-NOT SPECIFIED|Clerk#000000083|0|uffily daring theodolit|
+1413|91|O|75733.58|1997-06-14|3-MEDIUM|Clerk#000000342|0|, ironic instructions. carefully even packages dazzle|
+1414|77|O|38057.81|1995-08-16|1-URGENT|Clerk#000000883|0|ccounts. ironic foxes haggle car|
+1415|79|F|24654.79|1994-05-29|4-NOT SPECIFIED|Clerk#000000601|0|rays. blithely final ideas affix quickl|
+1440|98|O|50201.16|1995-08-10|5-LOW|Clerk#000000956|0| pending requests. closely s|
+1441|122|O|156477.94|1997-03-06|4-NOT SPECIFIED|Clerk#000000156|0|ter the excuses. ironic dependencies m|
+1442|112|F|7108.12|1994-07-05|4-NOT SPECIFIED|Clerk#000000935|0|nal pinto beans. slyly ironic ideas cajol|
+1443|44|O|44672.03|1996-12-16|5-LOW|Clerk#000000185|0|x blithely against the carefully final somas. even asymptotes are. quickly spe|
+1444|134|F|207907.60|1994-12-06|3-MEDIUM|Clerk#000000783|0|ove the bold accounts cajole fluffily about|
+1445|115|F|154653.32|1995-01-10|3-MEDIUM|Clerk#000000211|0|even packages wake fluffily |
+1446|41|O|27663.16|1998-02-16|5-LOW|Clerk#000000274|0|lly regular notornis above the requests sleep final accounts! |
+1447|91|F|108171.38|1992-10-15|2-HIGH|Clerk#000000880|0|inly against the blithely pending excuses. regular, pe|
+1472|149|O|65331.05|1996-10-06|5-LOW|Clerk#000000303|0|y special dolphins around the final dependencies wake quick|
+1473|94|O|80624.38|1997-03-17|3-MEDIUM|Clerk#000000960|0|furiously close accoun|
+1474|70|F|51697.18|1995-01-09|1-URGENT|Clerk#000000438|0|detect quickly above the carefully even |
+1475|5|O|185496.66|1997-11-12|2-HIGH|Clerk#000000972|0|cally final packages boost. blithely ironic packa|
+1476|145|O|18795.62|1996-06-27|2-HIGH|Clerk#000000673|0|ding accounts hinder alongside of the quickly pending requests. fluf|
+1477|76|O|231831.35|1997-08-24|5-LOW|Clerk#000000612|0|ly bold foxes. final ideas would cajo|
+1478|50|O|20791.50|1997-08-03|2-HIGH|Clerk#000000827|0|lessly. carefully express|
+1479|16|O|31471.04|1995-12-16|4-NOT SPECIFIED|Clerk#000000697|0|he furiously even foxes. thinly bold deposits|
+1504|2|F|89399.40|1992-08-28|3-MEDIUM|Clerk#000000381|0|, brave deposits. bold de|
+1505|37|F|55892.35|1992-08-21|2-HIGH|Clerk#000000544|0|s. slyly ironic packages cajole. carefully regular packages haggle |
+1506|148|F|195844.84|1992-09-21|3-MEDIUM|Clerk#000000620|0| dependencies. accounts affix blithely slowly unusual deposits. slyly regular |
+1507|121|F|96166.92|1993-10-14|3-MEDIUM|Clerk#000000305|0|stealthy, ironic de|
+1508|103|O|151282.65|1998-04-10|5-LOW|Clerk#000000117|0| after the furiously regular pinto beans hang slyly quickly ironi|
+1509|64|F|180455.98|1993-07-08|5-LOW|Clerk#000000770|0|the regular ideas. regul|
+1510|53|O|154590.05|1996-09-17|5-LOW|Clerk#000000128|0|ld carefully. furiously final asymptotes haggle furiously|
+1511|79|O|59651.38|1996-12-22|4-NOT SPECIFIED|Clerk#000000386|0|ts above the depend|
+1536|94|O|5184.26|1997-01-26|3-MEDIUM|Clerk#000000117|0|ges are! furiously final deposits cajole iron|
+1537|109|F|108317.51|1992-02-15|4-NOT SPECIFIED|Clerk#000000862|0|g to the even deposits. ironic, final packages |
+1538|29|O|179554.41|1995-06-18|4-NOT SPECIFIED|Clerk#000000258|0| instructions. regular theod|
+1539|112|F|39612.63|1995-03-10|5-LOW|Clerk#000000840|0|nstructions boost pa|
+1540|16|F|128014.15|1992-08-05|2-HIGH|Clerk#000000927|0|r ideas hinder blithe|
+1541|94|P|47286.32|1995-05-18|1-URGENT|Clerk#000000906|0|y. slyly ironic warhorses around the furiously regul|
+1542|143|F|132972.24|1993-09-15|3-MEDIUM|Clerk#000000435|0|t the furiously close deposits do was f|
+1543|52|O|139047.22|1997-02-20|1-URGENT|Clerk#000000398|0|unts. furiously pend|
+1568|17|O|76119.72|1997-01-30|4-NOT SPECIFIED|Clerk#000000554|0|d notornis. carefully |
+1569|104|O|87803.55|1998-04-02|5-LOW|Clerk#000000786|0|orbits. fluffily even decoys serve blithely. furiously furious realms nag acro|
+1570|124|O|35589.57|1998-03-16|1-URGENT|Clerk#000000745|0|pinto beans haggle furiousl|
+1571|103|F|151404.78|1992-12-05|2-HIGH|Clerk#000000565|0|ously furiously bold warthogs. slyly ironic instructions are quickly a|
+1572|11|O|47232.79|1996-02-24|2-HIGH|Clerk#000000994|0|fluffily ironic accounts haggle blithely final platelets! slyly regular foxes|
+1573|148|F|86918.57|1992-12-28|2-HIGH|Clerk#000000940|0|ess, ironic deposits use along the carefu|
+1574|134|O|179923.54|1996-12-12|3-MEDIUM|Clerk#000000809|0| ideas hinder after the carefully unusual |
+1575|145|O|197031.52|1995-09-13|3-MEDIUM|Clerk#000000497|0|. furiously regular dep|
+1600|94|F|130515.61|1993-03-03|3-MEDIUM|Clerk#000000627|0|tions cajole quietly above the regular, silent requests. slyly fin|
+1601|53|F|73962.95|1994-08-27|5-LOW|Clerk#000000469|0|ent deposits are ca|
+1602|1|F|4225.26|1993-08-05|5-LOW|Clerk#000000660|0|deposits. busily silent instructions haggle furiously. fin|
+1603|2|F|29305.47|1993-07-31|4-NOT SPECIFIED|Clerk#000000869|0|s. slyly silent deposits boo|
+1604|113|F|107139.29|1993-07-17|5-LOW|Clerk#000000512|0|lithely silent waters. blithely unusual packages alongside |
+1605|58|O|130687.64|1998-04-24|4-NOT SPECIFIED|Clerk#000000616|0|sleep furiously? ruthless, even pinto beans |
+1606|53|O|115877.40|1997-04-17|4-NOT SPECIFIED|Clerk#000000550|0|r requests. quickly even platelets breach before the ironically|
+1607|149|O|166335.03|1995-12-16|2-HIGH|Clerk#000000498|0| bold, pending foxes haggle. slyly silent |
+1632|67|O|183286.33|1997-01-08|3-MEDIUM|Clerk#000000351|0|onic requests are accounts. bold a|
+1633|16|O|52359.51|1995-10-14|2-HIGH|Clerk#000000666|0|y silent accounts sl|
+1634|70|O|145898.47|1996-09-10|1-URGENT|Clerk#000000360|0|arefully blithely ironic requests. slyly unusual instructions alongside|
+1635|4|O|70232.26|1997-02-13|3-MEDIUM|Clerk#000000958|0|s. slyly ironic requests affix slyly |
+1636|79|O|172021.87|1997-06-17|3-MEDIUM|Clerk#000000457|0|ding requests. slyly ironic courts wake quickl|
+1637|73|F|180912.15|1995-02-08|4-NOT SPECIFIED|Clerk#000000189|0| final accounts. blithely silent ideas cajole bravely. carefully express |
+1638|139|O|172436.30|1997-08-13|2-HIGH|Clerk#000000643|0|he fluffily regular asymp|
+1639|5|O|104166.56|1995-08-20|4-NOT SPECIFIED|Clerk#000000939|0|haggle furiously. final requests detect furious|
+1664|64|O|178060.22|1996-03-03|1-URGENT|Clerk#000000090|0|y quickly even asymptotes. furiously regular packages haggle quickly fin|
+1665|76|F|4819.91|1994-05-08|2-HIGH|Clerk#000000920|0|ly regular packages are fluffily even ideas. fluffily final|
+1666|95|O|128367.97|1995-10-18|1-URGENT|Clerk#000000849|0|ffily pending dependencies wake fluffily. pending, final accounts |
+1667|5|O|125030.37|1997-10-10|2-HIGH|Clerk#000000103|0|e accounts. slyly express accounts must are a|
+1668|142|O|137576.19|1997-07-12|4-NOT SPECIFIED|Clerk#000000148|0|eodolites. carefully dogged dolphins haggle q|
+1669|2|O|24362.39|1997-06-09|3-MEDIUM|Clerk#000000663|0|er ironic requests detect furiously blithely sp|
+1670|25|O|89999.72|1997-05-24|2-HIGH|Clerk#000000320|0|unusual dependencies. furiously special platelets main|
+1671|35|O|104391.11|1996-07-27|4-NOT SPECIFIED|Clerk#000000275|0|ly. slyly pending requests was above the |
+1696|4|O|102665.03|1998-01-08|4-NOT SPECIFIED|Clerk#000000041|0|bravely bold accounts above the quickly bold|
+1697|76|O|122621.31|1996-10-07|1-URGENT|Clerk#000000815|0|o x-ray blithely. pl|
+1698|40|O|141118.87|1997-04-23|2-HIGH|Clerk#000000432|0|slyly. carefully express deposit|
+1699|85|F|66408.29|1993-12-30|1-URGENT|Clerk#000000125|0|jole blithely. furiously un|
+1700|65|O|89143.36|1996-06-15|3-MEDIUM|Clerk#000000328|0|ely final dolphins wake sometimes above the quietly regular deposits. fur|
+1701|130|F|72835.95|1992-05-19|2-HIGH|Clerk#000000395|0|furiously. regular, close theodoli|
+1702|67|P|194119.31|1995-05-07|2-HIGH|Clerk#000000300|0|around the carefully final deposits cajole carefully according to the b|
+1703|134|F|121220.59|1993-01-28|3-MEDIUM|Clerk#000000463|0| pinto beans poach. bold courts boost. regular, express deposits at|
+1728|64|O|131604.34|1996-05-22|2-HIGH|Clerk#000000711|0|beans. slyly regular instructions sleep! slyly final packages|
+1729|133|F|12137.76|1992-05-19|2-HIGH|Clerk#000000158|0|pending foxes wake. accounts|
+1730|124|O|150886.49|1998-07-24|5-LOW|Clerk#000000794|0| fluffily pending deposits serve. furiously even requests wake furiou|
+1731|128|O|190490.78|1996-01-06|1-URGENT|Clerk#000000268|0|lithely regular, final instructions. ironic, express packages are above|
+1732|146|F|179854.51|1993-11-29|5-LOW|Clerk#000000903|0|inal requests integrate dolph|
+1733|148|O|165489.52|1996-05-12|2-HIGH|Clerk#000000789|0|e carefully according to the accounts. furiously pending instructions sleep|
+1734|7|F|44002.53|1994-06-11|2-HIGH|Clerk#000000722|0| final ideas haggle. blithely quick foxes sleep busily bold ideas. i|
+1735|22|F|98541.95|1992-12-27|1-URGENT|Clerk#000000458|0|ully idle requests wake qu|
+1760|115|O|82151.12|1996-05-17|5-LOW|Clerk#000000917|0| deposits. busily regular deposits wake blithely along the furiously even re|
+1761|106|F|211925.95|1993-12-24|2-HIGH|Clerk#000000817|0|efully slyly bold frets. packages boost b|
+1762|77|F|202227.17|1994-08-20|4-NOT SPECIFIED|Clerk#000000653|0|ly ironic packages. furi|
+1763|121|O|140685.01|1996-10-29|2-HIGH|Clerk#000000321|0|es. bold dependencies haggle furiously along |
+1764|29|F|47384.71|1992-03-25|1-URGENT|Clerk#000000182|0|. slyly final packages integrate carefully acro|
+1765|73|O|36551.43|1995-12-03|4-NOT SPECIFIED|Clerk#000000490|0| regular excuses wake slyly|
+1766|139|O|41032.81|1996-10-12|2-HIGH|Clerk#000000983|0|unusual deposits affix quickly beyond the carefully s|
+1767|25|P|136582.60|1995-03-14|2-HIGH|Clerk#000000327|0|eposits use carefully carefully regular platelets. quickly regular packages al|
+1792|49|F|107919.86|1993-11-09|5-LOW|Clerk#000000102|0|ructions haggle along the pending packages. carefully speci|
+1793|19|F|82504.56|1992-07-12|4-NOT SPECIFIED|Clerk#000000291|0|regular packages cajole. blithely special packages according to the final d|
+1794|140|O|179462.21|1997-09-28|1-URGENT|Clerk#000000686|0|ally silent pinto beans. regular package|
+1795|94|F|146849.33|1994-03-19|2-HIGH|Clerk#000000815|0| quickly final packages! blithely dogged accounts c|
+1796|47|F|33755.47|1992-11-21|2-HIGH|Clerk#000000245|0|eans use furiously around th|
+1797|125|O|51494.47|1996-05-07|3-MEDIUM|Clerk#000000508|0|quiet platelets haggle since the quickly ironic instructi|
+1798|52|O|46393.97|1997-07-28|1-URGENT|Clerk#000000741|0|al foxes are blithe|
+1799|61|F|46815.93|1994-03-07|4-NOT SPECIFIED|Clerk#000000339|0|ns sleep furiously final waters. blithely regular instructions h|
+1824|49|F|81351.53|1994-05-05|1-URGENT|Clerk#000000972|0|e blithely fluffily|
+1825|148|F|150582.77|1993-12-05|3-MEDIUM|Clerk#000000345|0|ironic, final accou|
+1826|82|F|124719.97|1992-04-16|4-NOT SPECIFIED|Clerk#000000718|0|the even asymptotes dazzle fluffily slyly regular asymptotes. final, unu|
+1827|106|O|210113.88|1996-06-22|4-NOT SPECIFIED|Clerk#000000369|0|luffily even requests haggle sly|
+1828|32|F|137369.50|1994-04-18|3-MEDIUM|Clerk#000000840|0|y quickly bold packag|
+1829|112|F|127532.20|1994-05-08|2-HIGH|Clerk#000000537|0| accounts wake above the furiously unusual requests. pending package|
+1830|133|F|85122.24|1995-02-23|1-URGENT|Clerk#000000045|0|according to the even,|
+1831|71|F|58032.77|1993-12-02|1-URGENT|Clerk#000000854|0| accounts. carefully even accounts boost furiously. regular ideas engage. |
+1856|106|F|189361.42|1992-03-20|4-NOT SPECIFIED|Clerk#000000952|0|. special pinto beans run acr|
+1857|133|F|102793.59|1993-01-13|2-HIGH|Clerk#000000083|0|hely final ideas slee|
+1858|143|O|30457.91|1997-12-13|1-URGENT|Clerk#000000389|0|thely. slyly final deposits sleep|
+1859|61|O|105094.09|1997-04-11|4-NOT SPECIFIED|Clerk#000000949|0| the foxes. bravely special excuses nag carefully special r|
+1860|10|O|9103.40|1996-04-04|3-MEDIUM|Clerk#000000556|0|osits. quickly bold deposits according to |
+1861|70|F|95063.41|1994-01-03|3-MEDIUM|Clerk#000000847|0|r the fluffily close sauternes. furio|
+1862|34|O|97981.06|1998-02-24|5-LOW|Clerk#000000348|0|ts snooze ironically abou|
+1863|74|F|96359.65|1993-09-23|4-NOT SPECIFIED|Clerk#000000658|0|old sentiments. careful, |
+1888|121|F|224724.11|1993-10-31|4-NOT SPECIFIED|Clerk#000000659|0|olites. pinto beans cajole. regular deposits affix. slyly regular|
+1889|25|O|96431.77|1997-03-16|1-URGENT|Clerk#000000854|0|p around the regular notornis. unusual deposits|
+1890|10|O|202364.58|1996-12-18|4-NOT SPECIFIED|Clerk#000000627|0|romise final, regular deposits. regular fox|
+1891|61|F|76848.96|1994-12-15|5-LOW|Clerk#000000495|0|unusual foxes sleep regular deposits. requests wake special pac|
+1892|25|F|133273.64|1994-03-26|5-LOW|Clerk#000000733|0|sts. slyly regular dependencies use slyly. ironic, spec|
+1893|125|O|116792.13|1997-10-30|2-HIGH|Clerk#000000111|0|olites. silent, special deposits eat slyly quickly express packages; hockey p|
+1894|76|F|44387.23|1992-03-30|1-URGENT|Clerk#000000626|0|e furiously. furiously even accounts are slyly final accounts. closely speci|
+1895|7|F|44429.81|1994-05-30|3-MEDIUM|Clerk#000000878|0|ress accounts. bold accounts cajole. slyly final pinto beans poach regul|
+1920|110|O|119605.91|1998-06-24|5-LOW|Clerk#000000018|0|hely; furiously regular excuses|
+1921|88|F|57584.12|1994-01-18|3-MEDIUM|Clerk#000000293|0|counts. slyly quiet requests along the ruthlessly regular accounts are |
+1922|56|O|11575.77|1996-07-13|3-MEDIUM|Clerk#000000984|0|side of the blithely final re|
+1923|136|O|171128.10|1997-07-07|1-URGENT|Clerk#000000471|0| express dolphins. |
+1924|76|O|169756.19|1996-09-07|4-NOT SPECIFIED|Clerk#000000823|0| of the ironic accounts. instructions near the final instr|
+1925|17|F|146382.71|1992-03-05|1-URGENT|Clerk#000000986|0|e slyly regular deposits. furiously |
+1926|94|O|100035.03|1996-01-31|2-HIGH|Clerk#000000568|0|cajole. even warhorses sleep carefully. |
+1927|140|O|23327.88|1995-09-30|3-MEDIUM|Clerk#000000616|0|riously special packages. permanent pearls wake furiously. even packages alo|
+1952|67|F|12896.25|1994-03-16|2-HIGH|Clerk#000000254|0| silent accounts boost |
+1953|149|F|57213.18|1993-11-30|3-MEDIUM|Clerk#000000891|0| fluffily along the quickly even packages. |
+1954|56|O|158853.63|1997-05-31|4-NOT SPECIFIED|Clerk#000000104|0| unusual excuses cajole according to the blithely regular theodolites.|
+1955|13|F|103085.13|1992-04-20|1-URGENT|Clerk#000000792|0|ly special ideas. sometimes final |
+1956|127|F|88704.26|1992-09-20|4-NOT SPECIFIED|Clerk#000000600|0|ironic ideas are silent ideas. furiously final deposits sleep slyly carefu|
+1957|31|O|77482.87|1998-07-21|2-HIGH|Clerk#000000639|0|nding excuses about the |
+1958|53|O|176294.34|1995-09-22|5-LOW|Clerk#000000343|0| haggle blithely. flu|
+1959|43|O|62277.18|1997-01-13|4-NOT SPECIFIED|Clerk#000000631|0| cajole about the blithely express requests. even excuses mold bl|
+1984|52|O|79230.47|1998-04-01|1-URGENT|Clerk#000000416|0| slyly special instructions. unusual foxes use packages. carefully regular req|
+1985|7|F|171522.54|1994-09-02|4-NOT SPECIFIED|Clerk#000000741|0|slyly slyly even pains. slyly reg|
+1986|149|F|34269.96|1994-05-05|2-HIGH|Clerk#000000609|0|across the theodolites. quick|
+1987|100|F|6406.29|1994-04-30|2-HIGH|Clerk#000000652|0|gular platelets alongside |
+1988|109|O|117132.72|1995-10-06|4-NOT SPECIFIED|Clerk#000000011|0|ly ironic dolphins serve quickly busy accounts. bu|
+1989|118|F|39263.28|1994-03-16|4-NOT SPECIFIED|Clerk#000000747|0|ely bold pinto beans ha|
+1990|119|F|48781.39|1994-12-16|2-HIGH|Clerk#000000114|0|e bold patterns. always regul|
+1991|19|F|139854.41|1992-09-07|4-NOT SPECIFIED|Clerk#000000854|0|ing accounts can haggle at the carefully final Tiresias-- pending, regular|
+2016|8|O|24347.36|1996-08-16|3-MEDIUM|Clerk#000000641|0|the carefully ironic foxes. requests nag bold, r|
+2017|101|O|70529.27|1998-05-13|3-MEDIUM|Clerk#000000427|0|nusual requests. blit|
+2018|19|P|25007.95|1995-04-05|4-NOT SPECIFIED|Clerk#000000920|0|gular accounts wake fur|
+2019|136|F|43789.14|1992-10-23|1-URGENT|Clerk#000000565|0| furiously bold packages. fluffily fi|
+2020|73|F|136162.13|1993-06-21|3-MEDIUM|Clerk#000000192|0|es. furiously regular packages above the furiously special theodolites are a|
+2021|70|O|27016.74|1995-07-15|1-URGENT|Clerk#000000155|0|ong the furiously regular requests. unusual deposits wake fluffily inside|
+2022|62|F|206742.11|1992-03-15|1-URGENT|Clerk#000000268|0| dependencies sleep fluffily even, ironic deposits. express, silen|
+2023|118|F|144123.37|1992-05-06|5-LOW|Clerk#000000137|0|ular courts engage according to the|
+2048|17|F|33401.77|1993-11-15|1-URGENT|Clerk#000000934|0|s cajole after the blithely final accounts. f|
+2049|31|O|153048.74|1995-12-07|2-HIGH|Clerk#000000859|0|ly regular requests thrash blithely about the fluffily even theodolites. r|
+2050|28|F|208517.98|1994-06-02|4-NOT SPECIFIED|Clerk#000000821|0|d accounts against the furiously regular packages use bli|
+2051|40|O|87988.34|1996-03-18|4-NOT SPECIFIED|Clerk#000000333|0|ctions sleep blithely. blithely regu|
+2052|91|F|141822.19|1992-04-13|2-HIGH|Clerk#000000767|0| requests sleep around the even, even courts. ironic theodolites affix furious|
+2053|142|F|125125.57|1995-02-07|1-URGENT|Clerk#000000717|0|ar requests: blithely sly accounts boost carefully across t|
+2054|41|F|144335.16|1992-06-08|4-NOT SPECIFIED|Clerk#000000103|0|l requests affix carefully about the furiously special|
+2055|97|F|57092.26|1993-09-04|1-URGENT|Clerk#000000067|0|. warhorses affix slyly blithely express instructions? fur|
+2080|95|F|45767.69|1993-06-18|5-LOW|Clerk#000000190|0|ironic, pending theodolites are carefully about the quickly regular theodolite|
+2081|121|O|145654.97|1997-07-05|2-HIGH|Clerk#000000136|0|ong the regular theo|
+2082|49|F|46753.63|1995-01-10|2-HIGH|Clerk#000000354|0|cial accounts. ironic, express dolphins nod slyly sometimes final reques|
+2083|101|F|31795.52|1993-07-14|3-MEDIUM|Clerk#000000361|0|al patterns. bold, final foxes nag bravely about the furiously express|
+2084|80|F|190652.53|1993-03-17|2-HIGH|Clerk#000000048|0|zle furiously final, careful packages. slyly ironic ideas amo|
+2085|49|F|45311.07|1993-11-21|3-MEDIUM|Clerk#000000818|0|ress, express ideas haggle|
+2086|142|F|188985.18|1994-10-19|1-URGENT|Clerk#000000046|0| permanently regular|
+2087|50|O|53581.41|1998-01-31|2-HIGH|Clerk#000000626|0|e always regular packages nod against the furiously spec|
+2112|64|O|17986.15|1997-02-05|2-HIGH|Clerk#000000351|0|against the slyly even id|
+2113|32|O|65678.21|1997-11-08|2-HIGH|Clerk#000000527|0|slyly regular instruct|
+2114|79|F|106446.02|1995-01-16|5-LOW|Clerk#000000751|0|r, unusual accounts haggle across the busy platelets. carefully |
+2115|106|O|134814.65|1998-05-23|4-NOT SPECIFIED|Clerk#000000101|0|odolites boost. carefully regular excuses cajole. quickly ironic pinto be|
+2116|23|F|60887.90|1994-08-26|1-URGENT|Clerk#000000197|0|efully after the asymptotes. furiously sp|
+2117|22|O|145713.03|1997-04-26|2-HIGH|Clerk#000000887|0|ely even dependencies. regular foxes use blithely.|
+2118|134|O|38974.67|1996-10-09|1-URGENT|Clerk#000000196|0|ial requests wake carefully special packages. f|
+2119|64|O|34632.57|1996-08-20|2-HIGH|Clerk#000000434|0|uickly pending escapades. fluffily ir|
+2144|136|F|119917.28|1994-03-29|3-MEDIUM|Clerk#000000546|0|t. carefully quick requests across the deposits wake regu|
+2145|134|F|18885.35|1992-10-03|1-URGENT|Clerk#000000886|0|sts would snooze blithely alongside of th|
+2146|118|F|179686.07|1992-09-14|4-NOT SPECIFIED|Clerk#000000476|0|ven packages. dependencies wake slyl|
+2147|100|F|91513.79|1992-09-06|4-NOT SPECIFIED|Clerk#000000424|0| haggle carefully furiously final foxes. pending escapades thrash. bold theod|
+2148|130|F|19612.03|1995-04-19|4-NOT SPECIFIED|Clerk#000000517|0|ross the furiously unusual theodolites. always expre|
+2149|101|F|105145.40|1993-03-13|5-LOW|Clerk#000000555|0|nusual accounts nag furiously special reques|
+2150|82|F|166961.06|1994-06-03|3-MEDIUM|Clerk#000000154|0|ect slyly against the even, final packages. quickly regular pinto beans wake c|
+2151|58|O|124608.69|1996-11-11|3-MEDIUM|Clerk#000000996|0|c requests. ironic platelets cajole across the quickly fluffy deposits.|
+2176|104|F|87248.17|1992-11-10|1-URGENT|Clerk#000000195|0|s haggle regularly accor|
+2177|136|O|183493.42|1997-01-20|3-MEDIUM|Clerk#000000161|0|ove the blithely unusual packages cajole carefully fluffily special request|
+2178|8|O|79594.68|1996-12-12|3-MEDIUM|Clerk#000000656|0|thely according to the instructions. furious|
+2179|41|O|77487.09|1996-09-07|2-HIGH|Clerk#000000935|0|ounts alongside of the furiously unusual braids cajol|
+2180|76|O|208481.57|1996-09-14|4-NOT SPECIFIED|Clerk#000000650|0|xpress, unusual pains. furiously ironic excu|
+2181|76|O|100954.64|1995-09-13|3-MEDIUM|Clerk#000000814|0|y against the ironic, even|
+2182|23|F|116003.11|1994-04-05|2-HIGH|Clerk#000000071|0|ccounts. quickly bold deposits across the excuses sl|
+2183|113|O|49841.12|1996-06-22|1-URGENT|Clerk#000000287|0| among the express, ironic packages. slyly ironic platelets integrat|
+2208|68|P|245388.06|1995-05-01|4-NOT SPECIFIED|Clerk#000000900|0|symptotes wake slyly blithely unusual packages.|
+2209|91|F|129086.93|1992-07-10|2-HIGH|Clerk#000000056|0|er above the slyly silent requests. furiously reg|
+2210|32|F|31689.46|1992-01-16|2-HIGH|Clerk#000000941|0| believe carefully quickly express pinto beans. deposi|
+2211|92|F|140031.23|1994-06-30|2-HIGH|Clerk#000000464|0|ffily bold courts e|
+2212|118|F|17231.05|1994-03-23|3-MEDIUM|Clerk#000000954|0|structions above the unusual requests use fur|
+2213|122|F|146136.10|1993-01-15|4-NOT SPECIFIED|Clerk#000000598|0|osits are carefully reg|
+2214|115|O|150345.63|1998-05-05|3-MEDIUM|Clerk#000000253|0|packages. fluffily even accounts haggle blithely. carefully ironic depen|
+2215|40|O|108239.46|1996-06-16|4-NOT SPECIFIED|Clerk#000000817|0|le final, final foxes. quickly regular gifts are carefully deposit|
+2240|56|F|174090.30|1992-03-06|4-NOT SPECIFIED|Clerk#000000622|0|accounts against the slyly express foxes are after the slyly regular |
+2241|103|F|165219.08|1993-05-11|1-URGENT|Clerk#000000081|0|y about the silent excuses. furiously ironic instructions along the sil|
+2242|82|O|15082.82|1997-07-20|4-NOT SPECIFIED|Clerk#000000360|0| pending multipliers. carefully express asymptotes use quickl|
+2243|49|O|10451.97|1995-06-10|2-HIGH|Clerk#000000813|0|ously regular deposits integrate s|
+2244|127|F|21207.08|1993-01-09|1-URGENT|Clerk#000001000|0|ckages. ironic, ironic accounts haggle blithely express excuses. |
+2245|58|F|150585.73|1993-04-28|3-MEDIUM|Clerk#000000528|0|ake carefully. braids haggle slyly quickly b|
+2246|113|O|85755.84|1996-05-27|4-NOT SPECIFIED|Clerk#000000739|0| final gifts sleep |
+2247|95|F|13491.31|1992-08-02|4-NOT SPECIFIED|Clerk#000000947|0|furiously regular packages. final brai|
+2272|139|F|127934.71|1993-04-13|2-HIGH|Clerk#000000449|0|s. bold, ironic pinto beans wake. silently specia|
+2273|136|O|142291.79|1996-12-14|5-LOW|Clerk#000000155|0|uickly express foxes haggle quickly against|
+2274|104|F|58273.89|1993-09-04|4-NOT SPECIFIED|Clerk#000000258|0|nstructions try to hag|
+2275|149|F|37398.90|1992-10-22|4-NOT SPECIFIED|Clerk#000000206|0| furiously furious platelets. slyly final packa|
+2276|43|O|141159.63|1996-04-29|4-NOT SPECIFIED|Clerk#000000821|0|ecial requests. fox|
+2277|89|F|79270.23|1995-01-02|4-NOT SPECIFIED|Clerk#000000385|0|accounts cajole. even i|
+2278|142|O|101878.46|1998-04-25|3-MEDIUM|Clerk#000000186|0|r pinto beans integrate after the carefully even deposits. blit|
+2279|80|F|142322.33|1993-02-23|3-MEDIUM|Clerk#000000898|0|de of the quickly unusual instructio|
+2304|46|F|93769.28|1994-01-07|4-NOT SPECIFIED|Clerk#000000415|0|onic platelets. ironic packages haggle. packages nag doggedly according to|
+2305|43|F|122964.66|1993-01-26|2-HIGH|Clerk#000000440|0|ove the furiously even acco|
+2306|28|O|244704.23|1995-07-26|2-HIGH|Clerk#000000975|0| wake furiously requests. permanent requests affix. final packages caj|
+2307|106|F|59417.76|1993-06-29|5-LOW|Clerk#000000952|0|furiously even asymptotes? carefully regular accounts|
+2308|25|F|58546.02|1992-10-25|4-NOT SPECIFIED|Clerk#000000609|0|ts. slyly final depo|
+2309|100|O|146933.07|1995-09-04|5-LOW|Clerk#000000803|0|he carefully pending packages. fluffily stealthy foxes engage carefully|
+2310|31|O|82928.12|1996-09-20|5-LOW|Clerk#000000917|0|wake carefully. unusual instructions nag ironic, regular excuse|
+2311|73|P|153233.93|1995-05-02|2-HIGH|Clerk#000000761|0|ly pending asymptotes-- furiously bold excus|
+2336|142|O|22294.51|1996-01-07|4-NOT SPECIFIED|Clerk#000000902|0|c, final excuses sleep furiously among the even theodolites. f|
+2337|142|O|45704.96|1997-06-18|4-NOT SPECIFIED|Clerk#000000754|0| quickly. final accounts haggle. carefully final acco|
+2338|140|O|28155.92|1997-09-15|2-HIGH|Clerk#000000951|0|riously final dugouts. final, ironic packages wake express, ironic id|
+2339|109|F|63470.78|1993-12-15|5-LOW|Clerk#000000847|0| against the regular |
+2340|65|O|30778.78|1996-01-12|1-URGENT|Clerk#000000964|0|ter the deposits sleep according to the slyly regular packages. carefully |
+2341|82|F|55950.21|1993-05-30|5-LOW|Clerk#000000443|0|sts-- blithely bold dolphins through the deposits nag blithely carefully re|
+2342|37|O|104038.78|1996-06-09|1-URGENT|Clerk#000000615|0|oost carefully across the regular accounts. blithely final d|
+2343|73|O|85381.00|1995-08-21|3-MEDIUM|Clerk#000000170|0|fluffily over the slyly special deposits. quickl|
+2368|13|F|101240.96|1993-08-20|1-URGENT|Clerk#000000830|0|t the bold instructions. carefully unusual |
+2369|110|O|73517.91|1996-12-24|2-HIGH|Clerk#000000752|0|iously even requests are dogged, express |
+2370|142|F|73924.21|1994-01-17|1-URGENT|Clerk#000000231|0|lyly final packages. quickly final deposits haggl|
+2371|19|O|193857.67|1998-01-07|1-URGENT|Clerk#000000028|0|ckages haggle at th|
+2372|31|O|104927.66|1997-11-21|5-LOW|Clerk#000000342|0|s: deposits haggle along the final ideas. careful|
+2373|28|F|55211.04|1994-03-12|4-NOT SPECIFIED|Clerk#000000306|0| even, special courts grow quickly. pending,|
+2374|4|F|115219.88|1993-10-29|4-NOT SPECIFIED|Clerk#000000081|0| blithely regular packages. blithely unusua|
+2375|5|O|106612.48|1996-11-20|3-MEDIUM|Clerk#000000197|0|unusual, pending theodolites cajole carefully |
+2400|37|O|92798.66|1998-07-25|5-LOW|Clerk#000000782|0|nusual courts nag against the carefully unusual pinto b|
+2401|148|O|88448.24|1997-07-29|4-NOT SPECIFIED|Clerk#000000531|0|ully unusual instructions boost carefully silently regular requests. |
+2402|67|O|70403.62|1996-09-06|4-NOT SPECIFIED|Clerk#000000162|0|slyly final sheaves sleep slyly. q|
+2403|55|O|111020.79|1998-04-11|3-MEDIUM|Clerk#000000820|0|furiously regular deposits use. furiously unusual accounts wake along the |
+2404|77|O|109077.69|1997-03-13|4-NOT SPECIFIED|Clerk#000000409|0|deposits breach furiously. ironic foxes haggle carefully bold packag|
+2405|73|O|115929.14|1996-12-23|3-MEDIUM|Clerk#000000535|0|ular, regular asympto|
+2406|7|O|182516.77|1996-10-28|5-LOW|Clerk#000000561|0|blithely regular accounts u|
+2407|55|O|112843.52|1998-06-19|2-HIGH|Clerk#000000068|0|uests affix slyly among the slyly regular depos|
+2432|103|O|62661.93|1996-07-13|1-URGENT|Clerk#000000115|0|re. slyly even deposits wake bra|
+2433|31|F|147071.86|1994-08-22|4-NOT SPECIFIED|Clerk#000000324|0|ess patterns are slyly. packages haggle carefu|
+2434|25|O|123956.25|1997-04-27|3-MEDIUM|Clerk#000000190|0|s. quickly ironic dolphins impress final deposits. blithel|
+2435|73|F|122490.66|1993-02-21|5-LOW|Clerk#000000112|0|es are carefully along the carefully final instructions. pe|
+2436|125|O|73990.08|1995-09-11|4-NOT SPECIFIED|Clerk#000000549|0|arefully. blithely bold deposits affix special accounts. final foxes nag. spe|
+2437|85|F|143411.69|1993-04-21|4-NOT SPECIFIED|Clerk#000000578|0|. theodolites wake slyly-- ironic, pending platelets above the carefully exp|
+2438|13|F|214494.39|1993-07-15|2-HIGH|Clerk#000000744|0|the final, regular warhorses. regularly |
+2439|55|O|41811.12|1997-03-15|2-HIGH|Clerk#000000819|0|lithely after the car|
+2464|145|O|30495.65|1997-11-23|5-LOW|Clerk#000000633|0|le about the instructions. courts wake carefully even|
+2465|34|O|180737.75|1995-06-24|1-URGENT|Clerk#000000078|0|al pinto beans. final, bold packages wake quickly|
+2466|19|F|161625.50|1994-03-06|1-URGENT|Clerk#000000424|0|c pinto beans. express deposits wake quickly. even, final courts nag. package|
+2467|35|O|7231.91|1995-07-16|4-NOT SPECIFIED|Clerk#000000914|0|pades sleep furiously. sometimes regular packages again|
+2468|112|O|160627.01|1997-06-09|4-NOT SPECIFIED|Clerk#000000260|0|ickly regular packages. slyly ruthless requests snooze quickly blithe|
+2469|124|O|192074.23|1996-11-26|5-LOW|Clerk#000000730|0| sleep closely regular instructions. furiously ironic instructi|
+2470|58|O|104966.33|1997-04-19|3-MEDIUM|Clerk#000000452|0|to the furiously final packages? pa|
+2471|89|O|34936.31|1998-03-12|4-NOT SPECIFIED|Clerk#000000860|0|carefully blithely regular pac|
+2496|136|F|140390.60|1994-01-09|2-HIGH|Clerk#000000142|0|slyly. pending instructions sleep. quic|
+2497|47|F|171326.48|1992-08-27|1-URGENT|Clerk#000000977|0|ily ironic pinto beans. furiously final platelets alongside of t|
+2498|97|F|45514.27|1993-11-08|5-LOW|Clerk#000000373|0|g the slyly special pinto beans. |
+2499|121|O|147243.86|1995-09-24|1-URGENT|Clerk#000000277|0|r the quickly bold foxes. bold instructi|
+2500|133|F|131122.82|1992-08-15|2-HIGH|Clerk#000000447|0|integrate slyly pending deposits. furiously ironic accounts across the s|
+2501|67|O|79380.51|1997-05-25|5-LOW|Clerk#000000144|0|ickly special theodolite|
+2502|70|F|33470.40|1993-05-28|4-NOT SPECIFIED|Clerk#000000914|0|lyly: carefully pending ideas affix again|
+2503|7|F|183671.08|1993-06-20|3-MEDIUM|Clerk#000000294|0|ly even packages was. ironic, regular deposits unwind furiously across the p|
+2528|55|F|92069.62|1994-11-20|1-URGENT|Clerk#000000789|0|ular dependencies? regular frays kindle according to the blith|
+2529|136|O|4104.30|1996-08-20|2-HIGH|Clerk#000000511|0|posits across the silent instructions wake blithely across |
+2530|128|F|58853.11|1994-03-21|3-MEDIUM|Clerk#000000291|0|ular instructions about the quic|
+2531|44|O|143212.85|1996-05-06|4-NOT SPECIFIED|Clerk#000000095|0|even accounts. furiously ironic excuses sleep fluffily. carefully silen|
+2532|94|O|116093.49|1995-10-11|2-HIGH|Clerk#000000498|0|the blithely pending accounts. regular, regular excuses boost aro|
+2533|50|O|168495.03|1997-03-24|1-URGENT|Clerk#000000594|0|ecial instructions. spec|
+2534|76|O|202784.54|1996-07-17|3-MEDIUM|Clerk#000000332|0|packages cajole ironic requests. furiously regular|
+2535|121|F|67018.30|1993-05-25|5-LOW|Clerk#000000296|0|phins cajole beneath the fluffily express asymptotes. c|
+2560|131|F|153426.79|1992-09-05|1-URGENT|Clerk#000000538|0|atelets; quickly sly requests|
+2561|58|O|137473.58|1997-11-14|1-URGENT|Clerk#000000861|0|ual requests. unusual deposits cajole furiously pending, regular platelets. |
+2562|10|F|136360.37|1992-08-01|1-URGENT|Clerk#000000467|0|elets. pending dolphins promise slyly. bo|
+2563|62|F|168952.10|1993-11-19|4-NOT SPECIFIED|Clerk#000000150|0|sly even packages after the furio|
+2564|77|F|3967.47|1994-09-09|2-HIGH|Clerk#000000718|0|usly regular pinto beans. orbits wake carefully. slyly e|
+2565|56|O|204438.57|1998-02-28|3-MEDIUM|Clerk#000000032|0|x-ray blithely along|
+2566|86|F|89992.48|1992-10-10|3-MEDIUM|Clerk#000000414|0|ructions boost bold ideas. idly ironic accounts use according to th|
+2567|70|O|263411.29|1998-02-27|2-HIGH|Clerk#000000031|0|detect. furiously ironic requests|
+2592|101|F|8225.96|1993-03-05|4-NOT SPECIFIED|Clerk#000000524|0|ts nag fluffily. quickly stealthy theodolite|
+2593|92|F|134726.09|1993-09-04|2-HIGH|Clerk#000000468|0|r the carefully final|
+2594|79|F|94866.39|1992-12-17|1-URGENT|Clerk#000000550|0|ests. theodolites above the blithely even accounts detect furio|
+2595|74|O|173130.20|1995-12-14|4-NOT SPECIFIED|Clerk#000000222|0|arefully ironic requests nag carefully ideas. |
+2596|43|O|74940.13|1996-08-17|1-URGENT|Clerk#000000242|0|requests. ironic, bold theodolites wak|
+2597|104|F|21964.66|1993-02-04|2-HIGH|Clerk#000000757|0|iously ruthless exc|
+2598|112|O|84871.50|1996-03-05|3-MEDIUM|Clerk#000000391|0| ironic notornis according to the blithely final requests should |
+2599|149|O|62807.13|1996-11-07|2-HIGH|Clerk#000000722|0|ts. slyly regular theodolites wake sil|
+2624|52|O|27148.63|1996-11-28|5-LOW|Clerk#000000930|0|ic, regular packages|
+2625|40|F|39382.74|1992-10-14|4-NOT SPECIFIED|Clerk#000000386|0| final deposits. blithely ironic ideas |
+2626|139|O|84314.51|1995-09-08|4-NOT SPECIFIED|Clerk#000000289|0|gside of the carefully special packages are furiously after the slyly express |
+2627|149|F|26798.65|1992-03-24|3-MEDIUM|Clerk#000000181|0|s. silent, ruthless requests|
+2628|56|F|165655.99|1993-10-22|5-LOW|Clerk#000000836|0|ajole across the blithely careful accounts. blithely silent deposits sl|
+2629|139|O|96458.03|1998-04-06|5-LOW|Clerk#000000680|0|uches dazzle carefully even, express excuses. ac|
+2630|85|F|127132.51|1992-10-24|5-LOW|Clerk#000000712|0|inal theodolites. ironic instructions s|
+2631|37|F|63103.32|1993-09-24|5-LOW|Clerk#000000833|0| quickly unusual deposits doubt around |
+2656|77|F|105492.37|1993-05-04|1-URGENT|Clerk#000000307|0|elets. slyly final accou|
+2657|25|O|148176.06|1995-10-17|2-HIGH|Clerk#000000160|0| foxes-- slyly final dependencies around the slyly final theodo|
+2658|14|O|163834.46|1995-09-23|3-MEDIUM|Clerk#000000400|0|bout the slyly regular accounts. ironic, |
+2659|83|F|79785.52|1993-12-18|4-NOT SPECIFIED|Clerk#000000758|0|cross the pending requests maintain |
+2660|127|O|16922.51|1995-08-05|5-LOW|Clerk#000000480|0|ly finally regular deposits. ironic theodolites cajole|
+2661|74|O|106036.84|1997-01-04|3-MEDIUM|Clerk#000000217|0|al, regular pinto beans. silently final deposits should have t|
+2662|37|O|87689.88|1996-08-21|3-MEDIUM|Clerk#000000589|0|bold pinto beans above the slyly final accounts affix furiously deposits. pac|
+2663|95|O|35131.80|1995-09-06|1-URGENT|Clerk#000000950|0|ar requests. furiously final dolphins along the fluffily spe|
+2688|98|F|181077.36|1992-01-24|2-HIGH|Clerk#000000720|0|have to nag according to the pending theodolites. sly|
+2689|103|F|41552.78|1992-04-09|4-NOT SPECIFIED|Clerk#000000698|0|press pains wake. furiously express theodolites alongsid|
+2690|94|O|224674.27|1996-03-31|3-MEDIUM|Clerk#000000760|0|ravely even theodolites |
+2691|7|F|30137.17|1992-04-30|5-LOW|Clerk#000000439|0|es at the regular deposits sleep slyly by the fluffy requests. eve|
+2692|62|O|24265.24|1997-12-02|3-MEDIUM|Clerk#000000878|0|es. regular asymptotes cajole above t|
+2693|19|O|66158.13|1996-09-04|1-URGENT|Clerk#000000370|0|ndle never. blithely regular packages nag carefully enticing platelets. ca|
+2694|121|O|102807.59|1996-03-14|5-LOW|Clerk#000000722|0| requests. bold deposits above the theodol|
+2695|58|O|138584.20|1996-08-20|1-URGENT|Clerk#000000697|0|ven deposits around the quickly regular packa|
+2720|31|F|161307.05|1993-06-08|1-URGENT|Clerk#000000948|0|quickly. special asymptotes are fluffily ironi|
+2721|79|O|59180.25|1996-01-27|2-HIGH|Clerk#000000401|0| ideas eat even, unusual ideas. theodolites are carefully|
+2722|35|F|50328.84|1994-04-09|5-LOW|Clerk#000000638|0|rding to the carefully quick deposits. bli|
+2723|61|O|104759.25|1995-10-06|5-LOW|Clerk#000000836|0|nts must have to cajo|
+2724|137|F|116069.66|1994-09-14|2-HIGH|Clerk#000000217|0| sleep blithely. blithely idle |
+2725|89|F|75144.68|1994-05-21|4-NOT SPECIFIED|Clerk#000000835|0|ular deposits. spec|
+2726|7|F|47753.00|1992-11-27|5-LOW|Clerk#000000470|0| blithely even dinos sleep care|
+2727|74|O|3089.42|1998-04-19|4-NOT SPECIFIED|Clerk#000000879|0|sual theodolites cajole enticingly above the furiously fin|
+2752|59|F|187932.30|1993-11-19|2-HIGH|Clerk#000000648|0| carefully regular foxes are quickly quickl|
+2753|16|F|159720.39|1993-11-30|2-HIGH|Clerk#000000380|0|ending instructions. unusual deposits|
+2754|145|F|25985.52|1994-04-03|2-HIGH|Clerk#000000960|0|cies detect slyly. |
+2755|118|F|101202.18|1992-02-07|4-NOT SPECIFIED|Clerk#000000177|0|ously according to the sly foxes. blithely regular pinto bean|
+2756|118|F|142323.38|1994-04-18|1-URGENT|Clerk#000000537|0|arefully special warho|
+2757|76|O|89792.48|1995-07-20|2-HIGH|Clerk#000000216|0| regular requests subl|
+2758|43|O|36671.88|1998-07-12|5-LOW|Clerk#000000863|0|s cajole according to the carefully special |
+2759|116|F|89731.10|1993-11-25|4-NOT SPECIFIED|Clerk#000000071|0|ts. regular, pending pinto beans sleep ab|
+2784|95|O|106635.21|1998-01-07|1-URGENT|Clerk#000000540|0|g deposits alongside of the silent requests s|
+2785|148|O|132854.79|1995-07-21|2-HIGH|Clerk#000000098|0|iously pending packages sleep according to the blithely unusual foxe|
+2786|79|F|178254.66|1992-03-22|2-HIGH|Clerk#000000976|0|al platelets cajole blithely ironic requests. ironic re|
+2787|103|O|3726.14|1995-09-30|1-URGENT|Clerk#000000906|0|he ironic, regular |
+2788|124|F|17172.66|1994-09-22|1-URGENT|Clerk#000000641|0|nts wake across the fluffily bold accoun|
+2789|37|O|219123.27|1998-03-14|2-HIGH|Clerk#000000972|0|gular patterns boost. carefully even re|
+2790|25|F|177458.97|1994-08-19|2-HIGH|Clerk#000000679|0| the carefully express deposits sleep slyly |
+2791|121|F|156697.55|1994-10-10|2-HIGH|Clerk#000000662|0|as. slyly ironic accounts play furiously bl|
+2816|58|F|42225.53|1994-09-20|2-HIGH|Clerk#000000289|0|kages at the final deposits cajole furious foxes. quickly |
+2817|40|F|71453.85|1994-04-19|3-MEDIUM|Clerk#000000982|0|ic foxes haggle upon the daringly even pinto beans. slyly|
+2818|49|F|120086.84|1994-12-12|3-MEDIUM|Clerk#000000413|0|eep furiously special ideas. express |
+2819|103|F|66927.16|1994-05-05|1-URGENT|Clerk#000000769|0|ngside of the blithely ironic dolphins. furio|
+2820|19|F|143813.39|1994-05-20|3-MEDIUM|Clerk#000000807|0|equests are furiously. carefu|
+2821|118|F|36592.48|1993-08-09|3-MEDIUM|Clerk#000000323|0|ng requests. even instructions are quickly express, silent instructi|
+2822|79|F|40142.15|1993-07-26|2-HIGH|Clerk#000000510|0|furiously against the accounts. unusual accounts aft|
+2823|79|O|171894.45|1995-09-09|2-HIGH|Clerk#000000567|0|encies. carefully fluffy accounts m|
+2848|70|F|116258.53|1992-03-10|1-URGENT|Clerk#000000256|0|ly fluffy foxes sleep furiously across the slyly regu|
+2849|46|O|180054.29|1996-04-30|2-HIGH|Clerk#000000659|0|al packages are after the quickly bold requests. carefully special |
+2850|100|O|122969.79|1996-10-02|2-HIGH|Clerk#000000392|0|, regular deposits. furiously pending packages hinder carefully carefully u|
+2851|145|O|7859.36|1997-09-07|5-LOW|Clerk#000000566|0|Tiresias wake quickly quickly even|
+2852|91|F|99050.81|1993-01-16|1-URGENT|Clerk#000000740|0|ruthless deposits against the final instructions use quickly al|
+2853|94|F|103641.15|1994-05-05|2-HIGH|Clerk#000000878|0|the carefully even packages.|
+2854|139|F|153568.02|1994-06-27|1-URGENT|Clerk#000000010|0| furiously ironic tithes use furiously |
+2855|49|F|48419.58|1993-04-04|4-NOT SPECIFIED|Clerk#000000973|0| silent, regular packages sleep |
+2880|8|F|145761.99|1992-03-15|2-HIGH|Clerk#000000756|0|ves maintain doggedly spec|
+2881|100|F|45695.84|1992-05-10|5-LOW|Clerk#000000864|0|uriously. slyly express requests according to the silent dol|
+2882|121|O|172872.37|1995-08-22|2-HIGH|Clerk#000000891|0|pending deposits. carefully eve|
+2883|121|F|170360.27|1995-01-23|5-LOW|Clerk#000000180|0|uses. carefully ironic accounts lose fluffil|
+2884|92|O|71683.84|1997-10-12|3-MEDIUM|Clerk#000000780|0|efully express instructions sleep against|
+2885|7|F|146896.72|1992-09-19|4-NOT SPECIFIED|Clerk#000000280|0|ly sometimes special excuses. final requests are |
+2886|109|F|94527.23|1994-11-13|4-NOT SPECIFIED|Clerk#000000619|0|uctions. ironic packages sle|
+2887|109|O|28571.39|1997-05-26|5-LOW|Clerk#000000566|0|slyly even pinto beans. slyly bold epitaphs cajole blithely above t|
+2912|94|F|27727.52|1992-03-12|5-LOW|Clerk#000000186|0|jole blithely above the quickly regular packages. carefully regular pinto bean|
+2913|43|O|130702.19|1997-07-12|3-MEDIUM|Clerk#000000118|0|mptotes doubt furiously slyly regu|
+2914|109|F|60867.14|1993-03-03|3-MEDIUM|Clerk#000000543|0|he slyly regular theodolites are furiously sile|
+2915|94|F|96015.13|1994-03-31|5-LOW|Clerk#000000410|0|ld packages. bold deposits boost blithely. ironic, unusual theodoli|
+2916|8|O|20182.22|1995-12-27|2-HIGH|Clerk#000000681|0|ithely blithe deposits sleep beyond the|
+2917|91|O|100714.13|1997-12-09|4-NOT SPECIFIED|Clerk#000000061|0| special dugouts among the special deposi|
+2918|118|O|21760.09|1996-09-08|3-MEDIUM|Clerk#000000439|0|ular deposits across th|
+2919|53|F|137223.14|1993-12-10|2-HIGH|Clerk#000000209|0|es. pearls wake quietly slyly ironic instructions--|
+2944|14|O|146581.14|1997-09-24|4-NOT SPECIFIED|Clerk#000000740|0|deas. permanently special foxes haggle carefully ab|
+2945|29|O|223507.72|1996-01-03|2-HIGH|Clerk#000000499|0|ons are carefully toward the permanent, bold pinto beans. regu|
+2946|125|O|102226.59|1996-02-05|5-LOW|Clerk#000000329|0|g instructions about the regular accounts sleep carefully along the pen|
+2947|70|P|43360.95|1995-04-26|1-URGENT|Clerk#000000464|0|ronic accounts. accounts run furiously d|
+2948|44|F|100758.71|1994-08-23|5-LOW|Clerk#000000701|0| deposits according to the blithely pending |
+2949|137|F|94231.71|1994-04-12|2-HIGH|Clerk#000000184|0|y ironic accounts use. quickly blithe accou|
+2950|136|O|183620.33|1997-07-06|1-URGENT|Clerk#000000833|0| dolphins around the furiously |
+2951|74|O|125509.17|1996-02-06|2-HIGH|Clerk#000000680|0|gular deposits above the finally regular ideas integrate idly stealthil|
+2976|29|F|145768.47|1993-12-10|4-NOT SPECIFIED|Clerk#000000159|0|. furiously ironic asymptotes haggle ruthlessly silently regular r|
+2977|73|O|25170.88|1996-08-27|3-MEDIUM|Clerk#000000252|0|quickly special platelets are furio|
+2978|44|P|139542.14|1995-05-03|1-URGENT|Clerk#000000135|0|d. even platelets are. ironic dependencies cajole slow, e|
+2979|133|O|116789.98|1996-03-23|3-MEDIUM|Clerk#000000820|0|even, ironic foxes sleep along|
+2980|4|O|187514.11|1996-09-14|3-MEDIUM|Clerk#000000661|0|y quick pinto beans wake. slyly re|
+2981|49|O|37776.79|1998-07-29|5-LOW|Clerk#000000299|0|hely among the express foxes. blithely stealthy requests cajole boldly. regu|
+2982|85|F|55582.94|1995-03-19|2-HIGH|Clerk#000000402|0|lyly. express theodolites affix slyly after the slyly speci|
+2983|62|F|58168.07|1992-01-07|1-URGENT|Clerk#000000278|0|r the even requests. accounts maintain. regular accounts|
+3008|40|O|156018.74|1995-11-08|3-MEDIUM|Clerk#000000701|0|ze quickly. blithely regular packages above the slyly bold foxes shall|
+3009|55|O|108424.94|1997-02-28|1-URGENT|Clerk#000000205|0|r ideas. carefully pe|
+3010|8|O|141647.08|1996-01-26|2-HIGH|Clerk#000000931|0| blithely final requests. special deposits are slyl|
+3011|91|F|46418.85|1992-01-14|5-LOW|Clerk#000000515|0|onic deposits kindle slyly. dependencies around the quickly iro|
+3012|32|F|91678.66|1993-05-05|1-URGENT|Clerk#000000414|0|ts after the regular pinto beans impress blithely s|
+3013|143|O|156407.40|1997-02-05|5-LOW|Clerk#000000591|0|the furiously pendin|
+3014|29|F|194159.59|1992-10-30|4-NOT SPECIFIED|Clerk#000000476|0|ep blithely according to the blith|
+3015|103|F|110826.83|1992-09-27|5-LOW|Clerk#000000013|0|ously regular deposits affix carefully. furiousl|
+3040|112|F|119201.64|1993-04-12|3-MEDIUM|Clerk#000000544|0|carefully special packages. blithe|
+3041|113|O|23039.46|1997-06-03|5-LOW|Clerk#000000092|0|s. unusual, pending deposits use carefully. thinly final|
+3042|20|F|104523.03|1994-11-21|3-MEDIUM|Clerk#000000573|0| the slyly ironic depo|
+3043|44|F|78221.69|1992-04-25|5-LOW|Clerk#000000137|0|cajole blithely furiously fina|
+3044|53|O|52433.54|1996-04-03|2-HIGH|Clerk#000000008|0|cajole final courts. ironic deposits about the quickly final re|
+3045|50|O|85822.67|1995-09-27|1-URGENT|Clerk#000000405|0| express courts sleep quickly special asymptotes. |
+3046|32|O|117817.52|1995-11-30|2-HIGH|Clerk#000000522|0|r deposits. platelets use furi|
+3047|25|O|37881.31|1997-03-21|1-URGENT|Clerk#000000962|0|as. slyly express deposits are dogged pearls. silent ide|
+3072|23|F|87475.82|1994-01-30|4-NOT SPECIFIED|Clerk#000000370|0|ely final deposits cajole carefully. ironic, re|
+3073|136|F|151419.50|1994-01-08|3-MEDIUM|Clerk#000000404|0|kly slyly bold accounts. express courts near the regular ideas sleep bli|
+3074|67|F|85861.93|1992-11-01|5-LOW|Clerk#000000546|0|yly even asymptotes shall have to haggle fluffily. deposits are|
+3075|127|F|37696.70|1994-05-07|3-MEDIUM|Clerk#000000433|0|ackages: carefully unusual reques|
+3076|92|F|93828.15|1993-07-23|2-HIGH|Clerk#000000099|0|busy foxes. deposits affix quickly ironic, pending pint|
+3077|121|O|99290.01|1997-08-06|2-HIGH|Clerk#000000228|0|kly. fluffily ironic requests use qui|
+3078|49|F|46310.83|1993-02-12|2-HIGH|Clerk#000000110|0|ounts are alongside of the blith|
+3079|100|O|148299.05|1997-09-12|5-LOW|Clerk#000000505|0|lly ironic accounts|
+3104|70|F|102693.61|1993-09-16|3-MEDIUM|Clerk#000000871|0|ges boost-- regular accounts are furiousl|
+3105|137|O|125396.80|1996-11-13|4-NOT SPECIFIED|Clerk#000000772|0|s. blithely final ins|
+3106|145|O|132494.97|1997-01-12|3-MEDIUM|Clerk#000000729|0|its use slyly final theodolites; regular dolphins hang above t|
+3107|26|O|107406.26|1997-08-21|1-URGENT|Clerk#000000669|0|ously even deposits acr|
+3108|85|F|63278.00|1993-08-05|1-URGENT|Clerk#000000574|0|s packages haggle furiously am|
+3109|124|F|216104.85|1993-07-24|5-LOW|Clerk#000000936|0|bold requests sleep quickly according to the slyly final|
+3110|88|F|115161.29|1994-12-17|2-HIGH|Clerk#000000564|0|round the fluffy instructions. carefully silent packages cajol|
+3111|133|O|154383.37|1995-08-25|5-LOW|Clerk#000000922|0|slyly regular theodolites. furious deposits cajole deposits. ironic theodoli|
+3136|23|F|145426.11|1994-08-10|4-NOT SPECIFIED|Clerk#000000891|0|tructions sleep slyly. pending di|
+3137|136|O|8958.65|1995-07-26|3-MEDIUM|Clerk#000000063|0|ymptotes wake carefully above t|
+3138|139|F|139579.18|1994-02-09|4-NOT SPECIFIED|Clerk#000000650|0|e fluffily final theodolites. even dependencies wake along the quickly ir|
+3139|17|F|40975.96|1992-01-02|3-MEDIUM|Clerk#000000855|0|ounts against the ruthlessly unusual dolphins|
+3140|145|F|54356.10|1992-04-09|1-URGENT|Clerk#000000670|0|carefully ironic deposits use furiously. blith|
+3141|26|O|115959.96|1995-11-10|1-URGENT|Clerk#000000475|0|es. furiously bold instructions after the carefully final p|
+3142|8|F|16030.15|1992-06-28|3-MEDIUM|Clerk#000000043|0|usual accounts about the carefully special requests sleep slyly quickly regul|
+3143|107|F|135647.68|1993-02-17|1-URGENT|Clerk#000000519|0| are final, ironic accounts. ironic |
+3168|136|F|69412.71|1992-01-30|5-LOW|Clerk#000000352|0|s sleep slyly? ironic, furious instructions detect. quickly final i|
+3169|19|F|126804.90|1993-12-21|3-MEDIUM|Clerk#000000252|0| even pinto beans are blithely special, special multip|
+3170|5|O|190142.17|1997-11-09|1-URGENT|Clerk#000000288|0|requests. furiously bold|
+3171|47|F|84405.78|1993-04-06|5-LOW|Clerk#000000940|0|ar deposits. idly r|
+3172|89|F|121360.83|1992-06-03|4-NOT SPECIFIED|Clerk#000000771|0|es. slyly ironic packages x-ra|
+3173|148|O|64892.73|1996-08-10|5-LOW|Clerk#000000516|0|ial requests lose along t|
+3174|127|O|92856.91|1995-11-15|5-LOW|Clerk#000000663|0|rts. silent, regular pinto beans are blithely regular packages. furiousl|
+3175|44|F|205282.63|1994-07-15|5-LOW|Clerk#000000629|0| across the slyly even realms use carefully ironic deposits: sl|
+3200|13|O|131103.31|1996-02-07|1-URGENT|Clerk#000000020|0| regular dependencies impress evenly even excuses. blithely |
+3201|97|F|90755.31|1993-07-02|4-NOT SPECIFIED|Clerk#000000738|0|. busy, express instruction|
+3202|88|F|50601.01|1992-12-24|5-LOW|Clerk#000000067|0|fluffily express requests affix carefully around th|
+3203|127|O|49357.72|1997-11-05|2-HIGH|Clerk#000000493|0|e furiously silent warhorses. slyly silent deposits wake bli|
+3204|10|F|41573.42|1992-12-26|1-URGENT|Clerk#000000693|0|ess somas cajole slyly. pending accounts cajole|
+3205|148|F|153637.79|1992-04-11|5-LOW|Clerk#000000803|0|e furiously. quickly regular dinos about the final pinto be|
+3206|122|O|64344.86|1996-08-09|1-URGENT|Clerk#000000755|0|ntegrate furiously final, express |
+3207|22|O|133038.59|1998-02-16|1-URGENT|Clerk#000000695|0|uriously accounts. fluffily i|
+3232|82|F|55619.01|1992-10-09|1-URGENT|Clerk#000000314|0|yly final accounts. packages agains|
+3233|140|F|54121.92|1994-10-24|5-LOW|Clerk#000000470|0|ly ironic epitaphs use stealthy, express deposits. quickly regular instruct|
+3234|14|O|147343.68|1996-04-05|4-NOT SPECIFIED|Clerk#000000367|0|ents according to the dependencies will sleep after the blithely even p|
+3235|46|O|104695.09|1995-11-15|5-LOW|Clerk#000000349|0| quickly pinto beans. ironi|
+3236|142|O|39470.39|1996-11-06|4-NOT SPECIFIED|Clerk#000000553|0|ithely slyly pending req|
+3237|19|F|10508.12|1992-06-03|1-URGENT|Clerk#000000606|0|inal requests. slyly even foxes detect about the furiously exp|
+3238|61|F|41375.69|1993-02-21|5-LOW|Clerk#000000818|0|lly express deposits are. furiously unusual ideas wake carefully somas. instr|
+3239|35|O|156802.80|1998-01-12|4-NOT SPECIFIED|Clerk#000000619|0| cajole carefully along the furiously pending deposits. |
+3264|94|O|162634.53|1996-11-02|5-LOW|Clerk#000000244|0|carefully. express, bold|
+3265|53|F|43315.15|1992-06-27|1-URGENT|Clerk#000000265|0|re quickly quickly pe|
+3266|4|P|68309.28|1995-03-17|5-LOW|Clerk#000000545|0|refully ironic instructions. slyly final pi|
+3267|112|O|33998.90|1997-01-07|5-LOW|Clerk#000000484|0| the packages. regular decoys about the bold dependencies grow fi|
+3268|142|F|36024.96|1994-06-25|5-LOW|Clerk#000000746|0|y brave requests unwind furiously accordin|
+3269|17|O|218697.85|1996-03-01|3-MEDIUM|Clerk#000000378|0|ts. accounts wake carefully. carefully dogged accounts wake slyly slyly i|
+3270|38|O|166669.86|1997-05-28|1-URGENT|Clerk#000000375|0|uffily pending courts ca|
+3271|34|F|86534.05|1992-01-01|1-URGENT|Clerk#000000421|0|s. furiously regular requests|
+3296|148|F|187553.35|1994-10-19|3-MEDIUM|Clerk#000000991|0|as! carefully final requests wake. furiously even|
+3297|139|F|9679.45|1992-11-03|2-HIGH|Clerk#000000220|0| after the theodolites cajole carefully according to the finally|
+3298|116|O|62716.67|1996-04-17|5-LOW|Clerk#000000241|0|even accounts boost |
+3299|91|F|42867.92|1993-12-26|3-MEDIUM|Clerk#000000853|0|bold deposits. special instructions sleep care|
+3300|118|O|27049.22|1995-07-15|5-LOW|Clerk#000000198|0|ses. carefully unusual instructions must have to detect about the blithel|
+3301|133|F|48497.09|1994-09-04|4-NOT SPECIFIED|Clerk#000000325|0|ular gifts impress enticingly carefully express deposits; instructions boo|
+3302|34|O|38330.42|1995-11-14|2-HIGH|Clerk#000000367|0|eep blithely ironic requests. quickly even courts haggle slyly|
+3303|145|O|97758.28|1997-12-14|4-NOT SPECIFIED|Clerk#000000661|0|nto beans sleep furiously above the carefully ironic |
+3328|7|F|139580.85|1992-11-19|5-LOW|Clerk#000000384|0|ake among the express accounts? carefully ironic packages cajole never.|
+3329|4|O|46107.70|1995-07-03|2-HIGH|Clerk#000000236|0|old deposits. special accounts haggle furiousl|
+3330|7|F|43255.19|1994-12-19|1-URGENT|Clerk#000000124|0|kages use. carefully regular deposits cajole carefully about |
+3331|91|F|65189.17|1993-05-21|2-HIGH|Clerk#000000901|0|uffily carefully sly accounts. blithely unu|
+3332|143|F|73739.06|1994-11-05|1-URGENT|Clerk#000000840|0|ans detect carefully furiously final deposits: regular accoun|
+3333|92|F|197973.22|1992-09-16|4-NOT SPECIFIED|Clerk#000000157|0|ctions boost slyly quickly even accounts. deposits along|
+3334|76|O|28930.68|1996-02-18|5-LOW|Clerk#000000532|0|ounts maintain carefully. furiously close request|
+3335|49|O|112603.34|1995-10-15|3-MEDIUM|Clerk#000000694|0| deposits poach. ironic ideas about the carefully ironi|
+3360|103|O|168750.48|1998-01-23|5-LOW|Clerk#000000254|0| the deposits. fluffily bold requests cajole regula|
+3361|49|F|75026.51|1992-08-23|4-NOT SPECIFIED|Clerk#000000577|0|unts detect furiously instructions. slow deposi|
+3362|140|O|183176.60|1995-07-29|5-LOW|Clerk#000000011|0|the quickly pending deposits. silent, ev|
+3363|52|O|91017.61|1995-09-23|2-HIGH|Clerk#000000615|0|posits. ironic, final deposits are furiously slyly pending |
+3364|46|O|108412.57|1997-06-21|1-URGENT|Clerk#000000280|0|y even foxes? blithely stea|
+3365|82|F|174634.12|1994-11-09|2-HIGH|Clerk#000000126|0|he slyly regular foxes nag about the accounts. fluffily |
+3366|52|O|13603.08|1997-05-18|1-URGENT|Clerk#000000160|0| pinto beans upon the quickly expres|
+3367|73|F|101339.68|1992-12-31|4-NOT SPECIFIED|Clerk#000000029|0|efully blithely ironic pinto beans. carefully close |
+3392|74|O|96057.42|1995-10-28|1-URGENT|Clerk#000000325|0|es thrash blithely depths. bold multipliers wake f|
+3393|98|O|183104.71|1995-07-04|2-HIGH|Clerk#000000076|0|even requests. excuses are carefully deposits. fluf|
+3394|149|O|162165.94|1996-05-05|4-NOT SPECIFIED|Clerk#000000105|0| blithely among the attainments. carefully final accounts nag blit|
+3395|149|F|141486.77|1994-10-30|4-NOT SPECIFIED|Clerk#000000682|0|ideas haggle beside the ev|
+3396|149|F|196443.16|1994-05-21|3-MEDIUM|Clerk#000000868|0|uffily regular platelet|
+3397|130|F|80084.61|1994-06-23|3-MEDIUM|Clerk#000000048|0|yly. final deposits wake f|
+3398|67|O|1147.42|1996-09-23|1-URGENT|Clerk#000000818|0|uthless, special courts atop the unusual accounts grow fur|
+3399|122|P|56938.16|1995-02-28|4-NOT SPECIFIED|Clerk#000000575|0|the carefully sly accounts. regular, pending theodolites wa|
+3424|103|O|42410.57|1996-08-21|1-URGENT|Clerk#000000190|0|ven requests are quickly pending accounts. blithely furious requests |
+3425|115|O|157040.57|1996-03-31|4-NOT SPECIFIED|Clerk#000000188|0|ions. deposits nag blithely alongside of the carefully f|
+3426|53|O|91929.93|1996-10-16|3-MEDIUM|Clerk#000000283|0|alongside of the slyly|
+3427|4|O|133451.14|1997-05-29|4-NOT SPECIFIED|Clerk#000000404|0|y final pinto beans snooze fluffily bold asymptot|
+3428|10|O|88047.04|1996-04-07|5-LOW|Clerk#000000953|0|lar excuses. slyly pending ideas detect p|
+3429|146|O|141902.54|1997-01-06|4-NOT SPECIFIED|Clerk#000000737|0|l deposits cajole furiously enticing deposits. blithe packages haggle careful|
+3430|113|F|161066.22|1994-12-12|4-NOT SPECIFIED|Clerk#000000664|0| regular attainments are at the final foxes. final packages along the blithe|
+3431|47|F|45536.27|1993-08-22|1-URGENT|Clerk#000000439|0| sleep. slyly busy Tiresias a|
+3456|46|F|32796.35|1993-06-01|5-LOW|Clerk#000000924|0|es promise slyly. ironicall|
+3457|25|P|174223.20|1995-04-27|4-NOT SPECIFIED|Clerk#000000849|0|ely thin asymptotes. deposits kindle. pending|
+3458|95|F|153069.14|1994-12-22|2-HIGH|Clerk#000000392|0|rges snooze. slyly unusua|
+3459|119|F|127134.05|1994-07-28|4-NOT SPECIFIED|Clerk#000000777|0|n instructions? carefully regular excuses are blithely. silent, ironi|
+3460|82|O|245976.74|1995-10-03|2-HIGH|Clerk#000000078|0|ans integrate carefu|
+3461|100|F|190960.69|1993-01-31|1-URGENT|Clerk#000000504|0|al, bold deposits cajole fluffily fluffily final foxes. pending ideas beli|
+3462|133|O|63590.17|1997-05-17|3-MEDIUM|Clerk#000000657|0|uriously express asympto|
+3463|89|F|85255.56|1993-08-18|1-URGENT|Clerk#000000545|0|ding to the carefully ironic deposits|
+3488|148|F|92716.17|1995-01-08|3-MEDIUM|Clerk#000000694|0|cording to the carefully regular deposits. re|
+3489|109|F|62453.97|1993-07-29|3-MEDIUM|Clerk#000000307|0|s detect. carefully even platelets across the fur|
+3490|91|O|100106.96|1997-05-26|5-LOW|Clerk#000000703|0|gular ideas. furiously silent deposits across the unusual accounts boost i|
+3491|83|O|50287.06|1998-06-24|1-URGENT|Clerk#000000560|0|nic orbits believe carefully across the |
+3492|103|F|168721.45|1994-11-24|5-LOW|Clerk#000000066|0|packages along the regular foxes lose final dependencie|
+3493|82|F|41686.10|1993-08-24|2-HIGH|Clerk#000000887|0|lyly special accounts use blithely across the furiously sil|
+3494|49|F|136058.70|1993-04-04|5-LOW|Clerk#000000559|0|r instructions haggle. accounts cajole. carefully final requests at the |
+3495|31|O|58666.79|1996-02-26|2-HIGH|Clerk#000000441|0|nticing excuses are carefully|
+3520|125|O|151233.65|1997-08-04|1-URGENT|Clerk#000000023|0|hely. ideas nag; even, even fo|
+3521|7|F|142029.67|1992-10-26|5-LOW|Clerk#000000812|0|y even instructions cajole carefully above the bli|
+3522|26|F|151515.08|1994-09-26|5-LOW|Clerk#000000250|0|deposits-- slyly stealthy requests boost caref|
+3523|149|O|129657.08|1998-04-07|2-HIGH|Clerk#000000688|0|are on the carefully even depe|
+3524|94|F|22767.49|1992-05-03|2-HIGH|Clerk#000000607|0|efully unusual tithes among the foxes use blithely daringly bold deposits. re|
+3525|109|O|100749.60|1995-12-22|4-NOT SPECIFIED|Clerk#000000084|0|s nag among the blithely e|
+3526|56|F|53827.34|1995-03-16|5-LOW|Clerk#000000364|0|to the quickly special deposits print agai|
+3527|56|O|145232.09|1997-06-21|5-LOW|Clerk#000000874|0|regular ideas across the quickly bold theodo|
+3552|35|O|103656.44|1997-04-23|2-HIGH|Clerk#000000973|0| the ironic packages. furiously |
+3553|91|F|119838.14|1994-05-18|3-MEDIUM|Clerk#000000270|0|counts mold furiously. slyly i|
+3554|44|O|98335.61|1995-06-17|5-LOW|Clerk#000000931|0|hely ironic requests haggl|
+3555|46|O|134442.37|1996-07-07|5-LOW|Clerk#000000585|0|s nag carefully regular, even pinto be|
+3556|16|F|114681.55|1992-09-23|4-NOT SPECIFIED|Clerk#000000140|0|e. dependencies need to haggle alongs|
+3557|121|F|85477.89|1992-11-09|2-HIGH|Clerk#000000291|0|ithely courts. furi|
+3558|28|O|112912.00|1996-02-29|1-URGENT|Clerk#000000841|0|around the furiously even requests. quickl|
+3559|106|F|30722.49|1992-10-24|3-MEDIUM|Clerk#000000634|0|sly deposits. fluffily final ideas cajole careful|
+3584|13|O|80487.97|1997-08-11|1-URGENT|Clerk#000000760|0|fully bold packages. fluffily final braids haggle final, ironic dolphins. b|
+3585|139|F|159015.39|1994-11-23|2-HIGH|Clerk#000000988|0|regular asymptotes. bold pains above the carefully pending asymptot|
+3586|121|F|112845.04|1993-12-05|2-HIGH|Clerk#000000438|0|he quickly final courts. carefully regular requests nag unusua|
+3587|79|O|174798.97|1996-05-10|4-NOT SPECIFIED|Clerk#000000443|0|ular patterns detect |
+3588|119|F|207925.83|1995-03-19|4-NOT SPECIFIED|Clerk#000000316|0|ong the pains. evenly unusual |
+3589|31|F|39103.37|1994-05-26|2-HIGH|Clerk#000000023|0|ithe deposits nag furiously. furiously pending packages sleep f|
+3590|149|P|218482.70|1995-05-13|5-LOW|Clerk#000000986|0|lyly final deposits.|
+3591|136|F|98140.86|1993-12-08|3-MEDIUM|Clerk#000000144|0|ual foxes haggle! unusual request|
+3616|128|F|60933.29|1994-02-16|4-NOT SPECIFIED|Clerk#000000268|0|uickly about the quickly final requests. fluffily final packages wake evenly|
+3617|40|O|126205.42|1996-03-19|3-MEDIUM|Clerk#000000886|0|the carefully regular platelets ha|
+3618|10|O|136954.81|1997-12-13|3-MEDIUM|Clerk#000000894|0|. ideas run carefully. thin, pending |
+3619|149|O|222274.54|1996-11-20|2-HIGH|Clerk#000000211|0|uests mold after the blithely ironic excuses. slyly pending pa|
+3620|44|O|59291.75|1997-03-07|5-LOW|Clerk#000000124|0|le quickly against the epitaphs. requests sleep slyly according to the|
+3621|142|F|106150.05|1993-05-06|3-MEDIUM|Clerk#000000643|0|kly unusual deposits. qu|
+3622|91|O|109202.90|1995-11-27|5-LOW|Clerk#000000012|0|c deposits are fluffily about the blithely final theo|
+3623|4|O|175017.68|1996-12-26|1-URGENT|Clerk#000000184|0|- ironic excuses boost quickly in place |
+3648|125|F|180417.11|1993-06-17|5-LOW|Clerk#000000717|0|foxes. unusual deposits boost quickly. slyly regular asymptotes across t|
+3649|40|F|124470.32|1994-07-06|5-LOW|Clerk#000000349|0|taphs boost above the final p|
+3650|46|F|189547.57|1992-05-28|4-NOT SPECIFIED|Clerk#000000454|0|kages sleep fluffily slyly|
+3651|100|O|113191.45|1998-04-27|1-URGENT|Clerk#000000222|0|ly unusual deposits thrash quickly after the ideas.|
+3652|107|O|107732.23|1997-02-25|4-NOT SPECIFIED|Clerk#000000024|0|sly even requests after the |
+3653|40|F|142866.39|1994-03-27|1-URGENT|Clerk#000000402|0| pearls. bold accounts are along the ironic,|
+3654|7|F|222653.54|1992-06-03|5-LOW|Clerk#000000475|0|s cajole slyly carefully special theodolites. even deposits haggl|
+3655|49|F|74882.22|1992-10-06|1-URGENT|Clerk#000000815|0|er the carefully unusual deposits sleep quickly according to|
+3680|127|F|124402.59|1992-12-10|4-NOT SPECIFIED|Clerk#000000793|0|ular platelets. carefully regular packages cajole blithely al|
+3681|52|F|36889.65|1992-04-04|1-URGENT|Clerk#000000566|0|. ironic deposits against the ironic, regular frets use pending plat|
+3682|32|O|67525.43|1997-01-22|2-HIGH|Clerk#000000001|0|es haggle carefully. decoys nag |
+3683|88|F|99960.46|1993-03-04|2-HIGH|Clerk#000000248|0|ze across the express foxes. carefully special acco|
+3684|23|F|89509.91|1993-07-20|2-HIGH|Clerk#000000835|0|bold accounts affix along the carefully ironic requ|
+3685|16|F|154958.89|1992-01-17|3-MEDIUM|Clerk#000000954|0| sleep fluffily special ide|
+3686|40|O|82190.77|1998-07-07|2-HIGH|Clerk#000000175|0|s. furiously final pinto beans poach carefully among |
+3687|43|F|99851.38|1993-02-03|1-URGENT|Clerk#000000585|0|gular accounts. slyly regular instructions can are final ide|
+3712|64|F|127527.05|1992-01-02|2-HIGH|Clerk#000000032|0| promise according |
+3713|149|O|215342.63|1998-05-07|3-MEDIUM|Clerk#000000325|0|s haggle quickly. ironic, regular Tiresi|
+3714|40|O|84493.55|1998-05-01|3-MEDIUM|Clerk#000000595|0|nding accounts. ironic pinto beans wake slyly. furiously pendin|
+3715|65|O|64000.93|1996-03-18|1-URGENT|Clerk#000000463|0| always silent requests wake pinto beans. slyly pending foxes are aga|
+3716|43|O|146221.66|1997-08-19|4-NOT SPECIFIED|Clerk#000000748|0| pending ideas haggle. ironic,|
+3717|28|O|176525.53|1998-06-03|4-NOT SPECIFIED|Clerk#000000974|0|t the carefully even ideas use sp|
+3718|31|O|63195.54|1996-10-23|2-HIGH|Clerk#000000016|0|refully. furiously final packages use carefully slyly pending deposits! final,|
+3719|118|O|139902.71|1997-02-16|2-HIGH|Clerk#000000034|0|, enticing accounts are blithely among the daringly final asymptotes. furious|
+3744|65|F|33085.68|1992-01-10|3-MEDIUM|Clerk#000000765|0|osits sublate about the regular requests. fluffily unusual accou|
+3745|112|F|19405.73|1993-09-29|5-LOW|Clerk#000000181|0|ckages poach slyly against the foxes. slyly ironic instructi|
+3746|74|F|80018.54|1994-09-11|4-NOT SPECIFIED|Clerk#000000188|0|. express, special requests nag quic|
+3747|149|O|204355.65|1996-08-20|1-URGENT|Clerk#000000226|0|refully across the final theodolites. carefully bold accounts cajol|
+3748|53|O|83804.38|1998-02-28|1-URGENT|Clerk#000000156|0|slyly special packages|
+3749|38|P|87073.89|1995-02-24|3-MEDIUM|Clerk#000000639|0|y regular instructions haggle blithel|
+3750|97|P|177181.67|1995-04-30|3-MEDIUM|Clerk#000000885|0|y. express, even packages wake after the ide|
+3751|10|F|202917.72|1994-04-27|4-NOT SPECIFIED|Clerk#000000925|0|sheaves. express, unusual t|
+3776|85|F|150349.92|1992-11-20|2-HIGH|Clerk#000000698|0|efully even platelets slee|
+3777|28|F|82467.29|1994-04-08|3-MEDIUM|Clerk#000000941|0| regular, special dolphins cajole enticingly ca|
+3778|106|F|221036.31|1993-05-26|1-URGENT|Clerk#000000187|0| above the express requests. packages maintain fluffily according to|
+3779|74|O|31538.94|1997-01-05|4-NOT SPECIFIED|Clerk#000000670|0| against the deposits. quickly bold instructions x-ray. pending fox|
+3780|41|O|65385.42|1996-04-13|5-LOW|Clerk#000000967|0| around the brave, pendin|
+3781|139|O|133864.82|1996-06-20|1-URGENT|Clerk#000000394|0|yly after the ruthless packages. pinto beans use slyly: never ironic dependenc|
+3782|65|O|145096.17|1996-08-24|1-URGENT|Clerk#000000121|0|counts are. pending, regular asym|
+3783|44|F|155017.92|1993-12-06|4-NOT SPECIFIED|Clerk#000000614|0| along the pinto beans. special packages use. regular theo|
+3808|79|F|228054.01|1994-04-24|1-URGENT|Clerk#000000717|0|odolites. blithely ironic cour|
+3809|148|O|143070.70|1996-05-01|5-LOW|Clerk#000000646|0| regular excuses. even theodolites are fluffily according to t|
+3810|100|F|124675.27|1992-09-17|1-URGENT|Clerk#000000660|0|ters sleep across the carefully final |
+3811|80|O|154967.89|1998-04-16|3-MEDIUM|Clerk#000000290|0|sits wake slyly abo|
+3812|41|O|70502.52|1996-08-13|3-MEDIUM|Clerk#000000727|0|al, final requests cajole|
+3813|146|O|77247.05|1998-06-29|1-URGENT|Clerk#000000531|0|g the furiously regular instructions|
+3814|118|P|149451.88|1995-02-22|5-LOW|Clerk#000000669|0| the furiously pending theodo|
+3815|104|O|14275.01|1997-08-26|1-URGENT|Clerk#000000249|0|es snooze carefully stealth|
+3840|100|O|187156.38|1998-07-17|4-NOT SPECIFIED|Clerk#000000713|0|yly slow theodolites. enticingly |
+3841|58|F|129033.13|1994-10-05|4-NOT SPECIFIED|Clerk#000000018|0| bold requests sleep quickly ironic packages. sometimes regular deposits nag |
+3842|28|F|131447.03|1992-04-09|5-LOW|Clerk#000000418|0|silent ideas. final deposits use furiously. blithely express excuses cajole fu|
+3843|10|O|34035.17|1997-01-04|4-NOT SPECIFIED|Clerk#000000693|0|eodolites; slyly unusual accounts nag boldly |
+3844|79|F|6793.45|1994-12-29|1-URGENT|Clerk#000000686|0|r dolphins. slyly ironic theodolites ag|
+3845|89|F|134333.33|1992-04-26|1-URGENT|Clerk#000000404|0|es among the pending, regular accounts sleep blithely blithely even de|
+3846|49|O|123120.06|1998-02-05|2-HIGH|Clerk#000000877|0|y alongside of the slyl|
+3847|34|F|7014.31|1993-03-12|5-LOW|Clerk#000000338|0|uriously even deposits. furiously pe|
+3872|134|O|198538.68|1996-09-06|5-LOW|Clerk#000000943|0|counts boost slyly against the ironic platelets-- blithely p|
+3873|55|O|95291.79|1998-03-30|4-NOT SPECIFIED|Clerk#000000791|0|express deposits-- even ideas |
+3874|119|F|66455.34|1993-06-09|3-MEDIUM|Clerk#000000208|0|ular asymptotes sleep blithely ironic ideas. blithel|
+3875|118|O|74483.95|1997-09-10|1-URGENT|Clerk#000000587|0| solve among the fluffily even |
+3876|29|O|95126.32|1996-08-02|5-LOW|Clerk#000000708|0|into beans. blithely|
+3877|17|F|178492.01|1993-05-21|5-LOW|Clerk#000000652|0|foxes. thinly bold reques|
+3878|88|O|59989.66|1997-03-23|1-URGENT|Clerk#000000314|0|e carefully regular platelets. special, express dependencies slee|
+3879|142|O|80274.22|1995-11-23|1-URGENT|Clerk#000000231|0|sts along the quickly ironic sentiments cajole carefully according to t|
+3904|149|O|39338.44|1997-11-15|4-NOT SPECIFIED|Clerk#000000883|0|sits haggle furiously across the requests. theodolites ha|
+3905|22|F|56227.04|1993-12-21|4-NOT SPECIFIED|Clerk#000000573|0|usly even accounts lose quietly above the slyly express p|
+3906|46|F|145630.76|1992-05-28|3-MEDIUM|Clerk#000000867|0|ironic theodolites haggle blithely above the final re|
+3907|67|F|240457.56|1992-08-19|3-MEDIUM|Clerk#000000084|0|gular pinto beans sleep f|
+3908|43|F|57127.71|1993-03-09|3-MEDIUM|Clerk#000000490|0|ounts cajole. regularly|
+3909|22|O|82746.74|1998-07-27|1-URGENT|Clerk#000000980|0|nic, special theodolites sleep furiously! furiously |
+3910|64|O|47272.67|1996-08-26|3-MEDIUM|Clerk#000000270|0|ickly. furiously final packag|
+3911|10|P|35019.95|1995-03-17|4-NOT SPECIFIED|Clerk#000000818|0|he fluffily final forges haggle slyly according to the blithely|
+3936|32|O|168618.39|1996-11-07|2-HIGH|Clerk#000000200|0|iously express packages engage slyly fina|
+3937|94|O|187516.29|1997-11-30|4-NOT SPECIFIED|Clerk#000000189|0|ckages boost carefully blithely q|
+3938|31|F|46918.22|1993-03-03|1-URGENT|Clerk#000000199|0|. unusual, final foxes haggle|
+3939|70|O|8720.45|1996-01-11|5-LOW|Clerk#000000647|0|ly ruthlessly silent requests. blithely regular requests haggle blithely wh|
+3940|149|O|129012.84|1996-02-14|5-LOW|Clerk#000000363|0|e above the ideas. quickly even dependencies along the blithely ir|
+3941|136|O|95453.80|1996-08-29|2-HIGH|Clerk#000000503|0|gular theodolites integrate quickly |
+3942|76|F|38596.81|1993-06-28|4-NOT SPECIFIED|Clerk#000000608|0|eas cajole bold requests. idly silent instructions |
+3943|40|O|60314.97|1996-10-09|5-LOW|Clerk#000000482|0|se alongside of the final pinto beans. regular packages boost across the ca|
+3968|25|O|121704.45|1997-02-17|4-NOT SPECIFIED|Clerk#000000431|0| the slyly special accounts; |
+3969|52|O|169797.40|1997-05-14|2-HIGH|Clerk#000000731|0|uriously final dependencies slee|
+3970|76|F|163709.85|1992-03-27|3-MEDIUM|Clerk#000000190|0|luffily furiously regular deposits. blithely special requests cajole blithely|
+3971|104|O|47925.47|1996-06-28|5-LOW|Clerk#000000287|0|alongside of the instructions ought to are |
+3972|124|F|1861.19|1994-04-21|3-MEDIUM|Clerk#000000049|0|y regular requests haggle quickly. pending, express acco|
+3973|103|F|91541.48|1992-03-24|4-NOT SPECIFIED|Clerk#000000114|0|somas according to the quickly even instructions wake fu|
+3974|94|O|56779.06|1996-03-05|4-NOT SPECIFIED|Clerk#000000938|0|deposits are furiously beneath the bl|
+3975|118|O|37804.43|1995-04-11|3-MEDIUM|Clerk#000000016|0|ts. regular, regular Tiresias play furiously. ironi|
+4000|70|F|84053.93|1992-01-04|5-LOW|Clerk#000000339|0|le carefully closely even pinto beans. regular, ironic foxes against the|
+4001|115|O|95929.46|1997-05-15|3-MEDIUM|Clerk#000000878|0|detect. asymptotes sleep furio|
+4002|104|O|76518.11|1997-04-08|5-LOW|Clerk#000000097|0| regular braids are. furiously even patterns agains|
+4003|112|F|17603.01|1993-01-27|1-URGENT|Clerk#000000177|0| blithe theodolites are slyly. slyly silent accounts toward|
+4004|70|F|220715.14|1993-05-07|3-MEDIUM|Clerk#000000273|0|accounts among the blithely regular sentiments |
+4005|140|O|129062.13|1996-11-20|2-HIGH|Clerk#000000341|0|ily according to the slyly iron|
+4006|35|F|70557.05|1995-01-04|3-MEDIUM|Clerk#000000765|0|ly ironic packages integrate. regular requests alongside of |
+4007|8|F|116193.97|1993-06-18|2-HIGH|Clerk#000000623|0|ecial packages. slyly regular accounts integrate |
+4032|10|O|62497.51|1998-02-26|3-MEDIUM|Clerk#000000686|0|iresias sleep slyly regular ideas. quickly unusual|
+4033|83|F|57740.74|1993-06-02|5-LOW|Clerk#000000181|0|ously bold instructions haggle furiously above the fluf|
+4034|94|F|186912.51|1993-11-14|4-NOT SPECIFIED|Clerk#000000548|0|ts x-ray. express requests affix fluffily regular theodolites. pending, fina|
+4035|118|F|22840.21|1992-02-19|5-LOW|Clerk#000000097|0|he ironic deposits sleep blith|
+4036|47|O|82563.10|1997-04-26|3-MEDIUM|Clerk#000000398|0|ly express deposits nag slyly. ironic, final asymptotes boost bra|
+4037|121|F|36389.43|1993-03-24|2-HIGH|Clerk#000000384|0|t carefully above the unusual the|
+4038|94|O|155045.39|1996-01-06|1-URGENT|Clerk#000000272|0|re slyly. silent requests wake quickly. regular packages play quickly |
+4039|29|O|143753.01|1997-11-16|1-URGENT|Clerk#000000358|0|ly ironic deposits. ironic reques|
+4064|130|O|148500.71|1996-10-10|4-NOT SPECIFIED|Clerk#000000598|0|ccounts. furiously unusual theodolites wake carefully about|
+4065|80|F|156345.64|1994-06-09|1-URGENT|Clerk#000000131|0|even foxes! slyly final deposits agai|
+4066|32|O|176911.21|1997-01-27|4-NOT SPECIFIED|Clerk#000000286|0|yly ironic dinos. quickly regular accounts haggle. requests wa|
+4067|16|F|136517.34|1992-10-07|2-HIGH|Clerk#000000027|0|tes boost furiously quick asymptotes. final deposits of the dolphins solv|
+4068|125|O|71852.67|1996-09-18|3-MEDIUM|Clerk#000000203|0|lly even accounts wake furiously across the unusual platelets. unusu|
+4069|73|F|198816.13|1992-05-13|3-MEDIUM|Clerk#000000359|0|deposits: slyly bold ideas detect furiously. f|
+4070|29|O|98275.37|1995-06-12|2-HIGH|Clerk#000000713|0|xpress ideas poach ab|
+4071|148|O|67789.42|1996-09-15|4-NOT SPECIFIED|Clerk#000000486|0|nal deposits. pending deposits d|
+4096|139|F|81089.61|1992-07-03|4-NOT SPECIFIED|Clerk#000000706|0|sits. quickly thin deposits x-ray blith|
+4097|10|O|134308.04|1996-05-24|1-URGENT|Clerk#000000475|0|ickly under the even accounts. even packages after the furiously express|
+4098|23|O|48478.54|1996-11-05|4-NOT SPECIFIED|Clerk#000000491|0|otes. quickly final requests after the stealthily ironic pinto bean|
+4099|17|F|207364.80|1992-08-21|1-URGENT|Clerk#000000379|0|r platelets. slyly regular requests cajole carefully against the|
+4100|4|O|3892.77|1996-03-12|3-MEDIUM|Clerk#000000429|0|posits. carefully unusual packages use pending deposits. regular she|
+4101|142|F|21640.10|1993-11-22|4-NOT SPECIFIED|Clerk#000000704|0|y around the express, careful epitaphs. accounts use fluffily. quickly p|
+4102|22|O|128786.57|1996-03-17|1-URGENT|Clerk#000000675|0|nding dependencies was slyly about the bl|
+4103|106|F|38164.23|1992-07-03|5-LOW|Clerk#000000679|0|fully ironic dependencies.|
+4128|139|O|5472.17|1995-10-07|4-NOT SPECIFIED|Clerk#000000635|0|ctions. dependencies from the slyly regular accounts nag slyly fu|
+4129|32|F|67226.28|1993-06-26|3-MEDIUM|Clerk#000000541|0|nwind. quickly final theodolites use packages. accounts|
+4130|104|O|47823.04|1996-03-10|5-LOW|Clerk#000000609|0|omise alongside of the carefully final foxes. blithel|
+4131|44|O|145971.60|1998-01-30|1-URGENT|Clerk#000000612|0| above the foxes hang |
+4132|19|P|65601.08|1995-05-29|4-NOT SPECIFIED|Clerk#000000158|0|ld asymptotes solve alongside of the express, final packages. fluffily fi|
+4133|101|F|31693.88|1992-08-07|4-NOT SPECIFIED|Clerk#000000268|0|al, express foxes. quickly pending deposits might cajole alongsi|
+4134|97|F|125191.12|1995-01-12|1-URGENT|Clerk#000000171|0|fully even deposits. regular de|
+4135|37|O|99577.55|1997-03-10|3-MEDIUM|Clerk#000000627|0|ly quietly even ideas. deposits haggle blithely|
+4160|55|O|82493.07|1996-08-20|5-LOW|Clerk#000000283|0|the carefully special accounts. furiously regular dugouts alongs|
+4161|118|F|198995.21|1993-08-21|5-LOW|Clerk#000000047|0|nts. fluffily regular foxes above the quickly daring reques|
+4162|22|F|72359.55|1992-02-10|5-LOW|Clerk#000000179|0|r packages are slyly accounts. furiously special foxes detect carefully re|
+4163|64|F|11493.80|1992-12-21|2-HIGH|Clerk#000000268|0| integrate furiously slyly regular depende|
+4164|94|O|8709.16|1998-07-03|2-HIGH|Clerk#000000720|0| regularly busy theodolites boost furiously quickly bold packages. express, s|
+4165|4|O|11405.40|1997-07-25|3-MEDIUM|Clerk#000000621|0|special foxes affix never blithely ironic pinto beans; blithely |
+4166|43|F|100671.06|1993-02-28|5-LOW|Clerk#000000757|0|quickly sly forges impress. careful foxes across the blithely even a|
+4167|28|O|62108.45|1998-06-17|1-URGENT|Clerk#000000917|0|kly furiously even deposits. unu|
+4192|146|O|197192.95|1998-04-19|1-URGENT|Clerk#000000369|0|equests above the slyly regular pinto beans unwi|
+4193|4|F|143191.54|1994-01-09|2-HIGH|Clerk#000000201|0|ng accounts haggle quickly. packages use fluffily ironic excu|
+4194|106|F|62972.29|1994-10-16|3-MEDIUM|Clerk#000000385|0| instructions are quickly even pinto beans. courts boost furiously regular, ev|
+4195|104|F|54478.95|1993-05-29|4-NOT SPECIFIED|Clerk#000000777|0| pinto beans cajole furiously theodolites-- slyly regular deposits doub|
+4196|106|O|201455.98|1998-05-15|3-MEDIUM|Clerk#000000532|0|affix carefully. quickly final requests |
+4197|92|O|217709.03|1996-08-13|4-NOT SPECIFIED|Clerk#000000264|0| pinto beans according|
+4198|143|O|105789.01|1997-06-16|3-MEDIUM|Clerk#000000583|0|g the special packages haggle pen|
+4199|5|F|30494.62|1992-02-13|1-URGENT|Clerk#000000309|0|e blithely. special deposits haggle slyly final foxes. carefully even|
+4224|70|O|150655.44|1997-07-14|1-URGENT|Clerk#000000034|0|jole quickly final dolphins. slyly pending foxes wake furiously bold pl|
+4225|128|O|72533.07|1997-06-03|3-MEDIUM|Clerk#000000992|0|r the platelets nag among the special deposits. ironic, ironic re|
+4226|92|F|29827.44|1993-03-09|5-LOW|Clerk#000000203|0|phins wake slyly regular packages. deposits haggle slowl|
+4227|133|F|92261.08|1995-02-24|1-URGENT|Clerk#000000063|0|ng the requests; ideas haggle fluffily. slyly unusual ideas c|
+4228|110|O|22072.16|1997-03-28|5-LOW|Clerk#000000309|0|pecial requests aft|
+4229|14|O|75145.87|1998-03-03|1-URGENT|Clerk#000000301|0|p furiously: final excuses hagg|
+4230|140|F|219709.60|1992-03-04|1-URGENT|Clerk#000000364|0|lly ironic deposits integrate carefully about the fu|
+4231|86|O|111403.66|1997-11-20|4-NOT SPECIFIED|Clerk#000000630|0|ly final accounts cajole furiously accounts. bravely ironic platelets am|
+4256|118|F|23067.48|1992-04-05|4-NOT SPECIFIED|Clerk#000000043|0|y alongside of the fluffily iro|
+4257|17|P|41723.86|1995-03-25|3-MEDIUM|Clerk#000000682|0|r ideas cajole along the blithely regular gifts.|
+4258|92|O|133829.35|1996-10-27|4-NOT SPECIFIED|Clerk#000000364|0|efully final platelets around the blit|
+4259|104|O|12918.70|1997-10-09|5-LOW|Clerk#000000781|0|es snooze slyly against the furiously unusual ideas. furious|
+4260|142|F|18566.14|1992-05-16|4-NOT SPECIFIED|Clerk#000000919|0|e among the fluffily bold accounts.|
+4261|118|F|83665.20|1992-10-03|1-URGENT|Clerk#000000662|0| about the even, pending packages. slyly bold deposits boost|
+4262|88|O|176278.57|1996-08-04|3-MEDIUM|Clerk#000000239|0| of the furious accounts. furiously regular accounts w|
+4263|4|O|158885.83|1998-03-16|1-URGENT|Clerk#000000265|0|sly ruthless deposits. final packages are instructions. fu|
+4288|34|F|75030.81|1992-12-04|4-NOT SPECIFIED|Clerk#000000823|0|usly carefully even theodolites: slyly express pac|
+4289|125|F|20752.62|1993-10-07|3-MEDIUM|Clerk#000000912|0|e carefully close instructions. slyly special reques|
+4290|41|F|26128.99|1995-01-15|3-MEDIUM|Clerk#000000688|0| slyly quickly bold requests. final deposits haggle pending ideas! som|
+4291|89|F|71822.86|1993-11-29|3-MEDIUM|Clerk#000000655|0| sleep fluffily between the bold packages. bold|
+4292|25|F|145906.24|1992-01-09|3-MEDIUM|Clerk#000000794|0| ruthlessly. slyly bo|
+4293|103|O|198322.91|1996-08-20|2-HIGH|Clerk#000000750|0|ly packages. regular packages nag according to t|
+4294|49|F|232194.74|1992-08-15|3-MEDIUM|Clerk#000000407|0|ng pinto beans breach. slyly express requests bo|
+4295|5|O|77754.62|1996-02-10|3-MEDIUM|Clerk#000000023|0|e boldly bold dependencies|
+4320|115|O|67049.37|1996-12-08|4-NOT SPECIFIED|Clerk#000000223|0|ages haggle after the slowly bold se|
+4321|16|F|118896.95|1994-07-18|3-MEDIUM|Clerk#000000041|0|ending deposits are carefully carefully regular packa|
+4322|142|O|149671.92|1998-03-13|3-MEDIUM|Clerk#000000433|0|totes nag across the fluffily special instructions. quickly silent hockey |
+4323|104|F|27598.17|1994-01-23|2-HIGH|Clerk#000000282|0|lve after the slyly regular multipliers. even, regular excus|
+4324|73|O|178249.05|1995-07-17|1-URGENT|Clerk#000000800|0|ccounts. slyly stealthy requests shall have t|
+4325|130|O|20214.49|1996-07-18|2-HIGH|Clerk#000000591|0|y around the always ev|
+4326|29|O|39048.94|1996-10-27|4-NOT SPECIFIED|Clerk#000000869|0|packages. carefully express deposit|
+4327|146|P|126235.35|1995-03-16|2-HIGH|Clerk#000000571|0|yly pending braids. final requests abo|
+4352|14|O|18653.09|1997-11-26|2-HIGH|Clerk#000000620|0|ly final platelets integrate carefully even requ|
+4353|73|O|21815.30|1997-12-12|2-HIGH|Clerk#000000790|0|uickly even ideas cajole|
+4354|145|F|179827.12|1994-09-30|4-NOT SPECIFIED|Clerk#000000046|0|pending notornis. requests serve |
+4355|4|O|186370.23|1996-11-16|1-URGENT|Clerk#000000362|0|ndencies use furiously across the regular |
+4356|97|F|39828.51|1994-04-11|5-LOW|Clerk#000000956|0| asymptotes sleep blithely. asymptotes sleep. blithely regul|
+4357|47|O|67045.94|1997-10-23|4-NOT SPECIFIED|Clerk#000000031|0|ages nag between the|
+4358|25|O|46298.53|1997-08-12|1-URGENT|Clerk#000000692|0|according to the fluffily special asymptotes |
+4359|16|F|107824.40|1993-03-03|1-URGENT|Clerk#000000393|0|sts. special, unusual deposits across the ironic theodo|
+4384|25|F|52562.16|1992-07-13|1-URGENT|Clerk#000000192|0|onic platelets. furiously regular asymptotes according to the special pac|
+4385|122|O|39190.62|1996-08-06|2-HIGH|Clerk#000000597|0|ully final requests. ironic, even dolphins above the regular |
+4386|61|O|134413.58|1998-02-06|5-LOW|Clerk#000000070|0| dolphins. silent, idle pinto beans |
+4387|110|O|116740.67|1995-10-23|1-URGENT|Clerk#000000025|0|ter the regular pinto beans. special, final gifts above the requests wi|
+4388|10|O|69668.22|1996-03-28|2-HIGH|Clerk#000000715|0|ts wake against the carefully final accounts. sly|
+4389|55|F|120324.82|1994-05-05|3-MEDIUM|Clerk#000000403|0|wly express excuses after the permanently even instructions are|
+4390|7|P|140608.69|1995-05-23|1-URGENT|Clerk#000000691|0|inal pinto beans. exp|
+4391|38|F|48284.06|1992-02-18|2-HIGH|Clerk#000000880|0|regular accounts. even depo|
+4416|149|F|76067.10|1992-06-30|5-LOW|Clerk#000000391|0| deposits. ideas cajole express theodolites: |
+4417|67|O|60868.39|1998-07-09|1-URGENT|Clerk#000000365|0|ideas are alongside of the blithely final reque|
+4418|61|F|47099.71|1993-03-25|3-MEDIUM|Clerk#000000731|0|pecial pinto beans. close foxes affix iron|
+4419|104|O|94030.43|1996-06-12|4-NOT SPECIFIED|Clerk#000000410|0|ages wake furiously slyly thin theodolit|
+4420|109|F|6088.41|1994-06-18|1-URGENT|Clerk#000000706|0|lly bold deposits along the bold, pending foxes detect blithely after the acco|
+4421|10|O|258779.02|1997-04-04|3-MEDIUM|Clerk#000000246|0|t the pending warhorses. express waters a|
+4422|70|P|107140.22|1995-05-22|3-MEDIUM|Clerk#000000938|0|ly bold accounts sleep special, regular foxes. doggedly regular in|
+4423|64|F|4913.06|1995-02-17|5-LOW|Clerk#000000888|0|excuses are ruthless|
+4448|70|O|127191.47|1998-05-21|2-HIGH|Clerk#000000428|0|. deposits haggle around the silent packages; slyly unusual packages|
+4449|10|O|48206.14|1998-02-08|5-LOW|Clerk#000000035|0|ourts are carefully even deposits. pending |
+4450|106|O|110194.31|1997-07-15|1-URGENT|Clerk#000000867|0|quests boost. furiously even realms are blithely bold requests. bl|
+4451|4|F|92851.80|1994-10-01|1-URGENT|Clerk#000000181|0|. carefully final foxes along the quickly express T|
+4452|13|F|64838.66|1994-06-21|5-LOW|Clerk#000000985|0|oxes are slyly. express, ironic pinto beans wake after the quickly pending re|
+4453|65|O|137030.40|1997-04-01|3-MEDIUM|Clerk#000000603|0|ages could have to nag slyly furiously even asymptotes! slowly regular |
+4454|142|F|159578.94|1994-02-02|5-LOW|Clerk#000000411|0|uriously regular pint|
+4455|19|F|102534.63|1993-10-11|3-MEDIUM|Clerk#000000924|0|even requests. bravely regular foxes according to the carefully unusual |
+4480|85|F|28658.26|1994-03-31|4-NOT SPECIFIED|Clerk#000000534|0|press, bold deposits boost blit|
+4481|148|O|77705.40|1996-03-30|5-LOW|Clerk#000000443|0|press sheaves cajole furio|
+4482|82|P|63535.56|1995-05-15|4-NOT SPECIFIED|Clerk#000000534|0|ravely bold accounts. furiously ironic instructions affix quickly. pend|
+4483|52|F|126597.21|1992-03-07|3-MEDIUM|Clerk#000000615|0|its. blithely idle accounts run; theodolites wake carefully around the fi|
+4484|131|O|237947.61|1996-12-24|1-URGENT|Clerk#000000392|0|ct across the pinto beans. quickly pending excuses engage furiously.|
+4485|53|F|182432.17|1994-11-13|3-MEDIUM|Clerk#000000038|0|es wake slyly even packages. blithely brave requests nag above the regul|
+4486|37|O|135613.18|1998-03-03|2-HIGH|Clerk#000000656|0|ffily according to the carefully pending acc|
+4487|46|F|109469.90|1993-02-23|3-MEDIUM|Clerk#000000017|0|s up the never pending excuses wake furiously special pinto beans. furiously i|
+4512|70|O|148682.82|1995-10-25|5-LOW|Clerk#000000393|0|ending instructions maintain fu|
+4513|85|O|119820.38|1996-03-15|5-LOW|Clerk#000000154|0|ests. final, final ideas|
+4514|97|F|143899.85|1994-04-30|3-MEDIUM|Clerk#000000074|0|deposits according to the carefull|
+4515|140|F|161745.44|1992-03-17|1-URGENT|Clerk#000000191|0|quests among the accounts sleep boldly about the regular f|
+4516|130|F|35949.14|1994-03-29|3-MEDIUM|Clerk#000000739|0|ing packages sleep slyly regular attainments|
+4517|113|O|47614.08|1998-03-07|4-NOT SPECIFIED|Clerk#000000231|0|uriously final deposits doze furiously furiously reg|
+4518|125|O|25861.74|1997-05-01|3-MEDIUM|Clerk#000000187|0|luffily against the spec|
+4519|136|F|68885.66|1993-03-30|4-NOT SPECIFIED|Clerk#000000938|0|ccording to the final |
+4544|112|O|151148.81|1997-08-07|3-MEDIUM|Clerk#000000435|0|g dependencies dazzle slyly ironic somas. carefu|
+4545|59|F|143276.28|1993-01-17|4-NOT SPECIFIED|Clerk#000000303|0|ep. requests use sly|
+4546|43|O|39906.87|1995-07-29|5-LOW|Clerk#000000373|0|ns sleep. regular, regular instructions maintai|
+4547|109|F|52114.01|1993-08-23|3-MEDIUM|Clerk#000000519|0|uctions thrash platelets. slyly final foxes wake slyly against th|
+4548|127|O|139915.23|1996-06-28|5-LOW|Clerk#000000798|0| in place of the blithely express sentiments haggle slyly r|
+4549|64|O|43889.17|1998-03-05|4-NOT SPECIFIED|Clerk#000000965|0|ully even deposits dazzle. fluffily pending ideas against the requests|
+4550|118|F|27461.48|1994-12-29|2-HIGH|Clerk#000000748|0|s haggle carefully acco|
+4551|109|O|82824.14|1996-02-09|2-HIGH|Clerk#000000462|0|ts. slyly quick theodolite|
+4576|139|O|56936.10|1996-08-14|5-LOW|Clerk#000000798|0|e pending deposits. |
+4577|79|O|104259.88|1998-05-02|5-LOW|Clerk#000000409|0|ly. unusual platelets are alw|
+4578|91|F|95761.93|1992-09-13|5-LOW|Clerk#000000121|0| to the furiously ironic instructions? furiou|
+4579|106|O|85927.85|1995-12-01|2-HIGH|Clerk#000000951|0|its wake quickly blithely specia|
+4580|82|F|118464.65|1993-11-15|4-NOT SPECIFIED|Clerk#000000086|0|rs wake blithely regular requests. fluffily ev|
+4581|79|F|89592.11|1992-09-04|4-NOT SPECIFIED|Clerk#000000687|0|ges. carefully pending accounts use furiously abo|
+4582|19|O|18247.86|1996-07-04|1-URGENT|Clerk#000000638|0|g the furiously regular pac|
+4583|22|F|206495.43|1994-09-25|3-MEDIUM|Clerk#000000240|0|equests. slyly even platelets was qui|
+4608|80|F|157767.86|1994-06-17|1-URGENT|Clerk#000000259|0|y even instructions detect slyly asymptotes. blithely final packa|
+4609|133|O|70462.84|1996-12-05|3-MEDIUM|Clerk#000000239|0|hang slyly slyly expre|
+4610|26|F|135934.60|1993-06-18|5-LOW|Clerk#000000616|0|e carefully express pinto|
+4611|29|F|166506.22|1993-01-10|2-HIGH|Clerk#000000152|0|. furiously regular instructions haggle dolphins. even instructions det|
+4612|61|F|82598.87|1993-09-20|3-MEDIUM|Clerk#000000397|0|bove the deposits. even deposits dazzle. slyly express packages haggle sl|
+4613|133|O|212339.55|1998-03-05|3-MEDIUM|Clerk#000000541|0|furiously blithely pending dependen|
+4614|61|O|151801.06|1996-04-22|1-URGENT|Clerk#000000974|0| sauternes wake thinly special accounts. fur|
+4615|29|F|10500.27|1993-08-27|3-MEDIUM|Clerk#000000982|0|jole after the fluffily pending foxes. packages affix carefully acco|
+4640|97|O|81138.17|1996-01-01|5-LOW|Clerk#000000902|0|requests. deposits do detect above the blithely iron|
+4641|134|F|98485.21|1993-01-20|4-NOT SPECIFIED|Clerk#000000755|0|ronic, final requests integrate slyly: specia|
+4642|148|F|117537.87|1995-02-27|1-URGENT|Clerk#000000295|0|cial requests wake carefully around the regular, unusual ideas. furi|
+4643|67|O|52414.19|1995-06-30|2-HIGH|Clerk#000000292|0|ously regular packages. unusual, special platel|
+4644|94|O|85901.70|1998-01-17|5-LOW|Clerk#000000961|0|requests. fluffily even ideas bo|
+4645|44|F|231012.22|1994-09-20|1-URGENT|Clerk#000000764|0|fully even instructions. final gifts sublate quickly final requests. bl|
+4646|83|O|124637.19|1996-06-18|1-URGENT|Clerk#000000036|0|n place of the blithely qu|
+4647|28|F|110958.36|1994-05-14|3-MEDIUM|Clerk#000000626|0|out the deposits. slyly final pinto beans haggle idly. slyly s|
+4672|79|O|199593.71|1995-11-07|1-URGENT|Clerk#000000475|0|lyly final dependencies caj|
+4673|82|O|58094.75|1996-08-13|4-NOT SPECIFIED|Clerk#000000914|0|c deposits are slyly. bravely ironic deposits cajole carefully after the |
+4674|37|F|115411.37|1994-04-19|1-URGENT|Clerk#000000122|0|careful hockey players. carefully pending deposits caj|
+4675|86|F|68817.08|1993-11-25|4-NOT SPECIFIED|Clerk#000000741|0|al deposits haggle slyly final|
+4676|14|O|182025.95|1995-09-01|2-HIGH|Clerk#000000407|0|s. slyly bold accounts sleep furiously special|
+4677|40|O|25661.87|1998-02-21|3-MEDIUM|Clerk#000000245|0|ly pending deposits after the carefully regular foxes sleep blithely after t|
+4678|88|O|131752.07|1998-08-02|4-NOT SPECIFIED|Clerk#000000175|0|side of the bold platelets detect slyly blithely ironic e|
+4679|88|F|7211.59|1993-01-20|2-HIGH|Clerk#000000905|0|ely regular accounts affix slyly. final dolphins are. furiously final de|
+4704|2|O|63873.14|1996-08-16|4-NOT SPECIFIED|Clerk#000000256|0|lithely final requests about the fluffily regular |
+4705|98|F|173340.09|1992-03-22|4-NOT SPECIFIED|Clerk#000000522|0| special instructions poa|
+4706|25|F|101709.52|1992-12-29|4-NOT SPECIFIED|Clerk#000000722|0| packages above the never regular packages nag packages. deposits c|
+4707|91|F|61052.10|1995-02-27|2-HIGH|Clerk#000000943|0|ully enticing accounts behind the regular|
+4708|85|F|56998.36|1994-10-01|1-URGENT|Clerk#000000383|0|ly thinly even accounts. unusu|
+4709|26|O|49903.57|1996-01-08|3-MEDIUM|Clerk#000000785|0|he furiously even deposits! ironic theodolites haggle blithely. r|
+4710|100|F|88966.68|1994-12-08|4-NOT SPECIFIED|Clerk#000000734|0|the final, regular foxes. carefully ironic pattern|
+4711|142|O|129546.56|1998-05-06|1-URGENT|Clerk#000000818|0|mptotes. unusual packages wake furiously qui|
+4736|139|O|67572.73|1995-11-20|2-HIGH|Clerk#000000563|0|blithely regular courts affix into the carefully ironic deposits. slyly exp|
+4737|79|F|62014.51|1993-03-11|4-NOT SPECIFIED|Clerk#000000275|0|ents use slyly among the unusual, ironic pearls. furiously pending |
+4738|5|F|149466.62|1992-04-08|2-HIGH|Clerk#000000150|0|deposits. thin acco|
+4739|148|F|68255.82|1993-02-21|5-LOW|Clerk#000000872|0|ing to the pending attainments: pending, express account|
+4740|68|O|42579.40|1996-07-05|2-HIGH|Clerk#000000420|0| dependencies haggle about the|
+4741|127|F|180692.90|1992-07-07|4-NOT SPECIFIED|Clerk#000000983|0|ly bold deposits are slyly about the r|
+4742|64|P|155356.80|1995-03-23|3-MEDIUM|Clerk#000000058|0|n packages. quickly regular ideas cajole blithely|
+4743|97|F|65702.39|1993-03-31|5-LOW|Clerk#000000048|0|pinto beans above the bold, even idea|
+4768|136|F|4820.55|1993-11-22|2-HIGH|Clerk#000000875|0|ctions snooze idly beneath the quick waters. fluffily u|
+4769|121|P|136765.03|1995-04-14|4-NOT SPECIFIED|Clerk#000000116|0|pon the asymptotes. idle, final account|
+4770|59|O|72150.68|1995-06-20|2-HIGH|Clerk#000000461|0|cial instructions believe carefully. |
+4771|95|F|49625.21|1992-12-14|1-URGENT|Clerk#000000571|0|lly express deposits serve furiously along the f|
+4772|28|F|64102.93|1994-09-14|1-URGENT|Clerk#000000708|0|es sleep. regular requests haggle furiously slyly |
+4773|122|O|196080.26|1995-12-23|1-URGENT|Clerk#000000327|0|ptotes was slyly along the|
+4774|52|F|124380.73|1993-04-20|3-MEDIUM|Clerk#000000299|0|eposits use blithely bold deposits. carefully regular gifts about the fin|
+4775|128|O|112444.42|1995-08-13|4-NOT SPECIFIED|Clerk#000000609|0|s integrate slyly slyly final instructions. carefully bold pack|
+4800|37|F|91795.13|1992-01-06|5-LOW|Clerk#000000625|0|ggle furiously along the pending pinto beans. deposits use: final foxe|
+4801|88|O|108353.08|1996-01-25|1-URGENT|Clerk#000000553|0|r the final sentiments. pending theodolites sleep doggedly across t|
+4802|130|O|5978.65|1997-01-23|3-MEDIUM|Clerk#000000400|0| ironic, thin packages wake furiously ironic, ironic deposits. the|
+4803|124|O|158776.68|1996-02-08|5-LOW|Clerk#000000892|0|lly unusual courts are ironic|
+4804|37|F|111547.31|1992-01-28|2-HIGH|Clerk#000000614|0|ly final accounts. blithely unusual theodolite|
+4805|16|F|172102.96|1992-04-25|4-NOT SPECIFIED|Clerk#000000514|0|even accounts wake furiously slyly final accounts; blithel|
+4806|7|F|35390.15|1993-04-21|5-LOW|Clerk#000000625|0|ave accounts. furiously pending wa|
+4807|53|O|138902.23|1997-01-09|3-MEDIUM|Clerk#000000310|0|kly. slyly special accounts|
+4832|34|O|84954.79|1997-12-04|3-MEDIUM|Clerk#000000548|0|final accounts sleep among the blithe|
+4833|133|O|84800.44|1996-05-12|3-MEDIUM|Clerk#000000256|0|r deposits against the slyly final excuses slee|
+4834|19|O|124539.00|1996-09-12|2-HIGH|Clerk#000000284|0|lar accounts. furiously ironic accounts haggle slyly |
+4835|146|F|70857.51|1994-10-25|1-URGENT|Clerk#000000250|0|s integrate furiously blithely expr|
+4836|65|O|78711.40|1996-12-18|1-URGENT|Clerk#000000691|0|c packages cajole carefully through the accounts. careful|
+4837|130|O|68519.84|1998-04-24|4-NOT SPECIFIED|Clerk#000000517|0|n accounts are regular, bold accounts. even instructions use request|
+4838|44|F|61811.33|1992-08-02|1-URGENT|Clerk#000000569|0|ffily bold sentiments. carefully close dolphins cajole across the |
+4839|25|F|71241.63|1994-05-10|1-URGENT|Clerk#000000925|0| even somas. slyly express ideas lose carefully. blithely unusu|
+4864|88|F|149614.34|1992-11-11|5-LOW|Clerk#000000423|0|ests nag within the quickly ironic asymptotes. ironic|
+4865|85|O|162113.46|1997-06-07|3-MEDIUM|Clerk#000000418|0|sits boost stealthily above the bl|
+4866|53|O|25767.07|1997-08-07|2-HIGH|Clerk#000000663|0|kages. unusual packages nag fluffily. qui|
+4867|10|F|9741.03|1992-05-21|1-URGENT|Clerk#000000891|0|ss the slyly regular dependencies. fluffily regular deposits within the car|
+4868|76|O|159005.35|1997-03-02|5-LOW|Clerk#000000729|0|regular asymptotes. regular packages sublate carefully al|
+4869|58|F|175422.13|1994-09-26|5-LOW|Clerk#000000802|0|boost! ironic packages un|
+4870|103|F|94534.07|1994-08-06|3-MEDIUM|Clerk#000000911|0|nto beans about the blithely regular d|
+4871|46|O|129636.99|1995-06-12|1-URGENT|Clerk#000000531|0|ven, special instructions across t|
+4896|85|F|93206.35|1992-08-22|1-URGENT|Clerk#000000622|0|sly pending deposits. final accounts boost above the sly, even|
+4897|80|F|115688.85|1992-09-17|5-LOW|Clerk#000000184|0|s. bold pinto beans sleep. evenly final accounts daz|
+4898|14|F|40572.64|1994-07-11|4-NOT SPECIFIED|Clerk#000000841|0|final patterns. special theodolites haggle ruthlessly at the blithely spec|
+4899|61|F|12291.83|1993-10-18|4-NOT SPECIFIED|Clerk#000000348|0| instructions. furiously even packages are furiously speci|
+4900|137|F|221320.76|1992-06-30|4-NOT SPECIFIED|Clerk#000000878|0|sleep quickly unusual |
+4901|79|O|146298.28|1997-12-31|4-NOT SPECIFIED|Clerk#000000980|0|inal dependencies cajole furiously. carefully express accounts na|
+4902|139|O|26011.20|1998-07-04|3-MEDIUM|Clerk#000000874|0| the slyly express dolphins. |
+4903|92|F|34363.63|1992-03-22|4-NOT SPECIFIED|Clerk#000000907|0|yly. multipliers within the fo|
+4928|4|F|59931.42|1993-10-04|4-NOT SPECIFIED|Clerk#000000952|0|slyly brave instructions after the ironic excuses haggle ruthlessly about|
+4929|149|O|135187.33|1996-02-29|3-MEDIUM|Clerk#000000109|0|uests. furiously special ideas poach. pending |
+4930|149|F|176867.34|1994-05-06|5-LOW|Clerk#000000593|0| haggle slyly quietly final theodolites. packages are furious|
+4931|50|F|115759.13|1994-11-17|1-URGENT|Clerk#000000356|0|leep. slyly express dolphins nag slyly. furiously regular s|
+4932|122|F|42927.07|1993-08-10|1-URGENT|Clerk#000000830|0|onic foxes. enticingly reg|
+4933|94|O|42945.82|1995-07-14|3-MEDIUM|Clerk#000000848|0|y special sauternes integr|
+4934|40|O|180478.16|1997-02-17|1-URGENT|Clerk#000000372|0|nes cajole; carefully special accounts haggle. special pinto beans nag |
+4935|40|F|162088.30|1993-05-25|4-NOT SPECIFIED|Clerk#000000601|0|c foxes. fluffily pendin|
+4960|124|F|153259.41|1995-02-26|5-LOW|Clerk#000000229|0|uriously even excuses. fluffily regular instructions along the furiously ironi|
+4961|58|O|89224.24|1998-04-06|3-MEDIUM|Clerk#000000731|0| braids. furiously even theodolites |
+4962|104|F|44781.32|1993-07-28|3-MEDIUM|Clerk#000000008|0| breach never ironic |
+4963|34|O|54175.35|1996-11-07|3-MEDIUM|Clerk#000000754|0|ully unusual epitaphs nod s|
+4964|101|O|204163.10|1997-07-28|4-NOT SPECIFIED|Clerk#000000144|0|ithely final theodolites. blithely regu|
+4965|52|F|110626.82|1993-10-21|5-LOW|Clerk#000000638|0|dependencies poach packages. sometim|
+4966|70|O|59186.02|1996-09-07|2-HIGH|Clerk#000000243|0|accounts. blithely ironic courts wake boldly furiously express |
+4967|98|O|103814.27|1997-02-17|3-MEDIUM|Clerk#000000397|0|e theodolites; furiously b|
+4992|62|F|203904.80|1992-05-10|1-URGENT|Clerk#000000166|0|telets nag carefully am|
+4993|13|F|145730.19|1994-08-04|4-NOT SPECIFIED|Clerk#000000258|0|ing instructions nag furiously. un|
+4994|43|O|216071.76|1996-06-29|4-NOT SPECIFIED|Clerk#000000868|0|oxes wake above the asymptotes. bold requests sleep br|
+4995|40|O|189651.76|1996-01-06|4-NOT SPECIFIED|Clerk#000000748|0|s. even deposits boost along the express, even theodolites. stealthily ir|
+4996|133|F|100750.67|1992-09-14|3-MEDIUM|Clerk#000000433|0|foxes. carefully special packages haggle quickly fluffi|
+4997|47|O|122611.05|1998-03-18|5-LOW|Clerk#000000040|0|egrate final pinto beans. fluffily special notornis use blith|
+4998|32|F|129096.80|1992-01-11|4-NOT SPECIFIED|Clerk#000000054|0|alongside of the quickly final requests hang always|
+4999|85|F|98643.17|1993-06-26|2-HIGH|Clerk#000000504|0| dolphins cajole blithely above the sly |
+5024|124|O|116127.69|1996-10-25|3-MEDIUM|Clerk#000000659|0|r foxes. regular excuses are about the quickly regular theodolites. regular, |
+5025|121|O|20099.43|1997-02-03|5-LOW|Clerk#000000805|0|ackages are slyly about the quickly |
+5026|28|O|13197.78|1997-09-06|1-URGENT|Clerk#000000955|0|y final requests us|
+5027|148|O|181346.56|1997-08-30|2-HIGH|Clerk#000000751|0|e-- final, pending requests along t|
+5028|13|F|30755.69|1992-04-17|2-HIGH|Clerk#000000180|0|ickly blithely express deposits. b|
+5029|11|F|19811.69|1992-11-14|3-MEDIUM|Clerk#000000469|0|. regular accounts haggle slyly. regul|
+5030|106|O|71781.23|1998-05-25|4-NOT SPECIFIED|Clerk#000000564|0| wake slyly furiously thin requests. ironic pinto beans ha|
+5031|139|F|91438.59|1994-12-02|3-MEDIUM|Clerk#000000788|0|lar instructions haggle blithely pending foxes? sometimes final excuses h|
+5056|52|O|62258.18|1997-02-15|5-LOW|Clerk#000000828|0|lithely above the express ideas. blithely final deposits are fluffily spec|
+5057|64|O|76164.41|1997-08-03|1-URGENT|Clerk#000000955|0|r ironic requests of the carefully ironic dependencies wake slyly a|
+5058|119|O|17031.01|1998-03-23|1-URGENT|Clerk#000000367|0| the pending packages wake after the quickly speci|
+5059|43|F|67173.82|1993-11-10|2-HIGH|Clerk#000000058|0|latelets. final, regular accounts cajole furiously ironic pinto beans? do|
+5060|112|F|65218.47|1992-07-07|4-NOT SPECIFIED|Clerk#000000333|0|e according to the excuses. express theodo|
+5061|101|F|52190.52|1993-08-14|1-URGENT|Clerk#000000009|0|e packages use fluffily according to the carefully ironic deposits. bol|
+5062|61|F|109247.00|1992-10-08|3-MEDIUM|Clerk#000000012|0|ithely. blithely bold theodolites affix. blithely final deposits haggle ac|
+5063|23|O|98753.57|1997-05-17|2-HIGH|Clerk#000000745|0|lyly after the pending foxes. express theodolites breach across t|
+5088|130|F|101616.44|1993-01-06|5-LOW|Clerk#000000930|0|ole slyly since the quickly ironic br|
+5089|130|F|109246.54|1992-07-29|1-URGENT|Clerk#000000677|0|cial platelets. quiet, final ideas cajole carefully. unusu|
+5090|89|O|132838.49|1997-03-09|1-URGENT|Clerk#000000953|0|ress accounts affix silently carefully quick accounts. carefully f|
+5091|148|O|47852.06|1998-05-21|3-MEDIUM|Clerk#000000311|0|egular decoys mold carefully fluffily unus|
+5092|22|O|195834.96|1995-10-30|5-LOW|Clerk#000000194|0|are blithely along the pin|
+5093|79|F|190693.92|1993-09-03|3-MEDIUM|Clerk#000000802|0|ully ironic theodolites sleep above the furiously ruthless instructions. bli|
+5094|106|F|74892.08|1993-03-29|4-NOT SPECIFIED|Clerk#000000406|0|uickly pending deposits haggle quickly ide|
+5095|97|F|184583.99|1992-04-22|2-HIGH|Clerk#000000964|0|accounts are carefully! slyly even packages wake slyly a|
+5120|16|O|28007.73|1996-06-05|1-URGENT|Clerk#000000332|0|against the slyly express requests. furiousl|
+5121|133|F|150334.57|1992-05-11|4-NOT SPECIFIED|Clerk#000000736|0|gular requests. furiously final pearls against the permanent, thin courts s|
+5122|70|O|79863.84|1996-02-10|5-LOW|Clerk#000000780|0|blithely. slyly ironic deposits nag. excuses s|
+5123|10|O|11850.45|1998-02-10|1-URGENT|Clerk#000000776|0|ic requests. furiously ironic packages grow above the express, ironic inst|
+5124|25|O|159170.80|1997-04-04|4-NOT SPECIFIED|Clerk#000000749|0|kly even courts. bold packages solve. |
+5125|28|O|38065.28|1998-02-07|5-LOW|Clerk#000000834|0|ructions. dolphins wake slowly carefully unusual |
+5126|112|F|92123.32|1992-10-12|4-NOT SPECIFIED|Clerk#000000270|0|s. unusual deposits |
+5127|73|O|48024.99|1997-01-15|5-LOW|Clerk#000000829|0|fully express pinto beans. slyly final accounts along the ironic dugouts use s|
+5152|44|O|60568.34|1997-01-04|3-MEDIUM|Clerk#000000963|0| for the blithely reg|
+5153|113|O|193832.28|1995-08-26|1-URGENT|Clerk#000000954|0| the furiously ironic foxes. express packages shall cajole carefully across|
+5154|8|O|28070.86|1997-04-13|3-MEDIUM|Clerk#000000316|0|inal requests. slyly regular deposits nag. even deposits haggle agains|
+5155|77|F|70183.29|1994-06-12|2-HIGH|Clerk#000000108|0|y pending deposits are ag|
+5156|125|O|59439.44|1996-11-04|5-LOW|Clerk#000000117|0|ngside of the multipliers solve slyly requests. regu|
+5157|142|O|167056.34|1997-07-06|4-NOT SPECIFIED|Clerk#000000689|0|closely above the unusual deposits. furiously|
+5158|76|O|240284.95|1997-01-21|1-URGENT|Clerk#000000541|0| regular foxes. even foxes wake blithely |
+5159|106|O|147543.26|1996-09-25|1-URGENT|Clerk#000000303|0|tegrate slyly around the slyly sly sauternes. final pa|
+5184|85|O|209155.48|1998-07-20|5-LOW|Clerk#000000250|0|nding accounts detect final, even|
+5185|148|O|206179.68|1997-07-25|3-MEDIUM|Clerk#000000195|0| regular ideas about the even ex|
+5186|52|O|208892.63|1996-08-03|1-URGENT|Clerk#000000332|0|pecial platelets. slyly final ac|
+5187|55|O|46380.69|1997-07-16|3-MEDIUM|Clerk#000000682|0|ckly according to t|
+5188|140|P|66268.86|1995-03-02|4-NOT SPECIFIED|Clerk#000000029|0|counts. finally ironic requests ab|
+5189|71|F|184172.31|1993-11-26|5-LOW|Clerk#000000940|0|e after the pending accounts. asymptotes boost. re|
+5190|58|F|89684.31|1992-04-26|5-LOW|Clerk#000000888|0|equests. slyly unusual|
+5191|77|F|119910.04|1994-12-11|4-NOT SPECIFIED|Clerk#000000318|0|ing, regular deposits alongside of the deposits boost fluffily quickly ev|
+5216|59|O|16763.95|1997-08-14|3-MEDIUM|Clerk#000000418|0|des boost across the platelets. slyly busy theodolit|
+5217|35|O|135745.58|1995-10-13|2-HIGH|Clerk#000000873|0|ons might wake quickly according to th|
+5218|82|F|73882.37|1992-07-30|4-NOT SPECIFIED|Clerk#000000683|0|y ruthless packages according to the bold, ironic package|
+5219|88|O|21267.72|1997-02-27|1-URGENT|Clerk#000000510|0|aggle always. foxes above the ironic deposits |
+5220|10|F|24844.39|1992-07-30|2-HIGH|Clerk#000000051|0| final packages. ideas detect slyly around|
+5221|13|O|71968.10|1995-06-09|4-NOT SPECIFIED|Clerk#000000324|0|lar accounts above the sl|
+5222|80|F|1051.15|1994-05-27|4-NOT SPECIFIED|Clerk#000000613|0|along the bold ideas. furiously final foxes snoo|
+5223|149|F|105561.21|1994-06-30|1-URGENT|Clerk#000000745|0|e. theodolites serve blithely unusual, final foxes. carefully pending packag|
+5248|70|P|86958.28|1995-04-15|2-HIGH|Clerk#000000737|0|theodolites cajole according to the silent packages. quickly ironic packages a|
+5249|103|F|123586.03|1994-09-06|3-MEDIUM|Clerk#000000019|0|refully bold accounts |
+5250|97|O|29673.73|1995-07-16|2-HIGH|Clerk#000000307|0|. carefully final instructions sleep among the finally regular dependen|
+5251|34|O|34004.48|1995-04-12|3-MEDIUM|Clerk#000000687|0| ironic dugouts detect. reque|
+5252|91|O|173145.37|1996-02-17|1-URGENT|Clerk#000000724|0| ironic accounts among the silent asym|
+5253|148|P|108361.46|1995-04-11|2-HIGH|Clerk#000000275|0|egular requests! blithely regular deposits alongside of t|
+5254|112|F|196989.09|1992-07-26|4-NOT SPECIFIED|Clerk#000000527|0|he express, even ideas cajole blithely special requests|
+5255|64|O|75074.07|1996-07-12|5-LOW|Clerk#000000591|0|ly slow forges. express foxes haggle. regular, even asymp|
+5280|34|O|68052.70|1997-12-03|3-MEDIUM|Clerk#000000604|0|riously ironic instructions. ironic ideas according to the accounts boost fur|
+5281|124|O|179418.31|1995-11-02|2-HIGH|Clerk#000000158|0|ackages haggle slyly a|
+5282|50|O|94446.69|1998-01-30|1-URGENT|Clerk#000000030|0|rding to the unusual, bold accounts. regular instructions|
+5283|131|F|18594.66|1994-06-04|3-MEDIUM|Clerk#000000579|0|ests. even, final ideas alongside of t|
+5284|61|O|40548.99|1995-07-09|4-NOT SPECIFIED|Clerk#000000155|0| careful dependencies use sly|
+5285|70|F|99377.51|1994-01-18|2-HIGH|Clerk#000000976|0|p across the furiously ironic deposits.|
+5286|116|O|79646.89|1997-09-26|5-LOW|Clerk#000000606|0|structions are furiously quickly ironic asymptotes. quickly iro|
+5287|25|F|30045.95|1993-12-22|5-LOW|Clerk#000000406|0|regular packages. bold instructions sleep always. carefully final p|
+5312|65|F|66697.95|1995-02-24|2-HIGH|Clerk#000000690|0|ter the even, bold foxe|
+5313|13|O|159870.44|1997-06-17|4-NOT SPECIFIED|Clerk#000000896|0|le. final courts haggle furiously according to the |
+5314|34|O|26999.83|1995-06-02|2-HIGH|Clerk#000000617|0|ions across the quickly special d|
+5315|139|F|55554.97|1992-10-29|4-NOT SPECIFIED|Clerk#000000035|0| furiously. quickly unusual packages use. sly|
+5316|100|F|62316.61|1994-01-31|1-URGENT|Clerk#000000734|0| requests haggle across the regular, pending deposits. furiously regular requ|
+5317|37|F|228002.51|1994-09-09|5-LOW|Clerk#000000687|0|jole quickly at the slyly pend|
+5318|59|F|106935.19|1993-04-04|2-HIGH|Clerk#000000663|0|efully regular dolphins. even ideas nag fluffily furiously even packa|
+5319|98|O|68619.29|1996-01-21|1-URGENT|Clerk#000000237|0|lent requests. quickly pe|
+5344|109|O|88216.32|1998-06-21|3-MEDIUM|Clerk#000000569|0|s. ironic excuses cajole across the|
+5345|31|O|111924.56|1997-08-24|1-URGENT|Clerk#000000057|0|r the slyly silent packages. pending, even pinto b|
+5346|37|F|149536.20|1993-12-26|2-HIGH|Clerk#000000220|0|gly close packages against the even, regular escapades boost evenly accordi|
+5347|49|F|173024.71|1995-02-22|3-MEDIUM|Clerk#000000180|0|onic, regular deposits. packag|
+5348|53|O|119164.96|1997-11-08|5-LOW|Clerk#000000497|0|totes. accounts after the furiously|
+5349|67|O|38038.84|1996-09-01|1-URGENT|Clerk#000000960|0|le along the carefully bold dolphins. carefully special packa|
+5350|76|F|113417.03|1993-10-10|5-LOW|Clerk#000000604|0|ccounts after the carefully pending requests believe |
+5351|122|O|76799.25|1998-05-11|1-URGENT|Clerk#000000443|0|to beans sleep furiously after the carefully even|
+5376|149|F|98422.83|1994-07-04|5-LOW|Clerk#000000392|0|. quickly ironic deposits integrate along|
+5377|64|O|117728.37|1997-04-24|2-HIGH|Clerk#000000917|0|ons nag blithely furiously regula|
+5378|43|F|101899.93|1992-10-25|1-URGENT|Clerk#000000520|0|n ideas. regular accounts haggle. ironic ideas use along the bold ideas. blith|
+5379|89|O|47010.15|1995-08-08|2-HIGH|Clerk#000000503|0|he unusual accounts. carefully special instructi|
+5380|148|O|123014.83|1997-10-12|1-URGENT|Clerk#000000481|0|le slyly about the slyly final dolphins. fu|
+5381|32|F|223995.46|1993-01-29|5-LOW|Clerk#000000531|0|arefully bold packages are slyly furiously ironic foxes. fluffil|
+5382|35|F|138423.03|1992-01-13|5-LOW|Clerk#000000809|0|lent deposits are according to the reg|
+5383|31|O|11474.95|1995-05-26|5-LOW|Clerk#000000409|0|ly bold requests hang furiously furiously unusual accounts. evenly unusu|
+5408|23|F|123477.05|1992-07-21|5-LOW|Clerk#000000735|0|egular requests according to the|
+5409|13|F|145040.38|1992-01-09|5-LOW|Clerk#000000171|0|eans. regular accounts are regul|
+5410|22|O|139104.17|1998-07-28|4-NOT SPECIFIED|Clerk#000000117|0|final deposits: pending excuses boost. ironic theodolites cajole furi|
+5411|61|O|62541.27|1997-05-16|3-MEDIUM|Clerk#000000800|0|equests cajole slyly furious|
+5412|142|O|109979.71|1998-01-20|2-HIGH|Clerk#000000151|0|ets boost furiously regular accounts. regular foxes above th|
+5413|94|O|224382.57|1997-10-17|1-URGENT|Clerk#000000066|0|e even excuses. always final depen|
+5414|100|F|167017.39|1993-03-25|4-NOT SPECIFIED|Clerk#000000242|0|lent dependencies? carefully express requests sleep furiously ac|
+5415|23|F|176864.83|1992-08-05|3-MEDIUM|Clerk#000000998|0|ly even ideas nag blithely above the final instructions|
+5440|130|O|3223.17|1997-01-12|1-URGENT|Clerk#000000154|0|posits boost regularly ironic packages. regular, ironic deposits wak|
+5441|41|F|131891.05|1994-07-21|4-NOT SPECIFIED|Clerk#000000257|0|after the furiously ironic |
+5442|43|O|139332.94|1998-01-13|4-NOT SPECIFIED|Clerk#000000954|0|ully. quickly express accounts against the|
+5443|131|O|124950.79|1996-10-10|4-NOT SPECIFIED|Clerk#000000492|0|al foxes could detect. blithely stealthy asymptotes kind|
+5444|130|P|172908.01|1995-03-18|1-URGENT|Clerk#000000677|0| asymptotes. asymptotes cajole quickly quickly bo|
+5445|115|F|114990.63|1993-07-26|5-LOW|Clerk#000000623|0|s. even, special requests cajole furiously even, |
+5446|7|F|29920.80|1994-06-21|5-LOW|Clerk#000000304|0| furiously final pac|
+5447|13|O|29029.84|1996-03-16|3-MEDIUM|Clerk#000000597|0|uternes around the furiously bold accounts wake after |
+5472|70|F|221636.83|1993-04-11|5-LOW|Clerk#000000552|0|counts. deposits about the slyly dogged pinto beans cajole slyly|
+5473|65|F|63041.33|1992-03-25|4-NOT SPECIFIED|Clerk#000000306|0|te the quickly stealthy ideas. even, regular deposits above|
+5474|55|F|131079.52|1992-06-01|4-NOT SPECIFIED|Clerk#000000487|0|gle blithely enticing ideas. final, exp|
+5475|139|O|10645.48|1996-07-07|5-LOW|Clerk#000000856|0|es shall boost slyly. furiously even deposits lose. instruc|
+5476|91|O|26906.38|1997-11-06|1-URGENT|Clerk#000000189|0|furiously final ideas. furiously bold dependencies sleep care|
+5477|107|O|130125.64|1997-12-30|5-LOW|Clerk#000000689|0|ckages. ironic deposits caj|
+5478|116|O|97502.23|1996-05-17|1-URGENT|Clerk#000000272|0|ckages. quickly pending deposits thrash furiously: bl|
+5479|70|F|70553.45|1993-12-22|3-MEDIUM|Clerk#000000335|0|ng asymptotes. pinto beans sleep care|
+5504|19|F|41492.25|1993-01-06|2-HIGH|Clerk#000000221|0|y pending packages. furiousl|
+5505|95|O|147329.51|1997-10-04|5-LOW|Clerk#000000719|0| final, regular packages according to the slyly ironic accounts nag ironica|
+5506|91|F|8413.31|1993-11-08|1-URGENT|Clerk#000000292|0|nusual theodolites. sly|
+5507|2|O|140363.70|1998-05-28|5-LOW|Clerk#000000692|0|the carefully ironic instructions are quickly iro|
+5508|56|O|3808.05|1996-06-21|1-URGENT|Clerk#000000128|0|y express packages cajole furiously. slyly unusual requests |
+5509|80|F|135335.96|1994-04-08|5-LOW|Clerk#000000164|0|usual deposits use packages. furiously final requests wake slyly about th|
+5510|37|F|126948.81|1993-01-08|3-MEDIUM|Clerk#000000819|0| nag slyly. carefully eve|
+5511|79|F|151089.96|1994-11-29|1-URGENT|Clerk#000000438|0|ng instructions integrate fluffily among the fluffily silent accounts. bli|
+5536|116|O|108196.56|1998-03-16|4-NOT SPECIFIED|Clerk#000000076|0| carefully final dolphins. ironic, ironic deposits lose. bold, |
+5537|118|O|102207.20|1996-10-03|3-MEDIUM|Clerk#000000742|0|ng to the daring, final |
+5538|139|F|90981.28|1993-12-25|1-URGENT|Clerk#000000992|0|ttainments. slyly final ideas are about the furiously silent excuses.|
+5539|119|F|39397.60|1994-07-31|5-LOW|Clerk#000000675|0|structions. slyly regular patterns solve above the carefully expres|
+5540|130|O|90707.58|1996-10-12|4-NOT SPECIFIED|Clerk#000000120|0|y ironic packages cajole blithely|
+5541|143|O|37526.68|1997-09-30|3-MEDIUM|Clerk#000000217|0|encies among the silent accounts sleep slyly quickly pending deposits|
+5542|49|O|6402.41|1996-04-20|4-NOT SPECIFIED|Clerk#000000100|0|riously among the regularly regular pac|
+5543|115|F|118201.53|1993-09-25|3-MEDIUM|Clerk#000000644|0|ckly regular epitaphs. carefully bold accounts haggle furiously|
+5568|31|O|105421.09|1995-06-07|3-MEDIUM|Clerk#000000491|0| nag. fluffily pending de|
+5569|109|F|126113.32|1993-04-30|4-NOT SPECIFIED|Clerk#000000759|0|e regular dependencies. furiously unusual ideas b|
+5570|112|O|78567.55|1996-08-12|2-HIGH|Clerk#000000795|0|eans. ironic, even requests doze |
+5571|103|F|79248.35|1992-12-19|4-NOT SPECIFIED|Clerk#000000184|0|ts cajole furiously carefully regular sheaves. un|
+5572|8|F|182966.39|1994-07-17|2-HIGH|Clerk#000000163|0|e fluffily express deposits cajole slyly across th|
+5573|37|O|158479.37|1996-08-15|3-MEDIUM|Clerk#000000055|0|lites. slyly final pinto beans about the carefully regul|
+5574|28|F|129803.03|1992-03-10|4-NOT SPECIFIED|Clerk#000000002|0|n deposits. special, regular t|
+5575|103|O|51839.94|1995-07-24|5-LOW|Clerk#000000948|0|uriously express frays breach|
+5600|95|O|53649.35|1997-02-08|4-NOT SPECIFIED|Clerk#000000019|0|lly regular deposits. car|
+5601|11|F|118570.79|1992-01-06|2-HIGH|Clerk#000000827|0|gular deposits wake platelets? blithe|
+5602|130|O|67979.49|1997-07-30|3-MEDIUM|Clerk#000000395|0|onic asymptotes haggl|
+5603|71|F|145100.47|1992-06-20|4-NOT SPECIFIED|Clerk#000000535|0| asymptotes. fluffily ironic instructions are. pending pinto bean|
+5604|46|O|98987.51|1998-04-14|4-NOT SPECIFIED|Clerk#000000123|0|ously across the blithely ironic pinto beans. sile|
+5605|35|O|172899.84|1996-08-22|2-HIGH|Clerk#000000538|0|sleep carefully final packages. dependencies wake slyly. theodol|
+5606|149|O|219959.08|1996-11-12|5-LOW|Clerk#000000688|0|uriously express pinto beans. packages sh|
+5607|92|F|24660.06|1992-01-01|4-NOT SPECIFIED|Clerk#000000137|0|c requests promise quickly fluffily ironic deposits. caref|
+5632|79|O|89503.11|1996-02-05|1-URGENT|Clerk#000000508|0|ons. blithely pending pinto beans thrash. furiously busy theodoli|
+5633|79|O|207119.83|1998-05-31|3-MEDIUM|Clerk#000000841|0|cial deposits wake final, final|
+5634|68|O|99494.67|1996-07-31|3-MEDIUM|Clerk#000000915|0|out the accounts. carefully ironic ideas are slyly. sheaves could h|
+5635|70|F|192217.86|1992-08-16|3-MEDIUM|Clerk#000000734|0|nal platelets sleep daringly. idle, final accounts about |
+5636|122|F|143350.75|1995-02-16|3-MEDIUM|Clerk#000000916|0|. boldly even Tiresias sleep. blithely ironic packages among the ca|
+5637|103|O|128776.90|1996-06-17|3-MEDIUM|Clerk#000000183|0|nic dolphins are regular packages. ironic pinto beans hagg|
+5638|109|F|79197.77|1994-01-17|1-URGENT|Clerk#000000355|0|enly bold deposits eat. special realms play against the regular, speci|
+5639|145|F|9669.46|1994-06-02|3-MEDIUM|Clerk#000000005|0|ending packages use after the blithely regular accounts. regular package|
+5664|119|O|186215.81|1998-07-23|2-HIGH|Clerk#000000789|0|the quickly ironic dolp|
+5665|100|F|129821.09|1993-06-28|4-NOT SPECIFIED|Clerk#000000513|0| carefully special instructions. ironic pinto beans nag slyly blithe|
+5666|14|F|121663.68|1994-02-02|2-HIGH|Clerk#000000396|0|mptotes. quickly final instructions are |
+5667|44|O|37301.25|1995-08-10|1-URGENT|Clerk#000000358|0|s print upon the quickly ironic packa|
+5668|109|F|13679.32|1995-03-22|4-NOT SPECIFIED|Clerk#000000047|0|p slyly slyly express accoun|
+5669|74|O|113156.30|1996-05-06|1-URGENT|Clerk#000000336|0|ng packages nag fluffily furio|
+5670|7|F|101429.61|1993-04-21|5-LOW|Clerk#000000922|0|he carefully final packages. deposits are slyly among the requests. |
+5671|43|O|176647.54|1998-02-06|2-HIGH|Clerk#000000838|0|k dependencies. slyly |
+5696|142|P|198723.30|1995-05-04|1-URGENT|Clerk#000000447|0|e quickly unusual pack|
+5697|55|F|99177.69|1992-10-05|1-URGENT|Clerk#000000112|0|pendencies impress furiously. bold, final requests solve ab|
+5698|95|F|154936.43|1994-05-21|3-MEDIUM|Clerk#000000455|0|he furiously silent accounts haggle blithely against the carefully unusual|
+5699|142|F|226314.91|1992-07-30|5-LOW|Clerk#000000311|0|o beans. ironic asymptotes boost. blithe, final courts integrate|
+5700|143|O|79901.18|1997-12-25|1-URGENT|Clerk#000000618|0|ly pending dolphins sleep carefully slyly pending i|
+5701|43|O|16689.19|1997-02-07|5-LOW|Clerk#000000798|0| blithely final pinto beans. blit|
+5702|97|F|153024.28|1993-09-07|4-NOT SPECIFIED|Clerk#000000743|0|ironic accounts. final accounts wake express deposits. final pac|
+5703|121|F|1816.28|1993-05-16|3-MEDIUM|Clerk#000000647|0|ly special instructions. slyly even reque|
+5728|80|F|85397.04|1994-12-11|4-NOT SPECIFIED|Clerk#000000426|0|furiously express pin|
+5729|44|F|88080.33|1994-10-10|2-HIGH|Clerk#000000843|0|uffily sly accounts about|
+5730|11|O|10934.84|1997-12-18|1-URGENT|Clerk#000000181|0|l platelets. ironic pinto beans wake slyly. quickly b|
+5731|8|O|57823.37|1997-05-17|5-LOW|Clerk#000000841|0| silent excuses among the express accounts wake |
+5732|37|O|28330.42|1997-08-03|1-URGENT|Clerk#000000910|0|he quickly bold asymptotes: final platelets wake quickly. blithely final pinto|
+5733|101|F|38545.97|1993-03-17|2-HIGH|Clerk#000000873|0|osits. pending accounts boost quickly. furiously permanent acco|
+5734|94|O|45860.94|1997-10-12|3-MEDIUM|Clerk#000000084|0|efully even braids detect blithely alo|
+5735|40|F|39358.51|1994-12-11|3-MEDIUM|Clerk#000000600|0| bold realms cajole slyly fu|
+5760|25|F|59404.77|1994-05-25|4-NOT SPECIFIED|Clerk#000000498|0|s among the blithely regular frays haggle ironically bold theodolites. al|
+5761|16|O|130345.90|1998-07-06|3-MEDIUM|Clerk#000000208|0|s asymptotes cajole boldly. regular, |
+5762|49|O|165019.32|1997-02-14|1-URGENT|Clerk#000000901|0|ly bold packages: slyly ironic deposits sleep quietly foxes. express a|
+5763|8|O|140838.11|1998-06-26|4-NOT SPECIFIED|Clerk#000000633|0|according to the furiously regular pinto beans. even accounts wake fu|
+5764|131|F|53212.95|1993-10-03|4-NOT SPECIFIED|Clerk#000000363|0| furiously regular deposits haggle fluffily around th|
+5765|52|F|249900.42|1994-12-15|5-LOW|Clerk#000000959|0|longside of the quickly final packages. instructions so|
+5766|49|F|47940.51|1993-09-27|5-LOW|Clerk#000000753|0|. quickly final packages print slyly. fu|
+5767|118|F|135643.87|1992-04-29|2-HIGH|Clerk#000000225|0|ts wake fluffily above the r|
+5792|26|F|158991.89|1993-04-04|2-HIGH|Clerk#000000731|0|packages. doggedly bold deposits integrate furiously across the|
+5793|37|O|119887.47|1997-07-13|2-HIGH|Clerk#000000294|0|thely. fluffily even instructi|
+5794|8|F|122823.78|1993-04-05|5-LOW|Clerk#000000855|0|t accounts kindle about the gifts. as|
+5795|37|F|35514.45|1992-05-05|2-HIGH|Clerk#000000581|0| even instructions x-ray ironic req|
+5796|149|O|23280.61|1996-01-23|3-MEDIUM|Clerk#000000326|0|eodolites. slyly ironic pinto beans at the silent, special request|
+5797|122|O|15313.61|1997-10-15|4-NOT SPECIFIED|Clerk#000000381|0|ng! packages against the blithely b|
+5798|106|O|125011.92|1998-03-30|5-LOW|Clerk#000000343|0|lent accounts affix quickly! platelets run slyly slyly final packages. f|
+5799|26|O|71381.21|1995-08-03|1-URGENT|Clerk#000000238|0| unusual deposits sleep blithely along the carefully even requests. care|
+5824|56|O|169107.85|1996-12-03|2-HIGH|Clerk#000000171|0|unusual packages. even ideas along the even requests are along th|
+5825|61|F|23020.62|1995-02-21|5-LOW|Clerk#000000494|0|regular packages use bravely.|
+5826|22|O|21119.86|1998-06-13|1-URGENT|Clerk#000000087|0|even, regular dependenc|
+5827|31|O|137297.71|1998-07-23|3-MEDIUM|Clerk#000000660|0|hely furiously blithe dolphins. slyly |
+5828|127|F|62172.34|1994-03-06|5-LOW|Clerk#000000377|0|ages boost never during the final packa|
+5829|125|O|183734.56|1997-01-11|1-URGENT|Clerk#000000196|0|gular accounts. bold accounts are blithely furiously ironic r|
+5830|85|F|28223.57|1993-03-25|3-MEDIUM|Clerk#000000233|0|lites haggle. ironic, ironic instructions maintain blit|
+5831|139|O|113505.19|1996-11-17|5-LOW|Clerk#000000585|0|s final, final pinto beans. unusual depos|
+5856|37|F|71460.49|1994-11-06|2-HIGH|Clerk#000000634|0|special excuses. slyly final theodolites cajole blithely furiou|
+5857|124|O|158345.31|1997-11-06|4-NOT SPECIFIED|Clerk#000000267|0|gage blithely. quickly special ac|
+5858|64|F|181320.50|1992-07-14|4-NOT SPECIFIED|Clerk#000000580|0|lyly pending dugouts believe through the ironic deposits. silent s|
+5859|5|O|210643.96|1997-04-23|1-URGENT|Clerk#000000993|0|requests boost. asymptotes across the deposits solve slyly furiously pendin|
+5860|13|F|9495.28|1992-02-20|4-NOT SPECIFIED|Clerk#000000079|0| beans. bold, special foxes sleep about the ir|
+5861|139|O|41450.19|1997-04-10|3-MEDIUM|Clerk#000000094|0|rthogs cajole slyly. express packages sleep blithely final |
+5862|64|O|30550.90|1997-02-20|1-URGENT|Clerk#000000039|0|leep beneath the quickly busy excuses. ironic theodolit|
+5863|65|F|67941.54|1993-11-22|3-MEDIUM|Clerk#000000774|0|ets about the slyly pending ideas sleep according to the blithely |
+5888|46|O|67167.19|1996-09-28|3-MEDIUM|Clerk#000000748|0|quickly against the furiously final requests. evenly fi|
+5889|22|O|15417.57|1995-05-23|5-LOW|Clerk#000000690|0|ites wake across the slyly ironic|
+5890|49|F|41162.24|1992-11-04|2-HIGH|Clerk#000000013|0|packages. final, final reques|
+5891|46|F|41760.00|1992-12-29|3-MEDIUM|Clerk#000000302|0|ounts haggle furiously abo|
+5892|101|P|92340.77|1995-05-09|5-LOW|Clerk#000000639|0| pending instruction|
+5893|2|F|44777.63|1992-07-08|4-NOT SPECIFIED|Clerk#000000560|0|final sentiments. instructions boost above the never speci|
+5894|71|F|70377.31|1994-08-13|2-HIGH|Clerk#000000776|0|regular deposits wake|
+5895|64|O|201419.83|1997-01-01|4-NOT SPECIFIED|Clerk#000000747|0| ironic, unusual requests cajole blithely special, special deposits. s|
+5920|119|F|142767.26|1994-11-20|2-HIGH|Clerk#000000081|0|ns: even ideas cajole slyly among the packages. never ironic patterns|
+5921|58|F|152940.00|1994-04-07|5-LOW|Clerk#000000125|0|kly special requests breach.|
+5922|143|O|142494.99|1996-11-14|5-LOW|Clerk#000000625|0| ironic instructions haggle furiously blithely regular accounts: even platele|
+5923|101|O|157968.27|1997-05-27|2-HIGH|Clerk#000000304|0|o beans haggle slyly above the regular, even dependencies|
+5924|31|O|106823.97|1995-10-10|3-MEDIUM|Clerk#000000433|0|arefully after the pains. blithely ironic pinto |
+5925|146|O|242588.87|1995-11-13|5-LOW|Clerk#000000602|0|ourts. boldly regular foxes might sleep. slyly express tithes against |
+5926|76|F|105770.53|1994-05-20|5-LOW|Clerk#000000071|0| carefully after the furiously even re|
+5927|116|O|84983.90|1997-08-28|4-NOT SPECIFIED|Clerk#000000972|0|endencies according to the slyly ironic foxes detect furiously about the furio|
+5952|148|O|128624.99|1997-04-14|3-MEDIUM|Clerk#000000950|0| regular, final pla|
+5953|7|F|95312.81|1992-03-28|1-URGENT|Clerk#000000049|0|ages are furiously. slowly bold requests|
+5954|28|F|167262.34|1992-12-03|1-URGENT|Clerk#000000968|0|requests along the blith|
+5955|94|P|67944.38|1995-03-27|5-LOW|Clerk#000000340|0|deas integrate. fluffily regular pa|
+5956|22|O|118036.54|1998-05-18|1-URGENT|Clerk#000000587|0|le even, express platelets.|
+5957|89|F|230949.45|1993-12-27|2-HIGH|Clerk#000000020|0| dependencies are slyly. bold accounts according to the carefully regular r|
+5958|115|O|145060.41|1995-09-16|3-MEDIUM|Clerk#000000787|0|e final requests detect alongside of the qu|
+5959|23|F|195515.26|1992-05-15|3-MEDIUM|Clerk#000000913|0|into beans use ironic, unusual foxes. carefully regular excuses boost caref|
+5984|70|F|83413.30|1994-06-18|5-LOW|Clerk#000000023|0|ickly final pains haggle along the furiously ironic pinto bea|
+5985|143|F|3942.73|1995-01-12|3-MEDIUM|Clerk#000000417|0|as nag fluffily slyly permanent accounts. regular depo|
+5986|115|F|92187.80|1992-04-22|2-HIGH|Clerk#000000674|0|iously unusual notornis are |
+5987|64|O|98956.82|1996-08-03|1-URGENT|Clerk#000000464|0| ideas. quietly final accounts haggle blithely pending escapade|
+5988|31|F|41655.51|1993-11-22|4-NOT SPECIFIED|Clerk#000000867|0|fully express accounts. final pi|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/part.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/part.tbl
new file mode 100644
index 0000000..691ba9d
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/part.tbl
@@ -0,0 +1,200 @@
+1|goldenrod lace spring peru powder|Manufacturer#1|Brand#13|PROMO BURNISHED COPPER|7|JUMBO PKG|901.00|ly. slyly ironi|
+2|blush rosy metallic lemon navajo|Manufacturer#1|Brand#13|LARGE BRUSHED BRASS|1|LG CASE|902.00|lar accounts amo|
+3|dark green antique puff wheat|Manufacturer#4|Brand#42|STANDARD POLISHED BRASS|21|WRAP CASE|903.00|egular deposits hag|
+4|chocolate metallic smoke ghost drab|Manufacturer#3|Brand#34|SMALL PLATED BRASS|14|MED DRUM|904.00|p furiously r|
+5|forest blush chiffon thistle chocolate|Manufacturer#3|Brand#32|STANDARD POLISHED TIN|15|SM PKG|905.00| wake carefully |
+6|white ivory azure firebrick black|Manufacturer#2|Brand#24|PROMO PLATED STEEL|4|MED BAG|906.00|sual a|
+7|blue blanched tan indian olive|Manufacturer#1|Brand#11|SMALL PLATED COPPER|45|SM BAG|907.00|lyly. ex|
+8|ivory khaki cream midnight rosy|Manufacturer#4|Brand#44|PROMO BURNISHED TIN|41|LG DRUM|908.00|eposi|
+9|thistle rose moccasin light floral|Manufacturer#4|Brand#43|SMALL BURNISHED STEEL|12|WRAP CASE|909.00|ironic foxe|
+10|floral moccasin royal powder burnished|Manufacturer#5|Brand#54|LARGE BURNISHED STEEL|44|LG CAN|910.01|ithely final deposit|
+11|chocolate turquoise sandy snow misty|Manufacturer#2|Brand#25|STANDARD BURNISHED NICKEL|43|WRAP BOX|911.01|ng gr|
+12|peru ivory olive powder frosted|Manufacturer#3|Brand#33|MEDIUM ANODIZED STEEL|25|JUMBO CASE|912.01| quickly|
+13|ghost blue olive sky gainsboro|Manufacturer#5|Brand#55|MEDIUM BURNISHED NICKEL|1|JUMBO PACK|913.01|osits.|
+14|linen seashell burnished blue gainsboro|Manufacturer#1|Brand#13|SMALL POLISHED STEEL|28|JUMBO BOX|914.01|kages c|
+15|navajo dark sky turquoise royal|Manufacturer#1|Brand#15|LARGE ANODIZED BRASS|45|LG CASE|915.01|usual ac|
+16|deep brown turquoise dim papaya|Manufacturer#3|Brand#32|PROMO PLATED TIN|2|MED PACK|916.01|unts a|
+17|burnished navy orange dodger cream|Manufacturer#4|Brand#43|ECONOMY BRUSHED STEEL|16|LG BOX|917.01| regular accounts|
+18|spring indian forest khaki midnight|Manufacturer#1|Brand#11|SMALL BURNISHED STEEL|42|JUMBO PACK|918.01|s cajole slyly a|
+19|dodger forest floral cream black|Manufacturer#2|Brand#23|SMALL ANODIZED NICKEL|33|WRAP BOX|919.01| pending acc|
+20|bisque salmon dark blanched linen|Manufacturer#1|Brand#12|LARGE POLISHED NICKEL|48|MED BAG|920.02|are across the asympt|
+21|lemon aquamarine firebrick floral almond|Manufacturer#3|Brand#33|SMALL BURNISHED TIN|31|MED BAG|921.02|ss packages. pendin|
+22|medium floral beige cornsilk olive|Manufacturer#4|Brand#43|PROMO POLISHED BRASS|19|LG DRUM|922.02| even p|
+23|firebrick bisque slate rose blanched|Manufacturer#3|Brand#35|MEDIUM BURNISHED TIN|42|JUMBO JAR|923.02|nic, fina|
+24|saddle dim white honeydew spring|Manufacturer#5|Brand#52|MEDIUM PLATED STEEL|20|MED CASE|924.02| final the|
+25|blush forest magenta metallic turquoise|Manufacturer#5|Brand#55|STANDARD BRUSHED COPPER|3|JUMBO BAG|925.02|requests wake|
+26|lawn forest misty chartreuse snow|Manufacturer#3|Brand#32|SMALL BRUSHED STEEL|32|SM CASE|926.02| instructions i|
+27|plum drab cornflower brown dim|Manufacturer#1|Brand#14|LARGE ANODIZED TIN|20|MED PKG|927.02|s wake. ir|
+28|green honeydew dim cyan bisque|Manufacturer#4|Brand#44|SMALL PLATED COPPER|19|JUMBO PKG|928.02|x-ray pending, iron|
+29|aquamarine sky salmon rose burlywood|Manufacturer#3|Brand#33|PROMO PLATED COPPER|7|LG DRUM|929.02| carefully fluffi|
+30|blush beige steel cyan navy|Manufacturer#4|Brand#42|PROMO ANODIZED TIN|17|LG BOX|930.03|carefully bus|
+31|grey ghost steel maroon moccasin|Manufacturer#5|Brand#53|STANDARD BRUSHED TIN|10|LG BAG|931.03|uriously s|
+32|yellow wheat midnight papaya blue|Manufacturer#4|Brand#42|ECONOMY PLATED BRASS|31|LG CASE|932.03|urts. carefully fin|
+33|maroon beige mint cyan peru|Manufacturer#2|Brand#22|ECONOMY PLATED NICKEL|16|LG PKG|933.03|ly eve|
+34|bisque pink dodger cream goldenrod|Manufacturer#1|Brand#13|LARGE BRUSHED STEEL|8|JUMBO BOX|934.03|riously ironic|
+35|green blue coral antique sandy|Manufacturer#4|Brand#43|MEDIUM ANODIZED BRASS|14|JUMBO PACK|935.03|e carefully furi|
+36|chiffon smoke floral misty navy|Manufacturer#2|Brand#25|SMALL BURNISHED COPPER|3|JUMBO CAN|936.03|olites o|
+37|turquoise ivory orange sandy maroon|Manufacturer#4|Brand#45|LARGE POLISHED TIN|48|JUMBO BOX|937.03|silent |
+38|seashell floral bisque midnight black|Manufacturer#4|Brand#43|ECONOMY ANODIZED BRASS|11|SM JAR|938.03|structions inte|
+39|rose dodger lace peru floral|Manufacturer#5|Brand#53|SMALL POLISHED TIN|43|JUMBO JAR|939.03|se slowly above the fl|
+40|rosy midnight floral misty green|Manufacturer#2|Brand#25|ECONOMY BURNISHED COPPER|27|SM CASE|940.04|! blithely specia|
+41|burlywood ghost antique metallic pale|Manufacturer#2|Brand#23|ECONOMY ANODIZED TIN|7|WRAP JAR|941.04|uriously. furiously cl|
+42|midnight azure bisque turquoise lawn|Manufacturer#5|Brand#52|MEDIUM BURNISHED TIN|45|LG BOX|942.04|the slow|
+43|medium khaki chocolate rosy blush|Manufacturer#4|Brand#44|PROMO POLISHED STEEL|5|WRAP CASE|943.04|e slyly along the ir|
+44|medium maroon wheat lavender blush|Manufacturer#4|Brand#45|MEDIUM PLATED TIN|48|SM PACK|944.04|pinto beans. carefully|
+45|lemon medium frosted indian linen|Manufacturer#4|Brand#43|SMALL BRUSHED NICKEL|9|WRAP BAG|945.04|nts bo|
+46|antique peru lemon sandy steel|Manufacturer#1|Brand#11|STANDARD POLISHED TIN|45|WRAP CASE|946.04|the blithely unusual |
+47|sky firebrick red linen dim|Manufacturer#4|Brand#45|LARGE BURNISHED BRASS|14|JUMBO PACK|947.04| even plate|
+48|slate thistle indian lace drab|Manufacturer#5|Brand#53|STANDARD BRUSHED STEEL|27|JUMBO CASE|948.04|ng to the depo|
+49|blue tan cornsilk burlywood beige|Manufacturer#2|Brand#24|SMALL BURNISHED TIN|31|MED DRUM|949.04|ar pack|
+50|yellow cornflower royal blush almond|Manufacturer#3|Brand#33|LARGE ANODIZED TIN|25|WRAP PKG|950.05|kages m|
+51|lime forest puff deep firebrick|Manufacturer#4|Brand#45|ECONOMY BURNISHED NICKEL|34|JUMBO PACK|951.05|n foxes|
+52|lemon lace khaki powder blanched|Manufacturer#3|Brand#35|STANDARD BURNISHED TIN|25|WRAP CASE|952.05| final deposits. fu|
+53|grey black dodger sandy puff|Manufacturer#2|Brand#23|ECONOMY BURNISHED NICKEL|32|MED BAG|953.05|mptot|
+54|peach smoke yellow pale cornflower|Manufacturer#2|Brand#21|LARGE BURNISHED COPPER|19|WRAP CASE|954.05|e blithely|
+55|antique cream pale tomato rose|Manufacturer#2|Brand#23|ECONOMY BRUSHED COPPER|9|MED BAG|955.05|ly final pac|
+56|midnight goldenrod drab ivory blanched|Manufacturer#1|Brand#12|MEDIUM PLATED STEEL|20|WRAP DRUM|956.05|ts. blithel|
+57|purple blue tan goldenrod medium|Manufacturer#3|Brand#32|MEDIUM BURNISHED BRASS|49|MED PKG|957.05|lly abov|
+58|linen beige coral dim salmon|Manufacturer#5|Brand#53|STANDARD POLISHED TIN|44|LG PACK|958.05| fluffily blithely reg|
+59|misty blush metallic spring blanched|Manufacturer#5|Brand#53|MEDIUM POLISHED TIN|2|LG BAG|959.05|regular exc|
+60|sky burnished salmon navajo hot|Manufacturer#1|Brand#11|LARGE POLISHED COPPER|27|JUMBO CASE|960.06| integ|
+61|drab burnished lime navy slate|Manufacturer#5|Brand#54|SMALL BURNISHED NICKEL|18|WRAP DRUM|961.06|es. blithely en|
+62|tan ivory aquamarine indian burlywood|Manufacturer#3|Brand#35|STANDARD BRUSHED BRASS|39|JUMBO BOX|962.06|ckly across the carefu|
+63|frosted puff hot rose coral|Manufacturer#3|Brand#32|STANDARD BURNISHED NICKEL|10|JUMBO CAN|963.06| quickly |
+64|aquamarine puff papaya hot floral|Manufacturer#2|Brand#21|MEDIUM ANODIZED BRASS|1|JUMBO CAN|964.06|efully regular pi|
+65|slate lavender tomato black floral|Manufacturer#5|Brand#53|MEDIUM BRUSHED COPPER|3|MED CAN|965.06|posits after the quic|
+66|cornflower orchid chocolate lawn light|Manufacturer#3|Brand#35|PROMO ANODIZED NICKEL|46|SM CASE|966.06|haggle blithely iro|
+67|slate cornsilk red orange plum|Manufacturer#2|Brand#21|SMALL BRUSHED TIN|31|WRAP DRUM|967.06| regular, p|
+68|bisque frosted pale puff sandy|Manufacturer#1|Brand#11|PROMO ANODIZED STEEL|10|WRAP BOX|968.06|eposits shall h|
+69|peru green linen blush olive|Manufacturer#5|Brand#52|MEDIUM POLISHED BRASS|2|SM BOX|969.06|ely final depo|
+70|violet midnight dark cornsilk wheat|Manufacturer#1|Brand#11|STANDARD BRUSHED STEEL|42|LG PACK|970.07|inal gifts. sl|
+71|sky green spring lavender white|Manufacturer#3|Brand#33|STANDARD PLATED BRASS|26|WRAP DRUM|971.07| packages alongside|
+72|orange spring burlywood chocolate black|Manufacturer#2|Brand#23|STANDARD ANODIZED TIN|25|JUMBO PACK|972.07|efully final the|
+73|cream misty tomato peach burlywood|Manufacturer#2|Brand#21|SMALL BRUSHED COPPER|35|WRAP DRUM|973.07|ts haggl|
+74|dim linen frosted thistle pale|Manufacturer#5|Brand#55|ECONOMY ANODIZED BRASS|25|JUMBO CASE|974.07|ent foxes|
+75|aquamarine lace wheat salmon blue|Manufacturer#3|Brand#35|SMALL BURNISHED NICKEL|39|SM JAR|975.07|s sleep furiou|
+76|bisque light yellow puff salmon|Manufacturer#3|Brand#34|MEDIUM BRUSHED COPPER|9|SM PKG|976.07|n accounts sleep qu|
+77|slate bisque burnished honeydew beige|Manufacturer#5|Brand#52|STANDARD BRUSHED COPPER|13|MED PKG|977.07|uests.|
+78|blush green slate bisque powder|Manufacturer#1|Brand#14|ECONOMY POLISHED STEEL|24|LG JAR|978.07|icing deposits wake|
+79|puff pink floral steel forest|Manufacturer#4|Brand#45|PROMO ANODIZED BRASS|22|JUMBO BAG|979.07| foxes are slyly regu|
+80|tomato saddle brown bisque light|Manufacturer#4|Brand#44|PROMO PLATED BRASS|28|MED CAN|980.08|unusual dependencies i|
+81|misty salmon cornflower dark frosted|Manufacturer#5|Brand#53|ECONOMY BRUSHED TIN|21|MED BAG|981.08|ove the furiou|
+82|khaki tomato purple indian lemon|Manufacturer#1|Brand#15|ECONOMY POLISHED TIN|12|WRAP BOX|982.08|ial requests haggle |
+83|chocolate green deep lace aquamarine|Manufacturer#1|Brand#12|PROMO BURNISHED NICKEL|47|SM CAN|983.08|ly regul|
+84|salmon peru frosted red chiffon|Manufacturer#4|Brand#45|SMALL ANODIZED NICKEL|26|JUMBO PACK|984.08|ideas nag|
+85|aquamarine cornsilk floral misty light|Manufacturer#5|Brand#55|PROMO ANODIZED NICKEL|16|LG BAG|985.08| silent|
+86|green blanched khaki steel deep|Manufacturer#4|Brand#44|STANDARD PLATED TIN|37|LG CASE|986.08| daring sheaves |
+87|pale khaki sandy antique black|Manufacturer#4|Brand#41|LARGE PLATED STEEL|41|WRAP PACK|987.08|yly final|
+88|blue olive azure burlywood lavender|Manufacturer#4|Brand#44|PROMO PLATED COPPER|16|SM CASE|988.08|e regular packages. |
+89|ghost khaki lawn pale dim|Manufacturer#5|Brand#53|STANDARD BURNISHED STEEL|7|MED JAR|989.08|y final pinto |
+90|ghost goldenrod violet blue purple|Manufacturer#5|Brand#51|ECONOMY POLISHED STEEL|49|JUMBO CAN|990.09|caref|
+91|dark beige metallic ivory burlywood|Manufacturer#2|Brand#21|STANDARD BRUSHED TIN|32|JUMBO PKG|991.09|counts dete|
+92|chiffon dodger turquoise thistle red|Manufacturer#2|Brand#22|STANDARD ANODIZED TIN|35|JUMBO PKG|992.09|he ironic accounts. sp|
+93|blanched yellow cornflower khaki mint|Manufacturer#2|Brand#24|LARGE ANODIZED TIN|2|WRAP DRUM|993.09| platel|
+94|azure pink orange goldenrod grey|Manufacturer#3|Brand#35|STANDARD POLISHED BRASS|32|SM BOX|994.09|s accounts cajo|
+95|coral chocolate wheat smoke magenta|Manufacturer#3|Brand#33|LARGE BRUSHED TIN|36|WRAP DRUM|995.09| final pinto beans |
+96|steel light drab chartreuse honeydew|Manufacturer#5|Brand#53|STANDARD BRUSHED STEEL|32|SM CASE|996.09|ng to the bli|
+97|metallic dodger aquamarine azure drab|Manufacturer#3|Brand#33|MEDIUM POLISHED BRASS|49|WRAP CAN|997.09|ss excuses sleep am|
+98|frosted goldenrod chartreuse dark honeydew|Manufacturer#5|Brand#54|STANDARD ANODIZED BRASS|22|MED JAR|998.09|e the q|
+99|lemon peru chiffon sienna deep|Manufacturer#2|Brand#21|SMALL BURNISHED STEEL|11|JUMBO PKG|999.09|press|
+100|light orange khaki chiffon purple|Manufacturer#3|Brand#33|ECONOMY ANODIZED TIN|4|LG BAG|1000.10|of the steal|
+101|hot yellow turquoise violet floral|Manufacturer#3|Brand#32|LARGE ANODIZED STEEL|26|JUMBO JAR|1001.10|ly even,|
+102|firebrick orchid blanched sienna grey|Manufacturer#3|Brand#31|MEDIUM BURNISHED BRASS|17|SM DRUM|1002.10|ular packa|
+103|navy cream lime orange metallic|Manufacturer#2|Brand#25|MEDIUM PLATED BRASS|45|WRAP DRUM|1003.10|e blithely blith|
+104|dim cream chocolate cornflower orange|Manufacturer#1|Brand#13|MEDIUM ANODIZED STEEL|36|JUMBO BAG|1004.10|ites sleep quickly|
+105|rose puff ivory blanched burnished|Manufacturer#1|Brand#15|SMALL POLISHED COPPER|27|LG DRUM|1005.10|odolites was |
+106|cornsilk beige deep lavender antique|Manufacturer#3|Brand#31|MEDIUM PLATED BRASS|28|WRAP DRUM|1006.10|unts maintain |
+107|cyan coral red slate orange|Manufacturer#5|Brand#53|SMALL BURNISHED TIN|12|MED BOX|1007.10|slyly special depos|
+108|honeydew papaya gainsboro tomato puff|Manufacturer#1|Brand#12|PROMO PLATED NICKEL|41|MED PKG|1008.10|after the carefully |
+109|lemon indian blanched chiffon coral|Manufacturer#3|Brand#33|ECONOMY POLISHED TIN|11|LG PACK|1009.10|instruction|
+110|cornflower azure beige floral black|Manufacturer#3|Brand#33|STANDARD BURNISHED COPPER|46|LG DRUM|1010.11|t quickly a|
+111|sandy almond chocolate moccasin misty|Manufacturer#5|Brand#54|LARGE BRUSHED COPPER|28|JUMBO JAR|1011.11|kly bold epitaphs |
+112|hot burnished tomato chocolate peru|Manufacturer#4|Brand#43|PROMO BRUSHED STEEL|42|JUMBO CAN|1012.11|the express, |
+113|almond coral royal honeydew lawn|Manufacturer#3|Brand#31|PROMO POLISHED TIN|23|LG CAN|1013.11|finally even |
+114|pale bisque antique white royal|Manufacturer#5|Brand#51|MEDIUM POLISHED NICKEL|41|MED PACK|1014.11|ully final foxes. pint|
+115|spring hot sienna azure dark|Manufacturer#4|Brand#45|STANDARD POLISHED STEEL|24|MED CAN|1015.11|counts nag! caref|
+116|thistle cornsilk pale firebrick black|Manufacturer#5|Brand#53|PROMO POLISHED NICKEL|33|SM PACK|1016.11|usly final courts |
+117|tomato grey chiffon burlywood ghost|Manufacturer#1|Brand#14|SMALL BRUSHED TIN|25|LG BAG|1017.11|ages acc|
+118|bisque plum royal burlywood brown|Manufacturer#2|Brand#25|PROMO ANODIZED TIN|31|MED PACK|1018.11|ly ironic pinto|
+119|olive black navajo burlywood ghost|Manufacturer#4|Brand#43|LARGE POLISHED STEEL|30|WRAP CASE|1019.11|out the quickly r|
+120|navy chocolate frosted burnished moccasin|Manufacturer#1|Brand#14|SMALL ANODIZED NICKEL|45|WRAP JAR|1020.12|lly a|
+121|smoke chiffon gainsboro magenta firebrick|Manufacturer#1|Brand#14|ECONOMY BRUSHED COPPER|13|SM PKG|1021.12|deposi|
+122|ghost royal chocolate peach ivory|Manufacturer#2|Brand#21|MEDIUM ANODIZED TIN|8|LG DRUM|1022.12|sts c|
+123|cream almond papaya salmon blanched|Manufacturer#1|Brand#12|SMALL BURNISHED TIN|31|JUMBO PKG|1023.12|ray regula|
+124|wheat azure khaki tan lime|Manufacturer#3|Brand#32|PROMO ANODIZED STEEL|1|LG BOX|1024.12|g the expr|
+125|metallic indian magenta navy medium|Manufacturer#1|Brand#12|STANDARD BRUSHED BRASS|17|WRAP BAG|1025.12|kages against|
+126|burnished navy indian medium deep|Manufacturer#4|Brand#45|MEDIUM BRUSHED NICKEL|4|LG BAG|1026.12|es sleep al|
+127|moccasin green orchid blue blanched|Manufacturer#5|Brand#52|SMALL BURNISHED NICKEL|14|LG JAR|1027.12|lithely expr|
+128|linen brown khaki snow gainsboro|Manufacturer#2|Brand#22|PROMO PLATED TIN|5|SM BAG|1028.12|e of the furiously ex|
+129|turquoise spring sandy tan ivory|Manufacturer#1|Brand#15|LARGE POLISHED TIN|20|SM JAR|1029.12| careful|
+130|gainsboro goldenrod chiffon orchid rose|Manufacturer#2|Brand#23|SMALL PLATED NICKEL|26|LG BOX|1030.13|ake slyly|
+131|tomato moccasin sandy blue frosted|Manufacturer#5|Brand#52|STANDARD ANODIZED BRASS|43|MED DRUM|1031.13|nts wake dar|
+132|seashell papaya thistle light powder|Manufacturer#4|Brand#45|STANDARD BURNISHED BRASS|2|WRAP DRUM|1032.13|ckly expre|
+133|navajo black dim peru deep|Manufacturer#1|Brand#13|SMALL BRUSHED NICKEL|19|LG PKG|1033.13| final pinto beans|
+134|turquoise goldenrod white moccasin spring|Manufacturer#4|Brand#42|SMALL POLISHED STEEL|35|SM PKG|1034.13|es. bold pa|
+135|metallic lime cyan forest burlywood|Manufacturer#2|Brand#21|MEDIUM BURNISHED STEEL|24|JUMBO CASE|1035.13|l frets |
+136|cornsilk blush powder tan rose|Manufacturer#2|Brand#22|SMALL PLATED STEEL|2|WRAP BAG|1036.13|kages print carefully|
+137|saddle dodger purple salmon rosy|Manufacturer#3|Brand#31|ECONOMY PLATED STEEL|25|MED PACK|1037.13|the t|
+138|dark goldenrod ghost hot grey|Manufacturer#1|Brand#13|ECONOMY BURNISHED COPPER|42|JUMBO DRUM|1038.13|ts solve acro|
+139|floral chocolate purple deep coral|Manufacturer#3|Brand#32|MEDIUM BRUSHED STEEL|7|SM BOX|1039.13|ter t|
+140|frosted chiffon thistle sky grey|Manufacturer#5|Brand#53|STANDARD PLATED STEEL|45|SM BOX|1040.14|oss the carefu|
+141|peru sienna salmon snow gainsboro|Manufacturer#3|Brand#35|STANDARD ANODIZED STEEL|23|SM PKG|1041.14|ans nag furiously pen|
+142|chartreuse sky deep slate thistle|Manufacturer#5|Brand#55|STANDARD ANODIZED BRASS|36|MED JAR|1042.14|he accounts. pac|
+143|moccasin dim almond black misty|Manufacturer#3|Brand#34|ECONOMY PLATED TIN|44|MED BAG|1043.14|nts across the|
+144|wheat brown orange almond aquamarine|Manufacturer#1|Brand#14|SMALL ANODIZED TIN|26|SM BOX|1044.14|owly |
+145|indian rosy chartreuse orchid grey|Manufacturer#5|Brand#53|PROMO BRUSHED COPPER|24|SM BAG|1045.14|es wake furiously blit|
+146|azure medium midnight burlywood blue|Manufacturer#3|Brand#34|STANDARD BRUSHED COPPER|11|WRAP PACK|1046.14|unts cajole|
+147|honeydew brown deep light chartreuse|Manufacturer#1|Brand#11|MEDIUM PLATED COPPER|29|JUMBO PKG|1047.14|wake never bold |
+148|yellow white tan khaki saddle|Manufacturer#3|Brand#31|STANDARD PLATED STEEL|20|SM BOX|1048.14|platelets wake fu|
+149|tan cyan forest ghost plum|Manufacturer#2|Brand#24|MEDIUM BURNISHED NICKEL|6|MED PKG|1049.14|leep requests. dog|
+150|antique red moccasin green burlywood|Manufacturer#3|Brand#35|LARGE BRUSHED TIN|21|SM BAG|1050.15|ironic foxes|
+151|chartreuse lime slate white thistle|Manufacturer#3|Brand#34|LARGE PLATED BRASS|45|MED CAN|1051.15|ccounts nag i|
+152|light indian linen dodger blush|Manufacturer#5|Brand#53|MEDIUM POLISHED STEEL|48|MED CASE|1052.15|thely regular t|
+153|powder firebrick chiffon purple khaki|Manufacturer#1|Brand#11|STANDARD PLATED TIN|20|MED BAG|1053.15|thlessly. silen|
+154|peru powder pink orchid lace|Manufacturer#1|Brand#11|ECONOMY ANODIZED TIN|1|JUMBO BAG|1054.15|posits |
+155|beige saddle cornsilk mint sky|Manufacturer#2|Brand#21|SMALL BRUSHED NICKEL|28|WRAP CASE|1055.15|lly ironic, r|
+156|steel gainsboro plum black cornsilk|Manufacturer#4|Brand#43|SMALL POLISHED NICKEL|2|LG PKG|1056.15| pinto beans. eve|
+157|navajo lime cream ghost drab|Manufacturer#1|Brand#11|ECONOMY ANODIZED STEEL|26|JUMBO PACK|1057.15|ial courts. ru|
+158|magenta light rosy almond goldenrod|Manufacturer#4|Brand#45|MEDIUM BURNISHED COPPER|47|LG JAR|1058.15| ideas detect slyl|
+159|linen drab antique steel aquamarine|Manufacturer#4|Brand#43|SMALL ANODIZED BRASS|46|SM BAG|1059.15| ironic requests-- pe|
+160|lace coral cream saddle maroon|Manufacturer#5|Brand#55|STANDARD POLISHED COPPER|47|JUMBO CAN|1060.16|nts are carefully|
+161|hot dodger rosy firebrick aquamarine|Manufacturer#2|Brand#22|STANDARD PLATED TIN|17|SM PACK|1061.16|r the bl|
+162|aquamarine coral black ivory misty|Manufacturer#3|Brand#33|MEDIUM ANODIZED COPPER|35|JUMBO PACK|1062.16|e slyly around th|
+163|blush yellow frosted lace drab|Manufacturer#2|Brand#21|ECONOMY PLATED TIN|34|WRAP DRUM|1063.16|nly s|
+164|brown cream linen frosted blue|Manufacturer#2|Brand#23|LARGE PLATED BRASS|35|JUMBO BAG|1064.16|mong th|
+165|chiffon dim coral sienna seashell|Manufacturer#1|Brand#15|STANDARD PLATED STEEL|24|SM CAN|1065.16| carefully fin|
+166|linen royal chiffon floral misty|Manufacturer#5|Brand#52|LARGE POLISHED COPPER|4|MED BAG|1066.16|ss the|
+167|almond lavender lawn pink ghost|Manufacturer#3|Brand#32|LARGE ANODIZED STEEL|46|WRAP BOX|1067.16|ic ac|
+168|lace frosted brown slate lawn|Manufacturer#1|Brand#13|SMALL BRUSHED COPPER|20|JUMBO DRUM|1068.16|ss package|
+169|tan lime sky almond midnight|Manufacturer#5|Brand#55|STANDARD POLISHED BRASS|10|JUMBO CASE|1069.16|lets alongside of|
+170|peru green hot medium thistle|Manufacturer#3|Brand#33|LARGE POLISHED COPPER|28|LG DRUM|1070.17|yly s|
+171|beige seashell almond linen blush|Manufacturer#1|Brand#11|STANDARD BURNISHED COPPER|40|LG JAR|1071.17| the r|
+172|medium goldenrod tan light brown|Manufacturer#5|Brand#53|PROMO PLATED NICKEL|28|MED CASE|1072.17|quick as|
+173|pale papaya tomato navajo coral|Manufacturer#1|Brand#12|ECONOMY BURNISHED TIN|17|LG CASE|1073.17|sly bold excuses haggl|
+174|plum pink medium saddle orchid|Manufacturer#1|Brand#15|ECONOMY BRUSHED COPPER|25|LG CASE|1074.17| accounts nag ab|
+175|azure blanched chartreuse moccasin ghost|Manufacturer#1|Brand#11|PROMO ANODIZED TIN|45|JUMBO JAR|1075.17|ole against the|
+176|lawn forest indian orchid peru|Manufacturer#2|Brand#24|SMALL ANODIZED STEEL|40|MED CAN|1076.17|blithely. ironic|
+177|indian deep sandy floral yellow|Manufacturer#2|Brand#21|MEDIUM BRUSHED STEEL|42|LG BAG|1077.17|ermanently eve|
+178|blue black cornflower white powder|Manufacturer#1|Brand#13|STANDARD POLISHED TIN|10|LG JAR|1078.17|regular instructions.|
+179|rose lime blush bisque ivory|Manufacturer#4|Brand#43|ECONOMY BRUSHED STEEL|20|LG JAR|1079.17|ely regul|
+180|purple blanched khaki blush aquamarine|Manufacturer#3|Brand#33|STANDARD BURNISHED NICKEL|7|WRAP BAG|1080.18|oss the |
+181|almond burlywood smoke peru dark|Manufacturer#2|Brand#24|MEDIUM PLATED STEEL|19|WRAP CAN|1081.18|al deposits |
+182|beige medium brown chartreuse purple|Manufacturer#3|Brand#31|MEDIUM ANODIZED COPPER|11|JUMBO CAN|1082.18|bits are |
+183|ivory white wheat pale orange|Manufacturer#5|Brand#52|PROMO POLISHED STEEL|35|LG PKG|1083.18|ly regular excus|
+184|burlywood cream cornsilk chocolate purple|Manufacturer#5|Brand#53|SMALL POLISHED TIN|42|LG BOX|1084.18|ding courts. idly iro|
+185|firebrick rose indian spring magenta|Manufacturer#4|Brand#44|ECONOMY POLISHED TIN|4|WRAP BAG|1085.18|even foxe|
+186|papaya cream chartreuse spring blush|Manufacturer#2|Brand#23|ECONOMY BRUSHED TIN|15|JUMBO PKG|1086.18|ly reg|
+187|white brown cyan lavender orange|Manufacturer#4|Brand#45|PROMO ANODIZED BRASS|45|MED CAN|1087.18|leep slyly s|
+188|gainsboro steel rosy dim drab|Manufacturer#5|Brand#54|ECONOMY ANODIZED BRASS|9|MED CAN|1088.18| above the silent p|
+189|dodger misty yellow brown grey|Manufacturer#2|Brand#22|MEDIUM BRUSHED BRASS|13|WRAP DRUM|1089.18|en requests. sauternes|
+190|chartreuse khaki ivory chiffon bisque|Manufacturer#5|Brand#53|LARGE BURNISHED NICKEL|23|WRAP BAG|1090.19| furiously even d|
+191|navajo metallic puff firebrick cornflower|Manufacturer#3|Brand#31|MEDIUM POLISHED BRASS|36|WRAP BOX|1091.19| asymptote|
+192|light seashell firebrick cornflower green|Manufacturer#3|Brand#34|STANDARD BRUSHED COPPER|17|MED BAG|1092.19|uickly regular, expr|
+193|turquoise sienna rose rosy chocolate|Manufacturer#4|Brand#45|ECONOMY BURNISHED BRASS|31|SM PKG|1093.19|final ideas wake furi|
+194|powder bisque cornflower navajo pink|Manufacturer#5|Brand#51|ECONOMY POLISHED STEEL|7|SM CAN|1094.19|y special accoun|
+195|lace floral blush linen cornflower|Manufacturer#4|Brand#41|STANDARD BRUSHED NICKEL|40|MED CASE|1095.19|oxes sleep care|
+196|pale blush lime grey blue|Manufacturer#3|Brand#33|SMALL BURNISHED NICKEL|3|JUMBO JAR|1096.19|uickly special |
+197|blue turquoise cornsilk rosy chartreuse|Manufacturer#5|Brand#52|SMALL ANODIZED COPPER|18|SM JAR|1097.19|lithely after the eve|
+198|orange khaki papaya cyan navajo|Manufacturer#4|Brand#41|PROMO BRUSHED NICKEL|43|SM PACK|1098.19|ackages? carefully re|
+199|royal slate lace olive gainsboro|Manufacturer#3|Brand#31|ECONOMY PLATED STEEL|23|JUMBO DRUM|1099.19|ickly regul|
+200|burnished mint navajo beige dark|Manufacturer#5|Brand#54|MEDIUM POLISHED BRASS|22|LG PKG|1100.20|furiously even depo|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/partsupp.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/partsupp.tbl
new file mode 100644
index 0000000..d8e5856
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/partsupp.tbl
@@ -0,0 +1,800 @@
+1|2|3325|771.64|, even theodolites. regular, final theodolites eat after the carefully pending foxes. furiously regular deposits sleep slyly. carefully bold realms above the ironic dependencies haggle careful|
+1|4|8076|993.49|ven ideas. quickly even packages print. pending multipliers must have to are fluff|
+1|6|3956|337.09|after the fluffily ironic deposits? blithely special dependencies integrate furiously even excuses. blithely silent theodolites could have to haggle pending, express requests; fu|
+1|8|4069|357.84|al, regular dependencies serve carefully after the quickly final pinto beans. furiously even deposits sleep quickly final, silent pinto beans. fluffily reg|
+2|3|8895|378.49|nic accounts. final accounts sleep furiously about the ironic, bold packages. regular, regular accounts|
+2|5|4969|915.27|ptotes. quickly pending dependencies integrate furiously. fluffily ironic ideas impress blithely above the express accounts. furiously even epitaphs need to wak|
+2|7|8539|438.37|blithely bold ideas. furiously stealthy packages sleep fluffily. slyly special deposits snooze furiously carefully regular accounts. regular deposits according to the accounts nag carefully slyl|
+2|9|3025|306.39|olites. deposits wake carefully. even, express requests cajole. carefully regular ex|
+3|4|4651|920.92|ilent foxes affix furiously quickly unusual requests. even packages across the carefully even theodolites nag above the sp|
+3|6|4093|498.13|ending dependencies haggle fluffily. regular deposits boost quickly carefully regular requests. deposits affix furiously around the pinto beans. ironic, unusual platelets across the p|
+3|8|3917|645.40|of the blithely regular theodolites. final theodolites haggle blithely carefully unusual ideas. blithely even f|
+3|10|9942|191.92| unusual, ironic foxes according to the ideas detect furiously alongside of the even, express requests. blithely regular the|
+4|5|1339|113.97| carefully unusual ideas. packages use slyly. blithely final pinto beans cajole along the furiously express requests. regular orbits haggle carefully. care|
+4|7|6377|591.18|ly final courts haggle carefully regular accounts. carefully regular accounts could integrate slyly. slyly express packages about the accounts wake slyly|
+4|9|2694|51.37|g, regular deposits: quick instructions run across the carefully ironic theodolites-- final dependencies haggle into the dependencies. f|
+4|1|2480|444.37|requests sleep quickly regular accounts. theodolites detect. carefully final depths w|
+5|6|3735|255.88|arefully even requests. ironic requests cajole carefully even dolphin|
+5|8|9653|50.52|y stealthy deposits. furiously final pinto beans wake furiou|
+5|10|1329|219.83|iously regular deposits wake deposits. pending pinto beans promise ironic dependencies. even, regular pinto beans integrate|
+5|2|6925|537.98|sits. quickly fluffy packages wake quickly beyond the blithely regular requests. pending requests cajole among the final pinto beans. carefully busy theodolites affix quickly stealthily |
+6|7|8851|130.72|usly final packages. slyly ironic accounts poach across the even, sly requests. carefully pending request|
+6|9|1627|424.25| quick packages. ironic deposits print. furiously silent platelets across the carefully final requests are slyly along the furiously even instructi|
+6|1|3336|642.13|final instructions. courts wake packages. blithely unusual realms along the multipliers nag |
+6|3|6451|175.32| accounts alongside of the slyly even accounts wake carefully final instructions-- ruthless platelets wake carefully ideas. even deposits are quickly final,|
+7|8|7454|763.98|y express tithes haggle furiously even foxes. furiously ironic deposits sleep toward the furiously unusual|
+7|10|2770|149.66|hould have to nag after the blithely final asymptotes. fluffily spe|
+7|2|3377|68.77|usly against the daring asymptotes. slyly regular platelets sleep quickly blithely regular deposits. boldly regular deposits wake blithely ironic accounts|
+7|4|9460|299.58|. furiously final ideas hinder slyly among the ironic, final packages. blithely ironic dependencies cajole pending requests: blithely even packa|
+8|9|6834|249.63|lly ironic accounts solve express, unusual theodolites. special packages use quickly. quickly fin|
+8|1|396|957.34|r accounts. furiously pending dolphins use even, regular platelets. final|
+8|3|9845|220.62|s against the fluffily special packages snooze slyly slyly regular p|
+8|5|8126|916.91|final accounts around the blithely special asymptotes wake carefully beyond the bold dugouts. regular ideas haggle furiously after|
+9|10|7054|84.20|ts boost. evenly regular packages haggle after the quickly careful accounts. |
+9|2|7542|811.84|ate after the final pinto beans. express requests cajole express packages. carefully bold ideas haggle furiously. blithely express accounts eat carefully among the evenly busy accounts. carefully un|
+9|4|9583|381.31|d foxes. final, even braids sleep slyly slyly regular ideas. unusual ideas above|
+9|6|3063|291.84| the blithely ironic instructions. blithely express theodolites nag furiously. carefully bold requests shall have to use slyly pending requests. carefully regular instr|
+10|1|2952|996.12| bold foxes wake quickly even, final asymptotes. blithely even depe|
+10|3|3335|673.27|s theodolites haggle according to the fluffily unusual instructions. silent realms nag carefully ironic theodolites. furiously unusual instructions would detect fu|
+10|5|5691|164.00|r, silent instructions sleep slyly regular pinto beans. furiously unusual gifts use. silently ironic theodolites cajole final deposits! express dugouts are furiously. packages sleep |
+10|7|841|374.02|refully above the ironic packages. quickly regular packages haggle foxes. blithely ironic deposits a|
+11|2|4540|709.87|thely across the blithely unusual requests. slyly regular instructions wake slyly ironic theodolites. requests haggle blithely above the blithely brave p|
+11|5|4729|894.90|ters wake. sometimes bold packages cajole sometimes blithely final instructions. carefully ironic foxes after the furiously unusual foxes cajole carefully acr|
+11|8|3708|818.74|inal accounts nag quickly slyly special frays; bold, final theodolites play slyly after the furiously pending packages. f|
+11|1|3213|471.98|nusual, regular requests use carefully. slyly final packages haggle quickly. slyly express packages impress blithely across the blithely regular ideas. regular depe|
+12|3|3610|659.73|jole bold theodolites. final packages haggle! carefully regular deposits play furiously among the special ideas. quickly ironic packages detect quickly carefully final|
+12|6|7606|332.81|luffily regular courts engage carefully special realms. regular accounts across the blithely special pinto beans use carefully at the silent request|
+12|9|824|337.06|es are unusual deposits. fluffily even deposits across the blithely final theodolites doubt across the unusual accounts. regular, |
+12|2|5454|901.70|s across the carefully regular courts haggle fluffily among the even theodolites. blithely final platelets x-ray even ideas. fluffily express pinto beans sleep slyly. carefully even a|
+13|4|612|169.44|s. furiously even asymptotes use slyly blithely express foxes. pending courts integrate blithely among the ironic requests! blithely pending deposits integrate slyly furiously final packa|
+13|7|7268|862.70|s sleep slyly packages. final theodolites to the express packages haggle quic|
+13|10|864|38.64|s after the slyly pending instructions haggle even, express requests. permanently regular pinto beans are. slyly pending req|
+13|3|9736|327.18|tect after the express instructions. furiously silent ideas sleep blithely special ideas. attainments sleep furiously. carefully bold requests ab|
+14|5|5278|650.07|e quickly among the furiously ironic accounts. special, final sheaves against the|
+14|8|5334|889.50|ss dependencies are furiously silent excuses. blithely ironic pinto beans affix quickly according to the slyly ironic asymptotes. final packag|
+14|1|3676|893.39|sits are according to the fluffily silent asymptotes. final ideas are slyly above the regular instructions. furiousl|
+14|4|4947|310.13| final deposits boost slyly regular packages; carefully pending theodolites |
+15|6|7047|835.70|blithely quick requests sleep carefully fluffily regular pinto beans. ironic pinto beans around the slyly regular foxe|
+15|9|3336|784.55|slyly. fluffily bold accounts cajole furiously. furiously regular dependencies wak|
+15|2|3316|265.89|e express instructions. ironic requests haggle fluffily along the carefully even packages. furiously final acco|
+15|5|5255|458.67|refully bold instructions among the silent grouches must boost against the express deposits:|
+16|7|5282|709.16|lithely ironic theodolites should have to are furiously-- |
+16|10|9412|887.53|ly special accounts wake. fluffily bold ideas believe blith|
+16|3|854|781.91| unusual excuses. requests after the carefully regular pinto |
+16|6|1491|918.51|unts cajole furiously across the fluffily pending instructions. slyly special accounts could have to boost b|
+17|8|8555|995.35|are furiously final accounts. carefully unusual accounts snooze across the requests. carefully special dolphins|
+17|1|7737|648.75|e blithely express accounts. foxes kindle slyly unusual dinos. quickly special f|
+17|4|3123|555.04|ly bold accounts. regular packages use silently. quickly unusual sentiments around the quickly ironic theodolites haggle furiously pending requests. care|
+17|7|3203|64.40|bold packages nag fluffily after the regular accounts. furiously ironic asymptotes sleep quickly enticing pinto beans. carefully pending accounts use about the |
+18|9|1125|664.17|. ironic, regular accounts across the furiously express |
+18|2|8132|52.44| final packages wake quickly across the blithely ironic instructions. regular pains integrate slyly across the deposits. carefully regular pinto beans among the close|
+18|5|3133|568.61|riously bold accounts. packages boost daringly. blithely regular requests cajole. regular foxes wake carefully final accounts. blithely unusual excuses det|
+18|8|6475|386.29|. furiously regular accounts cajole slyly across the pending|
+19|10|1416|144.80|o beans. even packages nag boldly according to the bold, special deposits. ironic packages after the pinto beans nag above the quickly ironic requests. bl|
+19|3|5467|405.70|nstructions use furiously. fluffily regular excuses wake. slyly special grouches are carefully regular Tiresias. regular requests use about the quickly furio|
+19|6|8800|635.66|sual requests sleep carefully. deposits cajole carefully over the regular, regular requests. quickly unusual asymptotes use some|
+19|9|1340|346.92| requests. final, pending realms use carefully; slyly dogged foxes impress fluffily above the blithely regular deposits. ironic, regular courts wake carefully. bold requests impress|
+20|1|2927|675.54|s, ironic deposits haggle across the quickly bold asymptotes. express, ironic pinto beans wake carefully enticingly special foxes. requests are at the c|
+20|4|2723|305.84|nal, bold frets cajole slyly regular, unusual platelets. slyly permanent deposits wake carefully carefully silent accounts. even, even requests wake quickly. furiously pending packages are|
+20|7|5905|546.66|ing deposits use furiously. ironically final pinto bea|
+20|10|4271|115.89|xcuses wake at the deposits. regular pinto beans nag slyly fluffi|
+21|2|6571|944.44|ing instructions impress bold foxes. ironic pinto beans use. thinly even asymptotes cajole ironic packages. quickly ironic pinto beans detect slyly regular deposits. ruthlessly even deposits are. sl|
+21|6|1704|139.05|posits cajole; quickly even requests sleep furiously. ironic theodolites sleep pending, express instructions. stealthily even platelets cajole carefully after the final, ironic p|
+21|10|7153|664.50|blithely enticing instructions use alongside of the carefully thin deposits. blithely bold requests are fluffily|
+21|4|367|584.86|ong the even theodolites. pending, pending accounts sleep-- courts boost quickly at the accounts. quickly fin|
+22|3|4410|786.18|even accounts. final excuses try to sleep regular, even packages. carefully express dolphins cajole; furiously special pinto bea|
+22|7|9779|635.84|l instructions cajole across the blithely special deposits. blithely pending accounts use thinly slyly final requests. instructions haggle. pinto beans sleep along the slyly pen|
+22|1|7834|359.16|sits wake fluffily carefully stealthy accounts. furiously ironic requests x-ray fluffily alongside of the pending asymptotes. slyly silent packages use along the instructions. fu|
+22|5|1434|597.21|ix across the blithely express packages. carefully regular pinto beans boost across the special, pending d|
+23|4|2739|460.12|platelets against the furiously bold Tiresias dazzle quickly into the special, bold courts. silent, regular instructions wake blithely ironic multipliers. ideas|
+23|8|5739|103.13| theodolites need to nag blithely final notornis. slyly idle packages cajole after the furiously stealthy packages. slyly regular accounts use furiously. carefully final accounts affix |
+23|2|9898|233.94|l, express packages wake permanently. quickly even deposits sleep quickly slyly silent id|
+23|6|7035|51.75|xcuses; decoys wake after the pending packages. final instructions are furi|
+24|5|5180|905.41|heodolites above the ironic requests poach fluffily carefully unusual pinto beans. even packages acc|
+24|9|2227|511.20|, silent packages boost around the instructions. special requests sleep slyly against the slyly regular deposits. final, final accounts haggle fluffily among the final requests. regular |
+24|3|7182|582.03| the final, ironic asymptotes. regular requests nag instead of the carefully unusual asymptotes. furiously pending attainments among the slyly final packages boost after th|
+24|7|5318|62.15| careful requests cajole blithely realms. special asymptotes sleep. pinto beans sleep carefully furiously ironic packages. furiously |
+25|6|9029|832.74|fully fluffily regular frets. sometimes even requests after the requests wake slyly at the quickly ruthless requests. a|
+25|10|9062|928.96|he foxes. final, final accounts sleep. boldly ironic excuses thrash quick|
+25|4|9946|694.35|ld, ironic requests. furiously special packages cajole furiously enticing instructions.|
+25|8|7340|746.59|dly final packages haggle blithely according to the pending packages. slyly regula|
+26|7|5020|683.96|es. fluffily express deposits kindle slyly accounts. slyly ironic requests wake blithely bold ideas|
+26|1|6577|892.20|riously pending pinto beans. furiously express instructions detect slyly according to the b|
+26|5|3499|382.11|imes even pinto beans among the busily ironic accounts doubt blithely quickly final courts. furiously fluffy packages despite the carefully even plate|
+26|9|9702|821.89| behind the blithely regular courts impress after the silent sheaves. bravely final ideas haggle |
+27|8|2111|444.01|the even, ironic deposits. theodolites along the ironic, final dolphins cajole slyly quickly bold asymptotes. furiously regular theodolites integrate furiously furiously bold requests. carefully|
+27|2|9080|157.03|ole express, final requests. carefully regular packages lose about the regular pinto beans. blithely re|
+27|6|3407|151.34|ironic theodolites are by the furiously bold ideas. ironic requests shall have to sublate final packages. furiously quick foxes alongside of the express, special deposits was boldly according |
+27|10|4283|348.61|ound the final foxes detect furiously across the even warhorses. quickly t|
+28|9|6643|204.86|y ironic deposits above the slyly final deposits sleep furiously above the final deposits. quickly even i|
+28|3|2452|744.57|ully regular theodolites haggle about the blithely pending packages. carefully ironic sentiments use quickly around the blithely silent requests. slyly ironic frays bo|
+28|7|302|690.30|uickly unusual requests alongside of the final courts integrate slyly |
+28|1|9988|666.53|beans haggle carefully around the slyly ironic acco|
+29|10|3506|799.27|leep fluffily according to the quietly regular requests: accounts integrate carefully bold foxes. carefully silent|
+29|4|8106|981.33|the ironic, bold asymptotes! blithely regular packages hang furiously above the dependencies. blithely permanent dependencies are furiously furiously ironic acco|
+29|8|9193|734.44|ly unusual packages. foxes cajole. theodolites nag|
+29|2|6252|186.21|thely carefully even packages. even, final packages cajole after the quickly bold accounts. fluffily quick accounts in place of the theodolites doze slyly f|
+30|1|4767|989.05|ts. slyly final pinto beans cajole ironic accounts. blithely final accounts use among the request|
+30|5|535|743.26|sual instructions wake carefully blithely even hockey playe|
+30|9|7756|568.86| special foxes across the dependencies cajole quickly against the slyly express packages! furiously unusual pinto beans boost blithely ironic Tir|
+30|3|7945|583.84| sleep. bold, regular deposits hang doggedly furiously bold requests. slyly bold excuses detect busily above the even gifts. blithely express courts are carefully. blithely final packages until th|
+31|2|9685|620.84|he blithely regular ideas. blithely unusual requests haggle fluffily. platelets|
+31|7|1951|120.99|refully regular pinto beans. ironic requests integrate furiously since the quickly ruthless platelets. quickly ironic attainments ha|
+31|2|1402|761.64|r platelets nag blithely regular deposits. ironic, bold requests |
+31|7|137|849.11|blithely ironic accounts. slyly ironic asymptotes sleep ironic, even accounts. regular accounts thrash quickly|
+32|3|2203|406.03|es? slyly enticing dugouts haggle carefully. regular packages alongside of the asymptotes are carefull|
+32|8|467|109.34|ainst the unusual braids nod fluffily packages. regular packages nod among the slyly express|
+32|3|7975|747.14|final foxes boost furiously pending packages. quickly regular depths promise blithely accoun|
+32|8|7938|856.09|s integrate according to the even dependencies. carefully regular reque|
+33|4|4028|891.46|, pending requests affix slyly. slyly ironic deposits wake accounts. express accounts sleep slowly. ironic, express accounts run carefully fluffily final dependencies. furiously unusual ideas|
+33|9|4410|929.05| packages sleep carefully. slyly final instructions boost. slyly even requests among the carefully pending platelets wake along the final accounts. quickly expre|
+33|4|1287|310.76|dolites above the slyly express deposits try to haggle blithely special gifts. blithely ironic reque|
+33|9|6006|327.19|ly. ironic dependencies haggle carefully silent instructions. furiously ironic dolphins are fluffily furiously even theo|
+34|5|9934|848.75|ven instructions besides the gifts are furiously among the slyly regular packages! instructions use carefully. even requests sleep quickl|
+34|10|4749|265.31|ckly regular theodolites eat above the bravely regular courts. ironic requests wake slyly.|
+34|5|5459|824.69|ong the slyly silent requests. express, even requests haggle slyly|
+34|10|5884|609.69|ully final tithes. slyly ironic deposits hang furiously about the regular, regular deposits|
+35|6|2500|451.58|nic packages boost carefully carefully even theodolites. blithely fina|
+35|1|8875|537.72|ully regular deposits: special accounts use. slyly final deposits wake slyly unusual, special ideas. asymptotes |
+35|6|596|669.19|slyly against the daring, pending accounts. fluffily special pinto beans integrate slyly after the carefully unusual packages. slyly bold accounts besides|
+35|1|2025|411.17|s cajole fluffily final deposits. furiously express packages after the blithely special realms boost evenly even requests. slow requests use above the unusual accoun|
+36|7|3907|630.91|al deposits detect fluffily fluffily unusual sauternes. carefully regular requests against the car|
+36|2|174|434.47|permanently express instructions. unusual accounts nag toward the accou|
+36|7|2625|569.91|ctions. pending requests are fluffily across the furiously regular notornis. unusu|
+36|2|8209|289.15|arefully regular requests cajole. special, express foxes sleep slowly. quickly unusual in|
+37|8|7171|824.96|usly into the slyly final requests. ironic accounts are furiously furiously ironic i|
+37|3|5542|126.59|ven deposits. ironic foxes cajole. slyly final deposits are furiously after the furiously even packages. slyly ironic platelets toward the slyl|
+37|8|7113|15.72|re bravely along the furiously express requests. blithely special asymptotes are quickly. fluffily regular packages alo|
+37|3|1449|745.64|y after the ironic accounts. blithely final instructions affix blithely. bold packages sleep carefully regular instructions. regular packages affix carefully. stealthy fo|
+38|9|1226|570.11| slyly even pinto beans. blithely special requests nag slyly about the ironic packages. |
+38|4|4237|662.75|lar warhorses cajole evenly against the attainments. requests cajole furiously furiously express requests. carefully regular platelets use fluffily after the silent, unusual ideas: bl|
+38|9|1135|160.70|express accounts haggle. carefully even pinto beans according to the slyly final foxes nag slyly about the enticingly express dol|
+38|4|3516|847.09|nal accounts. furiously pending hockey players solve slyly after the furiously final dependencies. deposits are blithely. carefully regular packages unwind busily at the deposits. fluffily |
+39|10|3633|463.10|kages are slyly above the slyly pending pinto beans. bold, ironic pinto beans sleep against the blithely regular requests. fluffily even pinto beans use. regular theodolites haggle against the quic|
+39|5|3682|300.43|ng requests are according to the packages. regular packages boost quickly. express Tiresias sleep silently across the even, regular ideas! blithely iro|
+39|10|5475|532.26| beans cajole carefully carefully express requests. instructions sleep furiously bold deposits. furiously regular depos|
+39|5|6259|737.86|y. special, even asymptotes cajole carefully ironic accounts. regular, final pinto beans cajole quickly. regular requests use warhorses. special, special accounts hinder boldly across the|
+40|1|7690|776.13|lets use fluffily carefully final deposits. blithely ironic instructions sublate against the furiously final ideas; slyly bold courts x-ray silent foxes. regular foxes wake blithely. slyl|
+40|6|1704|565.82|riously furiously silent asymptotes. final deposits cajole blithely ironic requests. furiously special pains into the blithely final instru|
+40|1|4521|374.71|ptotes haggle. slyly even requests nag fluffily silent packages. blith|
+40|6|6617|196.64|he slyly unusual epitaphs? ironic deposits at the furiously unusual instructions thrash blithely requests. requests are carefully blithely pending waters.|
+41|2|9040|488.55|ss the dinos wake along the blithely regular theodolites. foxes cajole quickly ironic, final foxes. blithely ironic packages haggle against |
+41|8|5946|391.81| slyly slyly regular requests. final deposits sleep fluffily. blithely bold instructions detect carefully. blithely pending requests are furiously ironically final ideas. regul|
+41|4|1550|916.55| the blithely final ideas. furiously regular asymptotes could cajole furious|
+41|10|560|37.59|special pinto beans against the unusual accounts cajole slyly final foxes. close, ironic|
+42|3|2893|716.81|requests nag. furiously brave packages boost at the furiously even waters. slyly pending ideas nag carefully caref|
+42|9|2927|709.06|g dugouts. carefully careful ideas are fluffily. carefully final pinto beans snooze. ironic deposits wake evenly along |
+42|5|3500|200.00|against the ironic, ironic forges. slyly final deposits wake blithely. ironic courts sleep furiously ab|
+42|1|3662|29.46|es sleep slyly among the slyly final requests. bold theodolites use silently against the final foxes. carefully pending requests use furiously. dogged, unusual asymptotes use |
+43|4|3211|805.78|gular accounts. bold theodolites nag slyly. quickly express excuses use blithely. blithely even ideas boost fluffily! blithely unusual ideas detect bli|
+43|10|6770|493.19|ing to the quickly even theodolites. quickly bold excuses haggle. sometimes unusua|
+43|6|9506|493.65|riously! slyly ironic sauternes affix. ironic theodolites sleep furiously about the express packages. slyly ironic deposits are blithely against the regular package|
+43|2|3232|307.12|counts: express, final platelets use slyly bold ideas. ironic theodolites about the blithely s|
+44|5|486|164.22| final notornis throughout the unusual pinto beans are about the special accounts. bold packages sleep fluffily above the|
+44|1|5310|114.37|quests. quickly unusual requests against the carefully final somas detect slyly bold a|
+44|7|3534|383.01|r the pending pinto beans! requests wake furiously after the special deposits. silent deposits mold quickly along the express, special |
+44|3|4798|833.15| run. ironic, special dolphins according to the even, ironic deposits haggle carefully alongside of the carefully regular excuses. regular frays haggle carefully ironic dependenc|
+45|6|1685|919.63|he doggedly final accounts; carefully regular packages cajole idly regular idea|
+45|2|5202|877.29|ngage blithely after the final requests. bold accounts sleep blithely blithely express dependencies. pinto beans through the carefully regular hockey players wake|
+45|8|5669|532.70|es play carefully doggedly unusual requests. bold grouches against the furiously ironic dugouts sleep furiously qu|
+45|4|1872|155.32| ironic, even pinto beans. bold theodolites haggle after the furiously ironic accounts. slyly bold courts|
+46|7|4171|244.65|lly quiet instructions. furiously express requests among the final ideas cajole carefully bold waters. furiously regular pac|
+46|3|8518|106.80|e unusual instructions shall have to detect slyly blithely ironic foxes. bold requests impress silent foxes. ironic, quiet realms haggle quickly pending, express pinto be|
+46|9|7225|14.78|ously about the fluffily pending accounts. fluffily even dugouts are quickly slyly express platelets; quickly bold pearls sleep slyly even instructions. furiously ironic packages poach quic|
+46|5|1381|985.88|ending platelets are carefully regular accounts. fluffily even accounts against the dependencies nag carefully final, |
+47|8|6989|292.52|even ideas. blithely final requests boost blithely. final, ironic instruct|
+47|4|4458|539.47|; finally enticing theodolites cajole enticing, silent warhorses! slyly bold pains c|
+47|10|2896|74.54|grate final asymptotes. pending requests kindle carefully final frets. ironic deposits above the slyly e|
+47|6|5873|296.63|after the regular dependencies. final, bold pains sleep quickly pend|
+48|9|5052|611.16|posits are blithely blithely final foxes. blithely even deposits haggle fluffily express requests. furiously final theodolites use sl|
+48|5|9451|191.36|ckages cajole never even, special foxes. regular dependencies wake after the blithely ironic instructions. thinly ironic reque|
+48|1|5564|668.19|al pinto beans. furiously final frays use slyly according to the ironic theodolites. regular ideas cajole furiously after the slyly even deposits. |
+48|7|1719|606.16|forges lose. packages cajole regular, bold accounts. never ironic accounts may promise about the permanently bold deposits. always express requests cajole fluffily regular c|
+49|10|9056|35.11| bold deposits? final, bold pinto beans are furiously slyly regular packages. sly|
+49|6|6646|908.15|ts sleep across the fluffily final deposits. carefully express accounts around the regular, express excuses x-ray inside the ironic theodolites. expre|
+49|2|5336|713.25|ld accounts. furiously blithe waters use furiously blithely idle dependencies. pending deposits along the permanently re|
+49|8|597|812.62|n foxes snooze furiously. courts integrate never. carefully unusual requests are carefully. quickly ironic deposits ha|
+50|1|1832|565.54|liers above the dolphins dazzle across the regular foxes. furiously regular packages haggle furiously blithely ironic grouches. ironic, even accounts haggle pending, furious instruction|
+50|7|43|690.87|aggle daringly along the close, express deposits. final requests snooze carefully carefully bold deposits. carefully unusual ideas doze furiously after the furious|
+50|3|6160|301.06|arefully ironic requests use. furiously pending waters play carefully carefully regular platelets. sly requests cajole furiously slyly regular pinto beans. bold packages boost fluffily. furiously i|
+50|9|2104|107.17|t blithely unusual theodolites. quickly final accounts affix fluffily regular requests. c|
+51|2|837|310.74|ly dogged, regular dependencies. express, even packages are |
+51|9|7318|85.03|al foxes. carefully ironic accounts detect carefully-- slyly even accounts use. furiously final platelets shall haggle sometimes after the blithely regu|
+51|6|138|728.95|requests according to the carefully unusual deposits promise slyly ironic packages. slyly ironic dependencies are accordin|
+51|3|8062|901.04|le ruthlessly furiously slow requests. fluffily slow depende|
+52|3|6533|54.92|efully. slyly special deposits haggle along the quick deposits. slyly pending requests use quickly packages. final, final dolphins doubt according to the quickly unusual excuses|
+52|10|1937|210.44|s. never even asymptotes nag carefully! regularly unusual foxes along the unusual requests haggle accounts. fluffily express pinto |
+52|7|4084|628.53| deposits wake slyly pending asymptotes. ironic asymptotes haggle. blithely ironic requests are qui|
+52|4|5524|424.93|cial, ironic packages. even dolphins boost. slyly final deposits integrate. final sheaves along the silent excuses use at the slyly close foxes; bold accounts are finally even packages. ironi|
+53|4|6443|192.78|carefully ironic accounts. blithely bold deposits detect furiously against the flu|
+53|1|5319|563.44|ly. fluffily final pearls boost carefully. special sauternes nod furiously even instructions. carefully regular dependencies across the slyly regular deposits|
+53|8|8200|388.08|fully requests. furiously final accounts cajole express, regular pearls. special deposits wake fluffily express accounts. quic|
+53|5|6929|224.83|xes. carefully ruthless asymptotes impress slyly. fluffily final deposits sleep against the ideas. slyly final packages wake. pending, express packages sleep quickly.|
+54|5|2515|686.51|ly along the packages. blithely close pinto beans are blithely alongside of the unusual packages. carefully even platelets boost alongside of the even foxes. ironic de|
+54|2|7079|798.98|he carefully unusual packages wake according to the ironic dolphins. permanently regular sheaves nag quickly. regular, ironic|
+54|9|2386|23.78|kly ironic foxes. final instructions hinder doggedly. carefull|
+54|6|536|259.24| furiously along the fluffily regular requests. carefully unusual accounts use fluffily final platelets. pending deposits integrate furiou|
+55|6|7874|611.04|ly special packages. furiously even warhorses integrate. silen|
+55|3|8460|236.27|round the special, bold asymptotes cajole alongside of the instructions. qui|
+55|10|8278|134.62|gedly silent pinto beans! furiously regular sentiments was furiously across the silent pinto beans. pending warthogs along the slyly |
+55|7|1289|130.33|ut the blithely final requests. requests nag blithely. |
+56|7|241|855.39|nto beans. finally regular sauternes are. carefully bold deposits according to the blithely express requests wake carefully ironic excuses? furiously final deposit|
+56|4|9104|54.79|tructions above the blithely pending foxes cajole blithely furiously even sentiments. special, exp|
+56|1|1330|52.29|xpress instructions haggle furiously regular deposits. quickly unusual packages sleep furiously final pinto|
+56|8|5799|926.25|ades grow around the dependencies. carefully special ideas cajole furiously across the blithely express requests. unusual tithes are caref|
+57|8|2972|123.11| asymptotes use carefully furiously final deposits. quickly regular deposits are furiously slyly ironic requests. blithely even excuses haggle: blithely special ideas|
+57|5|4721|411.08|instructions. quickly unusual deposits about the furiously special ideas believe among the furiously bold theodolites. unusual, even ideas nag: slow, special theodolites hagg|
+57|2|3788|211.66|ly according to the ironic requests-- slyly final accounts print carefully depths? pending, unusual accounts solve |
+57|9|4583|137.68|ts. blithely bold theodolites can boost carefully carefully even instr|
+58|9|4328|542.52|ven deposits wake requests. quickly bold platelets sleep furiously after the ironic requests. even accounts haggle quickly bold |
+58|6|4307|448.31|quickly carefully ironic foxes. bold platelets nag furiously regular packages. slyly specia|
+58|3|4136|512.24|packages cajole slyly quickly pending depths. special, bold realms cajole slyly. slyly ir|
+58|10|9689|25.09|long the unusual, express asymptotes. ironic ideas boost bold, special deposits? ironic foxes among the fin|
+59|10|8374|357.22|c decoys. carefully even pinto beans wake slyly alongside of the express accounts. regular grouches haggle.|
+59|7|4226|80.98|lar packages. regular depths use slyly after the fluffily regular packages; theodolites around the furiously ironic asy|
+59|4|99|598.55|he special pinto beans. fluffily even accounts cajole. fluffily regular foxes haggle among the|
+59|1|8184|45.50|ependencies. ironic dependencies wake carefully according to the blithely bold packages. quickly unusual ideas about th|
+60|1|6642|800.72| blithely. slyly final realms alongside of the excuses use quickly blithely bold foxes. final theodolites are slyly after the slyly regular excuses. never thin foxes about |
+60|8|5017|314.81| even pinto beans wake carefully. quickly regular deposits hinder along the furiously regular pack|
+60|5|148|504.10|s use fluffily. furiously regular deposits boost furiously against the even instructions. blithely final platelets wake. carefully pending asymptotes sleep blithely. regular, s|
+60|2|5792|92.64|s the carefully pending deposits. slyly regular pinto beans against the furiously regular grouches lose carefully around the enticingly final ideas. furiously express packages cajole bold pa|
+61|2|1540|858.64| could have to use upon the packages. fluffily special packages integrate slyly final theodolites. pending warhorses wake quickly after the blithely final fo|
+61|10|9170|771.26|ly. pinto beans sleep blithely about the patterns. slyly final accounts wake according to the furiously bold requests. slyly regular packages wake according to the ironic packages. requests acros|
+61|8|4762|633.74|final theodolites haggle. fluffily express ideas about the silent theodolites cajole ideas; fluffily special instructions are accordin|
+61|6|7312|153.74|gly final instructions. pending theodolites will wake furiously. slyly bold instructions run. furiously special foxes cajole f|
+62|3|1780|692.42|s around the even ideas cajole furiously somas. silent asym|
+62|1|5896|348.82| final accounts. furious deposits wake slyly. idly regular packages haggle blithely pending grouches. ironic accounts boost blithely. carefully express pa|
+62|9|9127|620.08|totes. unusual requests after the unusual accounts sleep fluffily bold notornis. slowly careful requests use according to the final ideas. pinto beans sleep. foxes are furiously furiously pe|
+62|7|9542|255.78|lly express requests haggle carefully. idle, pending pinto beans are furiously regular excuses. quickly sly attainments are furiously; even accounts are slyly quickl|
+63|4|1804|498.84|leep bravely. final accounts nag. forges sleep against the slyly ironic pa|
+63|2|1998|509.16|yly express theodolites. slyly bold ideas sleep furiously accordi|
+63|10|6839|274.15| among the carefully ironic accounts. carefully even accounts against the regular, final deposits detec|
+63|8|6325|463.69|arly express accounts. express, unusual escapades haggle. special packages must wake. express, regular requests sleep furiously ironic packages|
+64|5|5567|228.61|y even instructions. unusual requests serve slyly. special foxes sleep quickly. fluffily ir|
+64|3|4542|398.92|. quickly final ideas cajole carefully among the blithely silent requests. sometimes ironic accounts nag furiously against the pending instructions. f|
+64|1|9110|602.65| ironic accounts are carefully carefully final accounts. slyly ironic packa|
+64|9|2064|25.77| quickly regular ideas. carefully final requests snooze carefully regular, regular instructions. stealthily final pi|
+65|6|2918|846.26|inal, even foxes cajole. furiously final dolphins hang quickly ironic foxes. furiously special packages alongside of the bold foxes solve above the carefully final instructio|
+65|4|1779|393.63|ully after the quickly regular ideas. ironic, final multipliers above the carefully bold deposits breach slyly furiously express deposits. unusual accounts haggle carefully idea|
+65|2|2054|503.10|e express excuses. ironic, even accounts across the reg|
+65|10|2188|288.73|lent requests nag quickly. blithely silent platelets haggle ironic accounts. slyly bold instructions boost carefully final accounts. carefully even dependencies must nag blithely; qui|
+66|7|3077|809.13|nod carefully besides the furiously final theodolites. slyly final requests haggle. furiously silent excuses detect quickly. ironic deposits detect above the furiously final |
+66|5|1076|785.75|its across the blithely regular theodolites wake furiously among the furiously regular accounts. pains are slyly care|
+66|3|2568|447.08|ously even accounts boost slyly daring requests. even, regular realms kindle blithely. unusual, ironic ins|
+66|1|296|797.27|s nag enticingly outside the furiously final foxes. final accounts haggle fluffily accord|
+67|8|9923|306.37|ly according to the quickly ironic requests. express instructions after the slyly even instructions x-ray blith|
+67|6|7908|546.75|furiously express dolphins integrate carefully regular notor|
+67|4|3368|625.62|le slyly regular requests: regular platelets wake quickly across the quickly regular accounts. reg|
+67|2|5826|397.34|en, ironic deposits affix quickly unusual requests. busily ironic accounts are finally never even sauternes. ironic depos|
+68|9|3444|31.37|es impress furiously pending packages. always silent instructions above the fluffily bold packages haggle slyly blit|
+68|7|6762|5.16|lithely. carefully even grouches along the bold deposits might sleep slyly requests. blithel|
+68|5|8300|80.86|nooze according to the furiously even ideas. blithely regular accounts wake blithely. furiously regular Tiresias cajole regular deposits. regular theodolites eat alongside of the|
+68|3|5399|683.59|. finally final pinto beans play carefully unusual requests. never pending accounts are. regular, final theodolites wake furiously excuses. special request|
+69|10|6197|694.24|eep across the packages. regular, final foxes boost fluffily regular pinto beans. packages sleep along the final requests. bold, unusual packages cajo|
+69|8|8235|846.49|nt fluffily. carefully ironic instructions wake. blithely express foxes cajole slyly. unusual requests sleep quickly. final packages affix slyly according to the spec|
+69|6|9294|386.96|ar packages. blithely regular dependencies are dolphins. slyly ironic excuses nag quickly pending, regular ideas. furiously special sheaves haggle. close, regular pinto beans about the slyly bold|
+69|4|7017|344.28|heodolites. unusual, regular requests boost slyly pending deposits. slyly daring instruct|
+70|1|4536|348.27|ructions. blithely final packages cajole carefully after the express, even requests. furiously final theodolites cajole |
+70|9|8063|452.80|y regular deposits nag about the carefully regular instructions; furiously express accounts along the final, express instruct|
+70|7|2990|940.81|s deposits. unusual foxes are carefully according to the carefully even deposits. carefully ironic foxes cajole fluffily against the carefully pending deposits. slyly special depo|
+70|5|9074|182.58|ions after the fluffily regular foxes wake above the furiously regular requests: slyly regular deposits wake slyly daringly even Tiresias. express, express deposits are. always unusual pa|
+71|2|508|842.21|es cajole carefully around the furiously pending instructions. |
+71|1|8329|239.57|ins sleep carefully slyly express accounts! quickly even accounts boost carefully about the carefully regular excuses. dogged, even dolphins against the sometimes ironic packages believe bl|
+71|10|6768|744.67|ructions. daring requests solve carefully about the furiously pending pinto|
+71|9|5179|329.13|usly at the packages. blithely regular deposits haggle regular packages. quickly special theodolites at the blithely ironic instructions wake|
+72|3|9855|497.26|tithes. quickly pending foxes haggle enticingly according to the accounts. accounts detect slyly: final packages wake. fina|
+72|2|9346|41.04| pending instructions before the even, silent dep|
+72|1|2654|762.61|nusual packages: blithely bold Tiresias sleep furiously. slyly brave accounts according to the final, |
+72|10|4526|154.47|use across the never ironic packages. express, regular accounts above the pending, fluffy deposits are carefully across the slyly even pinto be|
+73|4|9873|947.99|tes use pending packages. final foxes wake final, unusual packages. blithely blithe ideas haggle sometimes slyly express accounts. express instructions nag furiously quickly|
+73|3|7729|920.66|ecial accounts sleep according to the slyly sly accounts. slyly express instructions nag. accounts cajole furiously quickly even foxes. furiously regular requests wake. carefully even frets haggle |
+73|2|5327|108.96| beans are furiously between the regular ideas! unusual pinto beans use. furiously silent requests against the carefully even somas wake care|
+73|1|3928|309.57|longside of the blithely final ideas. carefully ironic courts sleep along the enticingly pending requests. fluffily regular accounts use fluffily bold ideas. slyly ironic packa|
+74|5|3128|345.92|ic theodolites. express deposits haggle blithely pending packages. quickly express foxes could are slyly. deposits sleep deposits. final dependencies sleep ab|
+74|4|2479|930.97|o beans sleep dependencies. regular accounts use blithely asymptotes. u|
+74|3|9473|496.36| haggle carefully alongside of the regular requests. slyly regular accounts belie|
+74|2|6234|849.66| slyly regular foxes. silent accounts integrate. even deposits are quick|
+75|6|7086|624.39|sits are furiously fluffily even courts. furiously pending requests are blithely. pending, regular accounts play carefully slyly unusual platelets. blithely final requests against the ru|
+75|5|6308|759.36|refully ironic dependencies. pinto beans use according to the packages. regular platelets wake around the blithely p|
+75|4|9080|433.59|sits. permanent packages breach. carefully final waters wake. bold, pending foxes haggle furiously evenly express instructions. even deposits about the final|
+75|3|5439|884.01|ding excuses snooze special accounts. tithes alongside of the regular dep|
+76|7|6754|494.83|gular accounts solve. ironic deposits sleep slyly even packages. slyly pending accounts detect slyly express accounts. ironic forges can play furiously carefully express fox|
+76|6|2009|108.97|n packages. blithely even accounts sleep carefully furiously ironic accounts. carefully express requests|
+76|5|6371|552.38|ts use against the quickly ironic ideas. quickly even deposits are carefully a|
+76|4|7986|252.03| packages across the furiously ironic platelets cajole across the regular, ironic accounts. carefully enticing accounts among the blithely regular instructions detect regular pinto be|
+77|8|552|254.92|e after the carefully pending packages. carefully even dependencies cajole pending |
+77|7|8170|875.83|xcuses. blithely even foxes use fluffily. blithely even requests use. slyl|
+77|6|8541|936.13|e slyly express instructions haggle about the sometimes regula|
+77|5|1713|402.14|the even ideas kindle after the requests. regular theodolites cajole carefully about the blithely final ideas. carefully even dependencies at the flu|
+78|9|9915|729.94|around the special excuses. furiously even deposits serve boldly according to the platelets. carefully express accounts at the blithely unusual pinto beans sleep furiously against the u|
+78|8|7246|577.23|regular dependencies cajole doggedly ironic accounts. bold theodolites doze about the accounts. quickly final requests boost slyly final asymptotes. carefully final dolphins ha|
+78|7|1801|434.34|nts kindle furiously according to the even packages. blithely ironic platelets are slyly silent foxes. final, final packages would sleep. pinto beans a|
+78|6|9599|382.82| carefully special theodolites cajole among the quickly even asymptotes. foxes wake blithely across the carefully |
+79|10|4248|765.34|nusual, express asymptotes wake furiously. ironic pinto beans detect above the carefully express theodolites: even, dogged instructions nag. spe|
+79|9|465|28.33|uriously special frays cajole across the finally ironic pinto beans. ironic accounts sleep blithely. fluffily silent accounts are slyly at the slyly unusual ideas. even deposits nag slyly |
+79|8|3309|880.23|tect final, thin accounts? furiously ironic accounts boost regular deposits. carefully ironic attainments sleep. furiously special ins|
+79|7|8627|891.18|r dolphins grow blithely against the slyly ironic packages. deposits about the regular, ironic decoys are slyly around the carefully regular packages. slyly pending excuses sle|
+80|1|8893|127.65|ld accounts detect carefully. carefully bold courts along the regular deposits could have to affix ca|
+80|10|2243|775.79|endencies. bold, regular pinto beans wake furiously above|
+80|9|5385|945.72|cial asymptotes believe after the blithely unusual deposits. furiously silent pinto beans cajole quickly inside the slyly even deposits. regular, f|
+80|8|4034|797.05|ptotes cajole carefully. express ideas cajole carefully even somas. final pinto beans print fluffily across the |
+81|2|1605|550.29|es haggle blithely fluffily final requests. furiously regular foxes use. furiously unusual requests outside the furiously regular requests|
+81|2|5923|220.23|the final, quick accounts are blithely above the s|
+81|2|2942|409.73|accounts boost. fluffily unusual requests cajole fluffily slyly ironic requests. foxes cajole quick|
+81|2|58|492.19| instructions boost furiously across the foxes-- final depo|
+82|3|7793|697.31|he accounts cajole quickly after the even patterns. ironic platelets sublate regular, even asymptotes. quick courts affix according to|
+82|3|7698|585.86|pinto beans. slyly express excuses haggle. blithely even pinto beans about the quick inst|
+82|3|8268|604.25|e after the carefully even theodolites. regular, pending accounts boost. quickly final asymptotes haggle slyly. requests use final, bold pinto beans. bold, ruthle|
+82|3|5532|900.07| slyly? fluffily special dependencies haggle among the slyly special requests. regular, bold packages after the blithely ironic packages are slyly ironic packages. slyly final deposits w|
+83|4|3010|745.51|l foxes along the bold, regular packages integrate carefully express courts! final excuses sleep carefully ironic|
+83|4|8200|399.64|y final platelets are carefully carefully special platelets. carefully ironic requests wake blithely alongside of the slyly even accounts. bold, regular requests sleep |
+83|4|5974|657.22| even packages boost furiously. slyly regular gifts above the accounts are quickly express packages. slyly pending deposits besides the express, even asymptotes haggle after the ironic ins|
+83|4|3890|24.73|deposits. carefully even dependencies across the dependencies haggl|
+84|5|5711|233.61|arefully final platelets cajole blithely; quickly final accounts use furiously. furiously reg|
+84|5|208|469.80|carefully express dolphins nag about the slyly bold requests. slyly even packages wake among the furiously special attainments.|
+84|5|2909|969.44|silent requests cajole slowly bold ideas. special, special deposits according to the always silent packages are against the furiously silent packages. even, blithe accounts sleep slyly across |
+84|5|903|707.77|gly regular dependencies boost. slyly even accounts sleep. furiously final hockey players wake carefully with the reg|
+85|6|2628|608.77|xes wake furiously after the carefully even platelets. blithe theodolites are furi|
+85|6|118|917.83| against the even deposits. furiously bold ideas along the furious requ|
+85|6|2074|491.20|encies-- slyly regular requests about the quiet accounts detect quickly at the |
+85|6|8289|73.81|s cajole slyly along the slyly special accounts. regular, special deposits wake. furiously special foxes boost. blithely even packa|
+86|7|806|65.98|ackages. blithely pending accounts are slyly furiously pending theodolites. furiously eve|
+86|7|2773|250.04|ding accounts. slyly special requests will have to affix carefully along the furiously unusual packages. regular theodol|
+86|7|5546|816.53|s. slyly final requests wake. furious deposits must wake blithely among the blithely ironic instructions. special hockey players try to are bli|
+86|7|1418|332.65|press theodolites sleep carefully about the blithely unusual requests. quickly final deposits breach slyly |
+87|8|5679|688.33|t the carefully regular asymptotes. blithely stealthy pinto beans within the furiously expres|
+87|8|1272|435.42|ronic foxes sleep along the special foxes. final ideas wake quickly about the carefully special theodolites. blithely ironic packages are blithely. regular, regular pint|
+87|8|9041|617.20|furiously final deposits. furiously special dependencies solve across the regular, special ideas. carefully silent requests haggle furiously after the special, specia|
+87|8|1892|868.60|arhorses are. unusual requests use blithely furiously final ideas. final requests sleep theodoli|
+88|9|6116|334.58|ect furiously around the regular deposits. special, final platelets boost furiously. blithely unusu|
+88|9|395|71.50| the regular accounts-- furiously even accounts use quickly after the regular, regular deposits. furiously e|
+88|9|9979|81.82|f the regular, regular requests believe fluffily along the final, quiet decoys. furiously even accounts cajole. carefully express requests wake quickly among the ideas. quickly silent |
+88|9|276|821.43|gular pinto beans. slyly pending excuses breach blithely express accounts. thin deposits sleep slyly around the even accounts; fluffily busy patterns kindle. slyly final deposits along the |
+89|10|3430|744.87| integrate slyly dolphins. bold, final frets use beside the carefully even accounts. slyly close dependencies sleep quickly carefully final pinto beans. foxes promi|
+89|10|8599|776.53|ress packages use furiously. furiously regular packages thrash blithely about the slyly pe|
+89|10|7876|417.61|nstructions: furiously even requests are quietly unusual accounts. regular requests are after the blithely regular deposits. sl|
+89|10|924|920.02|ickly unusual asymptotes after the slyly unusual accounts are carefully doggedly ironic accounts. even, final accounts use furiousl|
+90|1|8037|409.38|eas. unusual, pending packages boost quietly final accounts. slyly final packages serve. slyly even instructions sleep carefully. quickly even foxes wake quickly. |
+90|1|9683|498.43| accounts! fluffily regular deposits x-ray about the unusual, final packages. furiously final deposits alongside of the caref|
+90|1|7849|666.13|carefully ironic accounts are around the slyly bold asymptotes. carefully regular packages use furiously. ironic platelets affix carefully final accounts-- fluffily final pinto beans across the fina|
+90|1|7629|50.84|onic requests wake fluffily unusual packages. furiously even frays after the daringly pending requests wake furiously alongside of the bold requests. fluffily ironic ideas nag. ironic,|
+91|2|7986|528.64|luffily final instructions. furiously unusual foxes haggle |
+91|3|3257|906.20|ackages cajole slyly. blithely bold deposits cajole. blithely |
+91|4|483|823.21|n: slyly ironic foxes nag blithely according to the furiously bold foxes. regular, regular accounts a|
+91|5|1265|703.41| quickly silent deposits use attainments. final requests along the carefully ironic accounts wake blithely about the carefully ironic excuses. furiously bold excuses wake final, final ex|
+92|3|9337|224.01| requests are slyly along the deposits. fluffy pains alongside of the deposits |
+92|4|2246|985.03|jole enticingly regular asymptotes. carefully unusual pinto beans nag carefully ironic ideas. quickly un|
+92|5|3199|91.63|ake carefully: carefully ironic requests sleep careful|
+92|6|1044|854.89|l instructions are fluffily silently regular accounts. quickly final dolphins w|
+93|4|3008|615.98|sits promise blithely fluffily special decoys. slyly regular packages along the slyly final deposits wake accord|
+93|5|5275|376.47|ounts boost fluffily along the thinly regular realms. busily regular a|
+93|6|3869|868.81|ly among the furiously silent accounts. closely regular pinto beans nag slyly! slyly e|
+93|7|7188|805.90|y furiously bold pinto beans. express asymptotes was quickly. carefully final accounts affix slyly! platelets according to the ca|
+94|5|5433|365.56| even excuses wake carefully. quickly unusual requests wake accounts. regularly pending packages are regular |
+94|6|7784|358.08|ironic packages wake slyly carefully regular accounts. quickly regular warhorses against the blithely ironic packages haggle doggedly sly|
+94|7|7232|478.94|y regular requests. carefully final asymptotes haggle carefully against the slyly unusual requests: blithely brave grouches are fu|
+94|8|3261|824.08|quests. enticingly final accounts sleep fluffily. quickly express asymptotes around th|
+95|6|5186|291.03|ites across the blithely pending theodolites do affix across the unusual, bold Tiresias. bold packages|
+95|7|6552|456.36|tes; final, final accounts boost blithely ironic pinto beans. blithely ironic deposits cajole above the quickly pending requests? i|
+95|8|367|987.22| express requests detect furiously. requests cajole carefully|
+95|9|7379|973.74| above the furiously unusual deposits haggle ironic ideas. express, even packages haggle slyly slyly special asymp|
+96|7|5739|202.06|re. slyly regular theodolites breach slyly even dinos. fluffily regular asymptotes haggle slyly. fluffily bold courts affix furiously. regular requests |
+96|8|4942|571.30|e carefully. bold packages sleep against the furiously express requests. express foxes above the dependencies use quickly according to the slyly expres|
+96|9|9985|672.29|ecial instructions-- blithely silent theodolites play. even, silent accounts sleep. blithely silent requests haggle final, f|
+96|10|7250|587.08|efully ironic foxes. regular, final pinto beans boost above the express a|
+97|8|6371|129.77|fluffily unusual accounts. slyly regular theodolites integrate furiou|
+97|9|2390|458.34| carefully unusual pinto beans; even deposits detect furiously|
+97|10|2618|239.34|al theodolites are daringly requests. warhorses sleep blithely requests. special accounts cajole slyly deposits. a|
+97|1|4580|761.41| beans. carefully final deposits alongside of the carefully final requests haggle idly blithely ironic accounts. foxes cajole slyly against the ironic, special packages. furiously brave excuses boo|
+98|9|9486|908.21|usly final deposits mold furiously above the even deposits. carefully ironic packages across the quickly regular dolphins are slyly according to the slyly even|
+98|10|8550|657.16| sleep carefully. bravely bold somas may sleep pendin|
+98|1|3443|139.00|gular pinto beans maintain quickly fluffily regular deposits. express requests sleep. even requests after the regu|
+98|2|3759|811.55|iously. final, express packages are across the ironic dependencies. slyly thin ideas according to the even Tiresias detect furiou|
+99|10|8487|438.38|lphins affix ironic packages. blithely ironic requests nag fluffily after the slyly ironic foxes. bold dependencies boost furiously. special, |
+99|1|7567|496.93|es? permanently even excuses haggle quickly across the dependencies.|
+99|2|7970|365.83|ending accounts cajole furiously. requests promise care|
+99|3|2789|843.88|ending accounts. furiously sly packages above the carefully unusual dolphins sleep after the thinly even deposits. requests wake abo|
+100|1|7885|490.61| accounts nag slyly against the bold excuses. pearls according to the fluffily ironic accounts haggle fluffily along the quickly final platelets|
+100|2|2070|196.73| dolphins. bold deposits along the even theodolites sleep furiously about the final pinto beans. furiously unusual courts cajole about the carefully bold asymptotes. accounts integrate slyly entic|
+100|3|4994|929.57| deposits. accounts are slyly regular ideas. slyly special pinto beans upo|
+100|4|9688|22.00|uctions according to the carefully ironic deposits haggle carefully express ideas? packages across the quickly final requests c|
+101|2|5589|305.40|blithely above the fluffily pending ideas. quickly quick accounts nod ruthlessly above the carefully pending packages. slyly s|
+101|4|8263|218.71|fluffily final requests. carefully even packages wake quickly about the quickly ironic foxes. fluffily even requests hang quickly about the pending, final requests. sp|
+101|6|6324|786.53|olites sleep quickly. slyly ironic theodolites affix. furiously bold accounts integrate among the pinto beans. final ideas hang slyly along the quickly regular packages. instructions cajole.|
+101|8|55|612.09|beans against the carefully express ideas wake quickly along the quickly unusual requests. blithely regular accounts cajole fluffily. enticingly pending theodolites haggle furiously fluffily pendi|
+102|3|1384|876.75|s wake quickly. carefully express deposits wake. silent, regular requests sleep slyly after the furiously ironic deposits. slyly unusual accounts cajole|
+102|5|9137|332.71|telets are final, special deposits. silently ironic deposits wake. pending, eve|
+102|7|9156|618.00| the unusual, ironic pinto beans. theodolites above the foxes sleep slyly car|
+102|9|6942|231.02|tions haggle against the furiously ironic deposits. quickly final asymptotes haggle carefully. regular sentiments might cajole silent courts. blithely bold frays |
+103|4|5913|905.88|e across the theodolites. carefully pending escapades haggle after the ironic theodolites. furiously pending ac|
+103|6|7742|414.42|bout the bold, regular deposits; blithely even accounts are regular, even platelets-- carefully express accounts nag slyly pen|
+103|8|5164|361.48|furiously thin deposits haggle blithely. blithely regular deposits above the carefully regular accounts are slyly carefully regular packages. silent, unusual|
+103|10|429|605.20| theodolites cajole quickly above the asymptotes-- slyly special packages can haggle carefully blithely final instructions. unusual, regular ideas|
+104|5|2368|946.39|packages. final packages wake enticingly. furiously regular asymptotes are always about the carefully regular deposits. slyly regular platelets cajole carefully. final pinto beans must pro|
+104|7|6110|15.57|ending requests. carefully regular deposits use blithely. bold, ironic deposits wake slyly carefully specia|
+104|9|6269|213.89| ideas against the final accounts wake slyly regular notornis. final deposits haggle a|
+104|1|3369|729.38|ong the foxes. foxes sleep quickly? carefully regular accounts sleep. special foxes boost quickl|
+105|6|4602|27.75|lar pearls cajole never carefully even depths. blithely regular ideas are quickly. unusual asymptotes nod carefully carefully regula|
+105|8|269|158.62| unusual courts eat pending excuses. ironic, ironic requests use. bravely |
+105|10|8855|810.86|. slyly special depths sleep. bold packages engage furiously bold packages. fluff|
+105|2|8519|904.17|ding to the furiously careful ideas. dogged theodolites wake fluffily among the slyly bold ideas. blithely brave warthogs above the slyly even theodolit|
+106|7|8649|732.15| slyly ironic instructions are. bold, final accounts cajole slyly ironic pinto beans. fluffily ironic accounts around the quickly special requests use blith|
+106|9|3144|779.68| final deposits along the slyly express theodolites cajole blithely after the ironic pinto beans. furiousl|
+106|1|1411|310.40|al accounts impress. even instructions engage furiously final foxes. silently final deposits wake qui|
+106|3|2297|281.98|inal packages. pending foxes sleep bold hockey players. courts across the blithely regular packages sleep fl|
+107|8|7249|168.03|he fluffily even packages. slyly regular dependencies nag fluffily above the final, unusual foxes. final, pending foxes affix. furiously final deposits cajole quickly blithely|
+107|10|4029|91.31|integrate. requests maintain quickly. carefully regular ideas about the instructions sle|
+107|2|4667|372.94|uctions sleep doggedly final requests. express, final theodolites cajole fluffily furiously silent deposits. blithely regular requests cajole quickly regular instruction|
+107|4|7912|474.77|fluffily across the final, bold accounts. quickly regular deposits grow carefully deposits. regular requests haggle blithely. slyly special platelets boost furiously care|
+108|9|4149|558.85| of the quickly pending theodolites. fluffily unusual frays wake accounts. carefully even foxes wake slyly. carefully special pinto beans |
+108|1|4898|241.47|lthily according to the fluffy deposits. furiously silent ideas according to the furiously special theodolites wake furiously a|
+108|3|5534|626.89|instructions. blithely regular instructions according to the permanent foxes cajole blithely slyly fluffy foxes. slyly regular asymptotes cajole foxes. slyly unusual deposits |
+108|5|3142|922.27|slyly express accounts are fluffily along the blithely unusual packages. pinto beans mold furiously. furiously bold instructions are blithely deposits. quickly special accounts detect t|
+109|10|524|275.19|st the permanently final requests. carefully pending pinto beans haggle quickly slyly ironic dolphins. blithely bold deposits wake blithely. even requests cajole foxes. iro|
+109|2|4470|992.21|ake furiously packages. blithely even foxes haggle furious|
+109|4|8176|936.60|d the express accounts. even theodolites wake quickly up the furiously bold foxes. furiously regular packages use regular, bold|
+109|6|7524|374.49|sual requests. final pinto beans cajole furiously. dependencies integrate slyly even excuses. blithely special requests wake around the slyly final pinto beans. furiously pending requests wake furi|
+110|1|2782|566.46|ly sly deposits. regular deposits are; theodolites haggle furiously bold foxes. final pinto beans wake blithely furiously ironic depths. unusual, regular platelets cajole. final, ironic|
+110|3|8914|183.47|yly even foxes. carefully regular requests doubt. pending, regular requests across the blithely final theodolites hag|
+110|5|1160|932.08|ake blithely. furiously ironic accounts sleep fluffily|
+110|7|4927|196.63|theodolites. instructions haggle even packages. waters impress furiously quickly express courts. furiously ironic depths nod quickly? ironic, ironic requests sle|
+111|2|1890|321.97|st foxes. daring depths use above the furiously special ideas. ironic foxes among the carefully final theodolites are alongside of the regular depths. e|
+111|5|9587|978.65|express instructions against the furiously final grouches haggle across the blithely ironic theodolites. slyly special dependencies in place of the carefully pending |
+111|8|8069|745.33|thely bold requests. unusual packages sleep. quickly pending ideas nag furiously to the carefully final p|
+111|1|2175|496.08|s around the daringly final pinto beans use furiously against the pi|
+112|3|8509|111.42|unts. carefully ironic instructions are final, bold foxes. bold excuses run according to the unusual packages. theodolites cajole carefully according to the fluffily pending deposits? sly|
+112|6|7652|783.52| carefully among the furiously final packages. regular instructions nag. slyly pending ideas hang fluffily blithely ironic instructions. ironic deposits haggle except the quickl|
+112|9|4835|913.75|ach slyly special dependencies. furiously ironic pinto beans sleep slyly pen|
+112|2|1211|815.88|l requests integrate furiously. quickly quiet packages are carefully regular accounts. regular packages eat quickly express, ironic sheaves. quickly dogged accounts o|
+113|4|9981|396.26|ithely express pains lose bravely fluffily pending foxes. blithely ironic sauternes cajole q|
+113|7|3804|860.68|ully about the carefully even accounts: thinly even foxes are carefully. |
+113|10|522|981.41| warthogs use quickly alongside of the furiously unusual requests. final deposits are blithely acro|
+113|3|4692|141.48| blithely unusual gifts snooze against the quickly ironic packages. regular packages across the carefully regular packages bo|
+114|5|7146|447.24|ideas will nag regular accounts! carefully final requests cajole furiously quickly final tithes. furiously express instructions a|
+114|8|3062|555.12|ts. furiously regular requests run carefully thin decoys. ironic platelets sleep alongside of the slyly silent deposits. reg|
+114|1|4519|382.87|nts. ironically express dolphins dazzle blithely. special instructions wake carefully along the ideas. quickly special dolphins sleep. furiously pendi|
+114|4|2113|570.79|o beans sleep among the ironic excuses. furiously even sheaves are. never regular instructions nod.|
+115|6|1817|82.84|uffily final accounts integrate furiously along the carefully busy excuses. slyly even asymptotes doubt quickly. fluffily thin theodoli|
+115|9|983|867.45|kly. requests nag after the blithely bold packages. express requests cajole theodolites. blithely express requests sleep after the furiously regular accounts. fluffily r|
+115|2|7781|861.93|lyly ironic pinto beans affix alongside of the furiously even ideas: quickly bold warhorses sle|
+115|5|3002|81.52|efully after the quickly regular deposits. daringly pending ideas sleep even ideas. silent, re|
+116|7|6580|705.50|iously. slyly regular requests detect slyly. carefully bold packages sleep furiously carefu|
+116|10|4975|413.86|tions. regular excuses detect. ideas haggle slyly about the slyly ironic courts. ironic foxes solve. ideas affix fluffily after the special, even dependencies. final platelets according|
+116|3|8679|866.56|aphs cajole blithely regular accounts. even packages doubt; bold instructions boost quickly. fluffi|
+116|6|5632|37.30|ccounts about the special packages nag across the carefu|
+117|8|5906|706.51|into beans sleep carefully blithely bold packages. even, bold instructions use across the carefully e|
+117|1|1986|322.08|eposits. special pinto beans use fluffily across the furiously regular pinto beans. furiously regular epitaphs nag fluffily packages. special accounts a|
+117|4|2577|761.86|riously. doggedly unusual ideas boost blithely blithely regula|
+117|7|4762|552.88| run furiously ironic accounts. slyly ironic deposits haggle slyly fluffy requests. flu|
+118|9|694|744.73|refully slow requests. requests against the special pac|
+118|2|6326|325.61| packages. express, final frays affix quickly above the final asymptotes. carefully regular requests doubt quickly f|
+118|5|7806|283.27| accounts affix carefully. regular, regular packages among the brave, pendin|
+118|8|4951|976.55|s orbits. even asymptotes above the instructions wake fluffily according to the sly, final excuses. express deposits across the blithely ironic depend|
+119|10|2307|473.64| blithely unusual dolphins boost busy, express ideas. regular requests use carefully furiously ironic deposits. carefully regular packages would sle|
+119|3|1452|676.92|ular instructions was slyly. furiously bold gifts boost f|
+119|6|4955|488.93|ias are along the express requests. fluffily pending ideas nag idly against the fluffily bold instructions? foxes cajole quickly. slyly special deposits haggle slyly e|
+119|9|583|782.47|yly pending requests-- carefully special instructions haggle carefully even instructions. blithely regular theodolites detect blithely final ideas. blithely ironic deposits among the sl|
+120|1|4976|201.21|inal, regular pinto beans haggle carefully! ironic ideas unwind among the slyly regular theodolites. regular platelets kindle blith|
+120|4|7744|365.79|l, special escapades! ideas sleep slyly instructions. carefully bold requests are. even accounts cajole. final accounts use slyly |
+120|7|5329|249.61|s cajole blithely. carefully bold requests believe blithely? brave accounts above the pending, dog|
+120|10|3102|566.34|ctions; realms beside the blithely final theodolites unwind blithely packages. regular dolphins sleep carefully-- carefully express accounts wake quickly. pending depths use never courts.|
+121|2|9741|147.45|ly according to the carefully regular asymptotes. silent excuses cajole carefully against the never silent instructions. furio|
+121|6|4246|850.42|usly final instructions. theodolites are according to the permanently ironic accounts. carefully pending accounts haggle about the pending instructio|
+121|10|7670|449.39|carefully daring packages. express packages use carefully about the quickly unusual packages. special ideas along |
+121|4|8709|655.22| detect carefully along the carefully even pinto beans. gifts haggle: ideas sleep ar|
+122|3|1019|451.29| blithely regular accounts. blithely final pains sleep besides the blithely final warhorses. furiously unusual requests haggle furiously|
+122|7|2490|637.28|efully special excuses grow slyly unusual packages. carefully quiet as|
+122|1|4957|650.84|quests. quickly bold requests nag across the furiously ironic accounts. ironically express instructions detect slyly carefully ironic requests. even, un|
+122|5|2083|739.25|counts. unusual requests alongside of the regular requests are carefully stealthy instructions? regular sauternes cajole. final theodolites breach carefully at the blithely final idea|
+123|4|9881|107.03|fully bold deposits detect slyly pending instructions. slyly special ideas detect blithely. slyly fluffy instructions hinder|
+123|8|5638|818.19|thely even pinto beans. furiously regular asymptotes affix furiously. regular, ironic tithes integrate quickly. blithely regular requests breach finally. decoys alon|
+123|2|2692|217.01|he ironic accounts nag fluffily after the bold, pending theodolites. blithely final ideas sleep carefully according to the blithely ironic foxes. regular requests are. furiousl|
+123|6|5311|149.65|eposits cajole according to the carefully pending packages. furiously final epitaphs solve alongside of the even requests|
+124|5|7102|901.98|ily accounts. furiously busy theodolites above the deposits thrash above the blithely final foxes. express instructions nod slyly furiously busy packages. special asymp|
+124|9|3969|908.64|l epitaphs. packages cajole among the furiously regular requests. closely|
+124|3|9295|882.54|s along the accounts poach quickly ironic deposits. even, final excuses thrash carefully about the express, special pains. carefully careful accounts breach slyly|
+124|7|9416|822.78|ously. theodolites affix around the slyly bold packages. even, ironic packages are carefully pains. furiously unusual requests sleep blith|
+125|6|2263|358.45|e. ironic, regular requests cajole fluffily along the even ideas. final ideas wake blithely. blithely bold |
+125|10|8276|668.65|nd the carefully express requests. slyly regular requests haggle. blithely unusual platelets solve fluffily fluffily regular|
+125|4|2600|455.70|ounts. thinly special accounts cajole carefully. even, special accounts after|
+125|8|5546|806.66| to the unusual courts are deposits! final, final pinto beans solve slyly. ironic accounts boost fluffily. furiously pending d|
+126|7|2647|221.89|lyly final pinto beans across the regular, even courts use slyly slyly pending braids! unusual requests along the furious|
+126|1|2373|194.38|fter the ideas. blithely daring sheaves print furiously among the blithely final packages. iron|
+126|5|1532|451.61|refully alongside of the quickly bold excuses. enticing, bold |
+126|9|5458|929.43|leep to the furiously special accounts. furiously final courts |
+127|8|7658|712.33|al pinto beans! slyly ironic excuses boost after the packages. express foxes integrate carefully. pending, regular theodolites |
+127|2|1467|237.98|regular accounts! quickly ironic packages haggle according to the accounts. carefully ironic |
+127|6|8281|3.14|ts above the furiously pending asymptotes cajole after the deposits. slyly ironi|
+127|10|8894|73.42|fter the sometimes special courts sleep about the slyly unusual reque|
+128|9|6982|425.29|ironic asymptotes. fluffily ironic packages use. ironic, regular ideas are in place of the quickly silent deposits. final, bold gifts across the ironic, regular pac|
+128|3|7602|224.49|xcuses. blithely unusual theodolites use slyly carefully even warthogs. slyly even dugouts haggle slyly final, express pinto beans. furiously bold packages thrash requests? slyly unusual packages |
+128|7|3766|947.16|arefully regular packages boost regularly. accounts are according to the blithely even dependencies. slyly silent accounts doubt slyl|
+128|1|7023|875.78| furiously quickly regular pinto beans. always special requests are. quickly regular deposits are furiously. slyly unusual theodolites haggle evenly; furiously special deposits wa|
+129|10|5721|129.69|ully express requests above the ironic, final requests cajole slyly along the quickly special packages. sl|
+129|4|7242|200.26|es across the furious escapades wake quickly slyly e|
+129|8|5299|330.59|final sentiments affix atop the silent foxes. busy pinto beans cajole. slyly final pinto beans haggle against the carefully expres|
+129|2|1968|27.22|ealthy, ironic deposits. slyly ironic pinto beans are blithely pinto beans. blithely ironic |
+130|1|4928|223.38|ths. slyly even theodolites detect according to the slyly final courts. carefully unusual deposits ar|
+130|5|6909|275.58|lly unusual accounts try to boost along the special packages. furiously bold requests x-ray blithely ironic waters. slyly unusual orbi|
+130|9|4850|442.81|ully regular deposits snooze. slyly silent foxes detect furiously furiously bold requests. slyly regular accounts breach. blithely bli|
+130|3|7387|883.99|aggle furiously. even ideas hinder deposits. even, final ideas are. unusual theodolites after the special, express foxes haggle carefully pending accou|
+131|2|3263|211.70|sits sleep quickly regular multipliers. slyly even platelets cajole after the furiously ironic deposits. slyly ironic requests should have to cajole: bl|
+131|7|125|861.84|l accounts grow quickly-- slyly ironic requests haggle? quickly express pinto bean|
+131|2|5138|572.43|grouches run with the carefully even packages. ironic, even deposits run slyly along the packages. special dependencies among the regular |
+131|7|8945|613.09| are carefully along the quickly final theodolites. packages after the quickly pending package|
+132|3|3092|687.29|y special decoys against the ideas affix against the sly|
+132|8|1904|925.73|the regular foxes wake ironic deposits. ironic, special requests use blithely instructions! final requests hang. blithely regular deposits haggle. ir|
+132|3|7441|357.06|ests. furiously unusual requests wake furiously. quickly unusual depos|
+132|8|5303|353.06|ep blithely after the sly accounts. slyly express dolphins cajole amon|
+133|4|5727|49.17|boost blithely across the ironic, regular instructions. packages use slyly unusual requests. bold accounts above the fu|
+133|9|404|478.18|ly ironic requests run instead of the blithely ironic accounts? regular ideas use fluffily: even, express packages sleep abov|
+133|4|4568|57.48|dolites. ironic accounts are blithely pinto beans. regular pinto beans haggle beneath|
+133|9|2813|277.26|s. pending, final accounts haggle blithely furiously pending deposits! carefully unusual attainments integrate. blithely bo|
+134|5|8879|848.14|lites. slyly final foxes after the bold requests cajole carefu|
+134|10|9013|102.99|pendencies. furiously express warthogs cajole furiously ironic, regular asymptotes. bold deposits boost among the furiously even theodolites. regular instructions integrate carefully |
+134|5|852|927.45| sleep unusual, express packages. unusual sentiments are furio|
+134|10|6270|388.28| to the furiously pending deposits nag along the slyly express asymptotes. slyly silent accounts shal|
+135|6|6940|465.82|ding foxes cajole. even dugouts haggle busily. fluffily pending packages about the express excuses boost slyly final packages. blithely express ideas cajole about the carefu|
+135|1|2443|9.83|atterns. pending, special deposits are furiously. express, regular deposits integrate quickly. unusual gifts cajole blithely stealthily pending deposit|
+135|6|7453|698.42|ven accounts. slyly final instructions nag slyly around the regular, unusual packages. slyly sp|
+135|1|2771|306.43|old deposits. furiously express instructions boost. pending dolphins use requests. slyly regular packages cajole quickly final ideas. pending, regular ideas nag carefully even, express pla|
+136|7|2237|548.19|ond the silent accounts haggle above the blithely regular packages|
+136|2|6068|806.19|structions. ironic theodolites haggle according to the final, daring pearls. carefully ironic somas are silently requests. express pa|
+136|7|8979|387.57|ans. express pinto beans wake carefully among the slyly ironic foxes: carefully final pinto beans haggle blithely. pending, final deposits promise furiously|
+136|2|9617|525.81| across the carefully pending warthogs. close, regular packages are quickly after the never ironic foxes. accounts sleep quickly along the furiously regular re|
+137|8|9057|302.26|slyly about the regular instructions. even, ironic theodolites use carefully around the even decoys. unusual, pending dolphin|
+137|3|4078|441.11|packages. blithely unusual sentiments should are. furiously regular accounts nag quickly carefully special asymptotes! idly ironic requests dazzle bold requests. carefully expres|
+137|8|467|371.85|ly special accounts detect carefully. furiously ironic deposits nag express packages. slyly quiet |
+137|3|7850|187.31|atelets sublate fluffily. enticingly unusual packages boost according to the blithely ironic foxes. pending requests mold sly|
+138|9|133|576.96|regular, final deposits maintain slyly even requests. regularly furious deposits use above the stealthy requests. ironic deposits are. carefully final frays are carefully. carefu|
+138|4|2535|885.35|lar deposits. courts sleep carefully. furiously express ideas boost furiously after the final, regular foxes. furiously bold deposits are. express accounts haggle blithely. |
+138|9|7907|119.83|epitaphs? quickly express foxes use pending accounts. special packages cajole blithely among the quickly unusual accounts? boldly ironic packages across the slyly ironic senti|
+138|4|967|309.03|pendencies integrate against the unusual pains. carefully unusual theodolites wake quickly across the deposits. blithely regular deposits alongside of the carefully regular deposits|
+139|10|2886|285.75|fully ironic requests according to the quickly final idea|
+139|5|9255|684.61|ickly furiously regular excuses. boldly express deposits sleep. ideas nag above the silent dependencies. slyly regular packages wake furiously. requests are carefully. quickly final fox|
+139|10|1042|972.23|gular, regular theodolites. regular asymptotes haggle carefully according to the permanently even deposits. slyly special account|
+139|5|3285|690.00|xpress pains. quickly regular ideas after the special, bold excuses wake furiously final ideas. slyly bold accounts nag packages. ironically regular|
+140|1|2379|501.05|of the silent, bold courts. slyly regular dependencies haggle. fluffily special deposits cajole carefully. quickly ironic depos|
+140|6|3533|781.45|ayers. carefully ironic pinto beans nod carefully furiously regular pinto beans. slyly ironic requests after the carefully regular packages are about the blithel|
+140|1|304|45.84|ing requests. carefully unusual foxes are final requests. slyly regular accounts wake permanently. quickly ironic theodolites hagg|
+140|6|7346|429.52| special pinto beans wake carefully unusual warthogs! furi|
+141|2|6776|293.63|fluffily unusual courts sleep. close pinto beans haggle quickly after the carefully ir|
+141|8|1660|139.18|egular accounts. enticingly bold theodolites eat slyly across the never ironic platelets. theodolites wake bli|
+141|4|7628|838.08|sly about the pinto beans. blithely ironic ideas sleep. foxes are quietly among the pinto beans. carefu|
+141|10|90|810.68|e doggedly regular ideas. foxes haggle slyly. slyly regular theodolites across the carefu|
+142|3|9219|897.49|thlessly special requests sleep blithely about the bold deposits. express, ironic instructions wake. final packages are blithely. deposits are carefully furiously even deposits. furiously regular a|
+142|9|13|334.33|are blithely blithely brave requests. slyly regular theodolites are furiously. blithely ironic dependencies haggle blithely. furiously unu|
+142|5|3076|860.55|gular requests about the pending packages wake furiously dogged accounts. th|
+142|1|3858|854.08|efully special deposits. blithely bold pinto beans haggle. slyly final ideas boost blithely. finally special requests mold along the blithely express packages. entic|
+143|4|7326|960.64|the slyly pending requests cajole quickly blithely regular platelets. even requests boost carefully. ironic, final instructions above the regular courts boost a|
+143|10|3923|741.01|le quickly furiously silent ideas. carefully regular requests ar|
+143|6|7152|772.24|fully furious accounts. final asymptotes cajole regular requests. carefully regular courts are quickly. slyly ironic ideas above the carefully regular requests wake|
+143|2|1952|199.37|l accounts are quickly after the unusual packages. regular accounts wake among the quickly even accounts. even, ironic|
+144|5|6295|457.37| pinto beans promise across the blithely bold packages. express, regular accounts play around the slyly silent deposits. specia|
+144|1|494|849.96|uriously ironic pearls wake idly furiously even pearls. foxes impress slyly busily express requests. carefully slow somas wake quick|
+144|7|1799|713.88|yly final requests. packages are. carefully daring accou|
+144|3|5427|361.83| foxes integrate carefully. deposits cajole fluffily. pending deposits kindle slyly carefully regular packages. even, thin accounts according to th|
+145|6|11|641.67| slyly regular packages are slyly carefully special dolphins. unusual braids use furiously about the final courts. slyly special|
+145|2|3838|568.91|ss, final asymptotes are. furiously express accounts run. furiously express dependencies eat carefully blithely ironic theodolites. closely ironic foxes among the silent asymptotes cajole|
+145|8|1344|422.60| dependencies. even patterns detect slyly after the ironic deposits. ironically even ideas wake slyly. even packages against the blithely express accounts haggle furiously carefully regular|
+145|4|4332|894.57|are carefully above the quickly silent deposits. evenly bold reque|
+146|7|2726|231.15|uriously after the fluffy accounts. furiously bold deposits cajole. requests might engage. quick accounts wake carefu|
+146|3|4628|123.91|ly across the dependencies. daringly ironic deposits are furiously; requests are. quickly regular accounts hang. carefu|
+146|9|5893|858.59|sual instructions believe. fluffily unusual warhorses nag. unusual dependencies sleep. slow sheaves haggle furiously. carefully ironic dependencies cajole slyly against the accounts. |
+146|5|9687|882.37|packages? ideas affix slyly even accounts: express requests wake slyly carefully special depths. ironic in|
+147|8|7442|939.14|sts against the furiously unusual instructions integrate ironic accounts. slyly final pinto beans sleep blithely carefully final asymptotes. slyly ironic warhorses befor|
+147|4|7647|102.19|refully regular orbits about the furiously express asymptotes haggle carefully according to the blithely regular ideas. blithely express excuses around the furiously |
+147|10|1596|466.37|ole. slyly final packages do haggle quickly. unusual accounts across the pending pinto beans was furiously according to the furiously brave deposits. pending deposits along the regular request|
+147|6|4235|235.91|ar pinto beans. regular instructions sleep carefully after the furiously blithe accounts. slowly pending ideas could nag careful, even accounts. attainments use slyly quickly|
+148|9|8007|177.40|final requests-- slyly regular theodolites haggle carefully across the blithely final dependencies. slyly even requests about the carefully even accounts sleep |
+148|5|1904|774.56|y even pinto beans. fluffily ironic packages sleep slyly. permanently brave requests boost furiously packages. boldly ironic deposits across the carefully bold pinto b|
+148|1|5393|264.09|ses. slyly pending packages haggle fluffily fluffily even instructions. fluffily regular packages are carefully about the furiously even asymptot|
+148|7|2957|250.98|. requests boost above the bold, special foxes. blithely regular platelets serve blithely slyly final ideas. carefully special idea|
+149|10|959|679.10|y to sleep carefully ironic requests. even, regular dependencies haggle. slyly unusual foxes haggle along the instructions. quickly even accounts nag furiously special accoun|
+149|6|7283|201.03|usly bold instructions. regular, final deposits alongside of the furiously ironic platelets are slyly even instructions. carefully bold accounts are. ironic, regular requests nag furious|
+149|2|7392|266.53|es detect along the regular instructions. bold ideas boost slyly. quickly unusual accounts doubt. carefully even foxes thrash slyly silent, ironic dolphins: Tiresias must wake |
+149|8|4104|312.37|ly express excuses. bold pinto beans boost blithely across the bold, final pinto beans. final deposits haggle carefully from the|
+150|1|8091|524.71|sleep furiously furiously bold warthogs. furiously express gifts according to the regularly silent sentiments boost within the f|
+150|7|2721|814.79|dependencies. special accounts wake carefully furiously regular accounts. regular accounts haggle along the express instructions. express pinto beans along the express, bold deposits run |
+150|3|3172|33.71| about the silent ideas. fluffily final requests impress. slyly final requests wake carefully about the slyly express foxes. slyly regular warthogs sleep fur|
+150|9|1640|327.34|slyly even deposits alongside of the furiously even accounts detect boldly quickly regular accounts. final accounts kindle carefu|
+151|2|391|281.90|dolites. boldly ironic packages cajole fluffily regular instructions. regular, ironic accounts are blithely. ironic accounts are alongside of th|
+151|9|253|840.06| haggle. somas are carefully. slyly regular requests sleep blithely atop the thinly express deposits. stealthily express packages cajole daringly express requests. carefully special requests after t|
+151|6|1484|71.68|riously final requests sleep according to the regular deposits? slyly ironic ideas wake furiously. quickly even theodolites use fluffily. regular, unusual courts according to the regular |
+151|3|9417|244.06| foxes along the hockey players are slyly about the blithely even packages. unusu|
+152|3|1396|164.60|d the instructions. carefully pending accounts haggle fluffily ruthless instruc|
+152|10|2653|432.12|carefully pending requests. quickly ironic requests haggle carefully special theodolites. blithely special requests aga|
+152|7|3599|77.38| quick excuses according to the pending, ironic requests snooze carefully slyly even foxes: slyly regular instru|
+152|4|1283|142.73|olites above the furiously even requests dazzle blithely against the busy, regular pains. furiously blit|
+153|4|4695|539.86|atelets. dolphins haggle blithely carefully ironic deposits? express, final accounts wake about the requests. even deposits should use quickly. regular,|
+153|1|8464|680.14|cording to the final instructions. carefully fluffy asymptotes haggle carefully |
+153|8|2916|685.52|ully express deposits boost daringly packages. furiously ironic accounts sleep slyly ironic instructions. special deposits integrate blithely. |
+153|5|6397|285.92|furiously special platelets haggle quickly even, bold pinto beans. blithely close pinto beans boost around the furiously regular packages. quickly express requests cajole.|
+154|5|3961|474.19|quickly pending requests nag express dependencies. furiously unusual requests about the regular, pending packages wake according to the ironic packages! theodolites wake about the unusual, regula|
+154|2|3231|829.29|ins along the packages use carefully requests. furiously unusual packages kindle fluffily quick|
+154|9|7315|960.19|uickly regular dolphins ought to believe among the q|
+154|6|1682|160.31|refully except the sly, even requests. careful ideas haggle after the slyly regular foxes: slyly special packages at the slyly regular deposits wake carefully theod|
+155|6|2893|222.02|. bold packages are toward the silent pinto beans. quickly fin|
+155|3|7077|413.24|lar instructions against the furiously unusual instructions breach furiously for the bold, even platelets. ironic accounts must have to are quickly across the |
+155|10|3029|413.02|the carefully ironic asymptotes. even, unusual accounts sleep furiously about the blithely regular ideas. quickly re|
+155|7|977|751.45|quickly silent deposits doubt above the unusual instructions. special r|
+156|7|4755|453.69|e of the excuses. slyly even theodolites boost about the slyly final foxes? requests after the carefully regular platelets sleep above the furiously pending d|
+156|4|7228|994.19|odolites wake quickly slyly final dinos. requests cajole slyly along the instructions. furiously regular deposits cajole slyly blithely ironic instructions. instructions wake. blithely even pinto be|
+156|1|3043|252.66|lithely express, silent decoys. bold, special requests along the carefully even accounts|
+156|8|3191|9.87|ar instructions-- quickly special deposits wake fluffily about the blithely e|
+157|8|5414|369.44|ong the carefully bold ideas boost across the regular, ironic requests. ironic fo|
+157|5|763|568.46|cial packages boost along the ideas. packages sleep slyly express packages. ironic, bold requests|
+157|2|3718|901.53|, ironic foxes. blithely even foxes wake about the carefully special req|
+157|9|3400|288.41|encies integrate carefully even accounts. regular, regular sentiments are against the slyly regular deposits-- even, even ideas use inside the carefull|
+158|9|2845|408.72|y. slyly final pinto beans believe fluffily pending, regular deposits. final, unusual ideas according to t|
+158|6|8715|452.31|instructions along the ironic, final requests are fluffily regular deposits. regular deposits cajole carefully about the silent instructions|
+158|3|4465|837.16| wake carefully in place of the furiously express deposits. slyly regular instructions engage. fluffily f|
+158|10|4251|431.90|etly special accounts boost carefully final multipliers. carefu|
+159|10|9200|356.66|ccording to the furiously final accounts. carefully fluffy foxes wake idly against the quickly final requests. evenly even pinto beans must have to are against the carefully regular de|
+159|7|3585|629.29|g platelets wake furiously slyly bold deposits? slyly regular accounts across the stealthily ironic accounts cajole along the special, ironic pearls. fluffily regular pinto|
+159|4|6085|171.40|ross the blithely special deposits are quickly carefully ironic Tiresias. quickly regular deposits was furiously. unusual accounts affix blithely about the regular deposits. asymptotes ab|
+159|1|3717|790.87|y blithe dependencies. final accounts haggle furiously. even, special asymptotes|
+160|1|2434|525.73|lithely. furiously silent theodolites after the ca|
+160|8|8324|999.93|ly final instructions. closely final deposits nag furiously alongside of the furiously dogged theodolites. blithely unusual theodolites are furi|
+160|5|6034|733.59| furiously against the final instructions. silent accounts sleep blithely after the boldly final requests. ex|
+160|2|6872|872.20|ions are carefully. carefully express foxes nag slyly before the carefully final excuses. accounts after the furiously ironic packages are furio|
+161|2|9365|790.03|scapades. packages use. slyly final accounts haggle across the quickly final th|
+161|10|8421|394.05|cial ideas. ironic instructions eat blithely slyly special packages. furiously final packages alongside of the furiously final instructions boost carefully against the quickly |
+161|8|9679|688.47|ns. blithely express requests sleep slyly foxes. blithely unusual ideas |
+161|6|679|893.72| the fluffily final requests. ironic, pending epitaphs affix slyly. qui|
+162|3|315|923.04| ideas. carefully final dugouts will have to wake quickly regular asymptotes. express grouches unwind carefully after the regula|
+162|1|2604|104.20|usly regular excuses. silent, even sheaves are according to the regular requests. packages grow blithely slyly regular accounts. ca|
+162|9|7014|236.07|cording to the stealthily fluffy theodolites. carefully unusual excuses around the regular deposits cajole slyly amo|
+162|7|4381|824.36|as across the furiously ironic notornis print blithely alongside of the final, pending deposits. fluffily express deposits slee|
+163|4|9241|955.81|cial dolphins. furiously bold foxes could have to use. never sly accounts cajole fluffily about the unusual, special pinto beans. pending, even requests around the quickly special deposits use f|
+163|2|3427|499.51|ithely bold packages integrate slyly quiet pinto beans. carefully even deposits boost slyly about the furiously fluffy packages. evenly regular dependencies wa|
+163|10|5323|920.75|sly even theodolites against the carefully bold packages wake final pinto beans. furiously pending deposits dazzle furiously. blithely exp|
+163|8|9676|573.48|ending accounts haggle blithely ironic, even packages. carefully pending packages wake carefully across the ruthlessly pending accounts! pinto beans wake. slyly final deposits boost slyly. fluffily|
+164|5|1295|341.95| bold instructions cajole slyly ironic deposits. quickly ironic foxes are carefully final, bold theodolites. ironic deposi|
+164|3|2134|84.02|ns believe. carefully express theodolites impress. carefully fina|
+164|1|3245|814.67|brave accounts cajole according to the final platelets. furiously final dolphins across the furi|
+164|9|3028|64.89|fully furiously regular requests. furiously bold orbits serve about the regular packages? carefully final deposits p|
+165|6|4424|943.82|ular requests. regular accounts cajole against the blithely ironic deposits. blithely even packages cajole. furiously final deposits cajole. thinly pending deposits hagg|
+165|4|5534|717.83| quickly regular deposits above the fluffily thin deposits haggle furiously against the quickly final depend|
+165|2|3780|730.28| furiously quickly regular foxes. pending requests engage evenly blithel|
+165|10|6114|210.84|foxes. foxes haggle. dolphins use carefully according to the fluffily regular packages. blithely special accounts according to the slyly final frets breach blithely after the care|
+166|7|6527|309.00|lly. dependencies haggle carefully at the slyly special packages. regular, final packages|
+166|5|6508|714.49|y express deposits cajole furiously above the carefully even theod|
+166|3|9364|581.52|pinto beans. pinto beans cajole furiously carefully special requests-- quickly |
+166|1|6713|631.58| sleep carefully. quickly even deposits run carefully fluffily ironic orbits. ironic deposits wake furiously. close sheaves along the special packages sleep carefully special instr|
+167|8|4933|666.70|ular deposits among the even dolphins are quickly express accounts. final, ironic theodolites cajole closely. th|
+167|6|5789|524.27| are furiously final, even dugouts. ironic, regular packages nag fu|
+167|4|4756|336.75|es are carefully along the carefully express tithes. furiously even deposits cajole slyly slyly regular deposits. bold excuses about the carefully ironic requests sleep blithely instructions|
+167|2|6748|704.97|t the silent ideas are blithely carefully even packages; blithely|
+168|9|347|394.83|hely blithely final theodolites. blithely final deposits among the quickly even ideas haggle about the blithely bold d|
+168|7|1281|771.90|, pending packages. ironic pinto beans use carefully. fluffily bold deposits|
+168|5|9089|508.37|ests are always. regular ideas sleep fluffily; special, express instructions cajole slowly. pending platelets boost furiously against the bold, even instructions. bold instructi|
+168|3|7519|963.43|requests above the quickly regular deposits use carefully aft|
+169|10|6914|619.53|uickly along the dependencies. furiously pending notornis cajole at the carefully special attainments. carefully ironic packages impress slyly care|
+169|8|6589|947.03|gside of the quickly regular asymptotes. quickly even theodolites against the theodolites promise express requests. ironic accounts wake careful|
+169|6|6731|713.35| the quickly special excuses wake blithely alongside of the carefully silent accounts. regular dolphin|
+169|4|7691|476.19|slyly alongside of the warthogs. fluffily even instructions poach under the slyly pending packages. blithely silent deposits use across the fur|
+170|1|7516|581.65| pinto beans. unusual ideas was fluffily. excuses cajole carefully final dependencies. platelets nag quickly according to the furiously ironic requests. carefully regular dependenci|
+170|9|838|667.16|orges do sleep furiously. fluffily furious requests among the final requests sleep after the slyly bold ideas? regular pinto beans might ha|
+170|7|6498|251.19| fluffily regular accounts integrate. blithely even packages cajole fluffily. furiously ironic excuses haggle by the finally final requ|
+170|5|6593|202.07|ep blithely final packages. quickly bold pains cajole carefully across the somet|
+171|2|8217|859.60|ress deposits. carefully special requests are furiously final requests. accounts cajole carefully blith|
+171|1|2311|864.96|s are along the blithely final deposits. regular asymptotes nag slyly against the requests. accounts cajole carefully carefully |
+171|10|8561|22.69|y close ideas are quickly silently regular packages. even, silent requests wake against the slyly special dependencies; regular accounts sleep doggedly furiously final pinto beans. slyly unusual pac|
+171|9|7589|935.29|s above the theodolites wake slyly along the carefully unusual dependencies. carefully express theodolites a|
+172|3|9799|184.96|ts. slyly even asymptotes nag blithely regular accounts. final platelets cajole furiously slyly bold packages. ironic accounts sleep slyly. pendi|
+172|2|8333|920.74|ronic foxes. quickly unusual accounts cajole blithely. blithely bold deposits cajole. blithely close pinto beans cajole requests. quickly express excuses around the quickly even deposits nag agai|
+172|1|3589|437.86|posits should have to boost furiously near the unusual ideas. final packages cajole blithely. carefully final deposits boost carefully. carefully special attainments boost quickly af|
+172|10|1661|687.13|y among the slyly even requests. ideas according to the slyly pending dinos print quickly slyly ironic foxes. pending, even excuses dazzle car|
+173|4|2536|353.84|ons-- final, silent dependencies sleep across the special, special excuses. furiously even accounts must have to mold after the ironic accounts. reque|
+173|3|8307|70.22|alongside of the furiously even packages. furiously final requests snooze blithely alongside of the carefull|
+173|2|6050|683.78|e after the slyly regular ideas. unusual pinto beans cajole even asymptotes-- silent, stealthy requests after the even accounts haggle blithely regular instructions. slyly ev|
+173|1|6162|877.84|es. slyly bold requests after the blithely regular dependencies cajole slyly even ideas. unusual deposits integrate about the final somas. |
+174|5|2103|681.95|sual, express requests wake furiously ruthless, final accounts. carefully ironic somas dazzle furiously. unusual asymptotes sleep-- patterns about the furiousl|
+174|4|6795|143.48|regular theodolites. special accounts integrate across the carefully ironic Tiresias. blithely even platelets detect. foxes about t|
+174|3|111|135.46| express packages-- quickly unusual courts lose carefully requests. bold accounts solve about the theodolites; pinto beans use. ironic foxes|
+174|2|8404|126.20|nding accounts mold furiously. slyly ironic foxes boost express sheaves. daringly final packages along the stealthy dependencies are blithely ironic requests. furiously pending pin|
+175|6|5515|487.68|ages sleep against the Tiresias. slyly pending packages print slyly above the evenly ironic dolphins. furiously ironic packages use f|
+175|5|7522|784.93| affix. quickly final theodolites haggle furiously after the slowly even pinto beans. furiously final packages use slyly. slyly regular reque|
+175|4|8501|706.61|int above the instructions. furiously regular requests integrate blithely according to the instructions. slyly pending foxes are asymptotes. slyly ruthless accounts wake. r|
+175|3|9456|978.56| regular packages. carefully ironic packages use. blithely ironic accounts among the pending, |
+176|7|7180|179.09|riously final requests. accounts doubt blithely regular somas. slyly even platelets are. theodolites across |
+176|6|3589|157.38|inal excuses. express deposits haggle carefully even deposits. carefully unusual requests haggle along the fluffily bold deposits. even, final requests affix. furi|
+176|5|5407|947.51|ending accounts eat carefully instructions. carefully pending packages detect slyly express accounts. foxes wake fluffily across th|
+176|4|1783|861.63|g the carefully special platelets. dogged, ironic asymptotes wake requests. regular excus|
+177|8|1239|44.75|requests use furiously regular, final requests. regular requests on the pending, ironic deposits use slyly among the excuses. carefully regular sheaves are.|
+177|7|4349|63.36|osits sleep among the fluffily unusual instructions. ironic dolphins cajole. furiously bold deposits sleep carefully. even, unusual accounts|
+177|6|9872|252.42|sual platelets. bold foxes affix furiously. pending, pending accounts lose furiously. pending platelets along the unusual, even foxes wake regular, even theo|
+177|5|4727|859.82|es are. slyly ironic packages haggle around the slyly bold deposits. bold foxes haggle blithely. f|
+178|9|4231|558.56|deposits. patterns use against the furiously unusual accounts. accounts wake carefully above the careful|
+178|8|1919|362.26| ironic dependencies. blithely regular packages detect fluffily special theodolites. regular instructions poach-- ironic deposits along the final requests |
+178|7|6836|864.93|y. ideas integrate regular pinto beans. special foxes wake above the slyly ironic asymptotes. quickly ironic ideas sleep. silent dependencies against the slyly bold packa|
+178|6|6922|475.18| regular patterns. fluffily express accounts about the furiously bold deposits cajole slyly about the furiously silent foxe|
+179|10|6956|444.38|g the furiously careful excuses haggle quickly thinly special Tiresias. furiously express foxes after the quickly regular deposits sleep ironic packages|
+179|9|1954|372.75|even dependencies print carefully. deposits boost blithely about the ironic, ironic accounts. express, regular deposits are. bli|
+179|8|2710|277.15|d the frets. pending packages doze quickly across the furiously regular deposits. pending, even deposits impress ironic ideas. quickly regular r|
+179|7|4776|8.39|sly special pinto beans. pinto beans cajole. carefully unusual ideas around the silent accounts are blithely carefully ev|
+180|1|2467|440.25| instructions affix. regular packages cajole quickly. carefully express asymptotes use furiously around the pendin|
+180|10|1108|934.59|hinly after the regular, unusual asymptotes! carefully regular theodolites sublate. regular, ironic deposits against the regular pinto beans nag ca|
+180|9|724|426.16|e, regular accounts. furiously final ideas are furiously above the bold, silent asymptotes. sly instructions are carefully quickly final sentiments. furiously ironic foxes cajole bold, exp|
+180|8|5899|864.83|hin the carefully furious pinto beans. furiously ironic pinto beans use slyly above the even instructio|
+181|2|2416|844.44|ully. theodolites throughout the blithely unusual pinto bea|
+181|2|3242|886.53| express ideas nag carefully brave accounts. slyly express deposits would affix. final, special requests against the slyl|
+181|2|215|938.29| accounts boost furiously furiously blithe theodolites. slyly bold requests unwind special, unusual requests. furious ideas boost quickly pending |
+181|2|1122|657.25|lyly fluffily pending foxes. fluffily ironic pains haggle. thinly regular requests against the deposits affix after the never ev|
+182|3|9699|535.27|ound the furiously regular foxes. pending requests dazzle along |
+182|3|960|519.36|arefully pending dependencies are always slyly unusual pin|
+182|3|6243|741.46|accounts are slyly. furiously ironic requests haggle. express, special instructions against the ironic theodolites use s|
+182|3|6146|365.00|s. blithely express theodolites sleep blithely alongside of the requests?|
+183|4|30|875.44|slyly. furiously regular instructions cajole slyly about the pending, final theodolites. blithely final deposits cajole fluffily alo|
+183|4|4482|424.86|es. depths affix fluffily. bold instructions haggle. ruthless instructions must have to boost|
+183|4|8707|884.26|posits wake. blithely pending requests nag furiously alongside of the p|
+183|4|333|678.16|ost final, final theodolites. slyly bold foxes dazzle carefully furiously regular accounts. regular, sly instructions about the furiously regular excuses nag blithely abou|
+184|5|7069|449.45|nal ideas. blithely final ideas haggle against the pinto beans. qu|
+184|5|9193|576.88|uickly quick dependencies could detect furiously. final packages p|
+184|5|6400|551.90|ss dependencies. quickly even pinto beans are. express accounts a|
+184|5|831|186.84|kages cajole carefully furiously ironic instructions. deposits use bl|
+185|6|1475|538.58|unts hinder slyly. quickly express ideas sleep carefully |
+185|6|6244|213.04|ly unusual decoys are furiously quickly regular packages. bold, ironic foxes cajole fluffily around|
+185|6|7245|426.74|sleep blithely alongside of the regular excuses. even, regular|
+185|6|8014|510.23|lithely even ideas. regular platelets wake carefully ironic, special instructions! final pearls above the fluffily quiet ideas use furiously about the |
+186|7|1095|252.84|. carefully regular pinto beans according to the blithely close asymptotes haggle carefully special requests. packages cajole up the furi|
+186|7|1945|18.75|nic foxes boost carefully careful packages: express, fluffy dolphins nag quickly ironic packages. slyly bold requests nag amon|
+186|7|8838|729.42|ing asymptotes. enticingly regular theodolites mai|
+186|7|7898|812.37|ctions sleep silently carefully bold platelets. furiously ironic dependencies boost. regular de|
+187|8|8656|238.66|tes use along the even foxes? final foxes haggle pinto beans. slyly ironic theodolites are according to the deposits. furiously pending reques|
+187|8|4945|316.64|eposits boost quickly bold requests. furiously regular ideas boost boldly. special, express dependencies are fluffily slyly reg|
+187|8|3183|362.75|t the bold platelets. fluffily express platelets cajole fluffily along the always bold requests. blith|
+187|8|7440|989.71|e slyly against the slyly regular pinto beans. requests haggle carefully around the asymptotes. regular, regular asymptotes use furiously some|
+188|9|4835|771.95|pains are fluffily about the fluffily pending asymptot|
+188|9|2620|331.70|elets nag slyly regular pinto beans. slyly even dugouts above the blithely unusual theodolites su|
+188|9|730|713.62|nag against the final accounts. blithely pending attainments lose. silent requests wake quickly. careful|
+188|9|5430|920.20|uriously. special, regular instructions sleep along the accounts. quickly even foxes across the regular theodolites hang u|
+189|10|1305|392.50|packages. regular, unusual accounts lose furiously fluffily regular platelets. requests sleep carefully dependenc|
+189|10|8777|573.22|beans cajole slyly ironic requests. requests are quickly unusual, even packages. ironic frays haggle. blithely pending requests nod slyly. express, silent requests against the slyly unusual |
+189|10|6369|946.07|ts hinder slyly regular, unusual foxes. final sentiments use above the slyly r|
+189|10|2505|593.23| the deposits. special deposits sleep-- furiously regular sauternes solve furiously across the furiously regular pack|
+190|1|535|621.53|unts must have to haggle; slyly ironic accounts affix slyly alongside of the carefully even accounts. furious deposits haggle quietly among the packages. blithely |
+190|1|5845|608.91| haggle along the carefully unusual pinto beans. quickly final accounts sleep a|
+190|1|4579|396.60|inal, final foxes. regular, even deposits wake blithely! silent, regular packages integrate according to the slyly regular deposits. ironic, ironic notornis ha|
+190|1|2861|458.00|s cajole slyly across the daring, final pinto beans. carefully quiet requests affix along the a|
+191|2|8310|521.06|the slowly regular deposits. special accounts along the quickly unusual|
+191|3|1693|464.46|y. slyly unusual waters across the special pinto beans nag blithely according to the busy deposits. carefully regular accounts are against the regular accounts; perman|
+191|4|597|126.96|ly final accounts should have to boost above the doggedly express pinto beans. blithely regular packages cajole furiously bold requests. fluf|
+191|5|9673|119.41|press deposits kindle theodolites! slyly final dependencies against the blithely final packages sleep slyly regular requests. theodolites cajole furiously quickly bold a|
+192|3|606|198.69|inal platelets integrate regular accounts. accounts wake ironic, silent accounts. slyly unusual accounts kindle carefully-|
+192|4|2656|916.16|uickly. slyly bold ideas affix special, close theodolites. ironic, pending requests use carefully. blithely regular |
+192|5|1811|359.59|ly carefully special asymptotes. furiously pending instructions haggle blithely bravely pending requests. carefully f|
+192|6|8305|861.23|s against the carefully regular foxes haggle fluffily across the pending accounts. blithely final packages sleep after the furiously ironic theodolites. quickly bold r|
+193|4|6184|335.98| quickly bold deposits cajole furiously ruthless courts. carefully|
+193|5|4762|606.19|ns sleep against the furiously regular asymptotes. carefully even asymptotes across the daringly final packages sleep fluf|
+193|6|385|571.71|ons. slyly ironic deposits wake furiously ironic, unus|
+193|7|9791|478.52|quests. carefully even requests use regular excuses. pending accounts are. furiously even pinto beans haggle furi|
+194|5|4289|662.17|ic Tiresias serve along the ironic, express accounts. quickly final requests are slyly among the carefully special requests. accounts boost.|
+194|6|377|430.21|efully instead of the special ideas. fluffily unusual asymptotes cajole blithely after the regular ideas. final accounts along the silent ex|
+194|7|5294|913.46|indle fluffily despite the carefully silent instructions. furiously regular hockey players cajole slyly unusual accounts. furiously regular realms cajole furiously according to the e|
+194|8|7890|79.40|ctions sleep. carefully unusual theodolites should wake furiously across the deposits-- furiously bold excuses boost furiously carefully slow accounts. boldly final accounts grow. regular excuse|
+195|6|9985|20.39|efully among the fluffily even accounts! requests are slyly ag|
+195|7|2947|271.39|yly regular requests cajole carefully. carefully fina|
+195|8|319|102.58|ts. ironic foxes wake carefully slyly special pinto beans. blithely silent excuses hinder blithely quietly regular accounts. quickly careful foxes maintain slyly above the slyly express fo|
+195|9|2803|992.27|xes according to the regular foxes wake furiously final theodolites. furiously regular packages sleep slyly express theodolites. slyly thin instructions sleep r|
+196|7|3843|859.90|l platelets use blithely alongside of the enticingly final deposits. fluffily final requests boost furiously ag|
+196|8|2515|966.01|final theodolites. fluffily even deposits are against the|
+196|9|4778|37.61|fully final requests cajole fluffily across the furiously ironic accounts. qui|
+196|10|1068|928.25| cajole about the blithely regular ideas. final ideas hin|
+197|8|9678|753.88|ously. slyly stealthy requests use alongside of the express, unusual packages. final deposits wake. carefully unusual theodolites cajole slyly about the regular foxes. slyly iron|
+197|9|2631|279.05|e blithely. quickly final deposits wake fluffily excuses. even, unusual deposits x-ray among the final accounts. even ideas above the blithely ironic requests sleep furiously slyly final inst|
+197|10|7598|845.51|lets according to the regular deposits wake furiously about the carefully daring theodolites. blithely express dolphins poach after th|
+197|1|8950|897.33|ideas. requests wake above the blithely unusual deposits. slyly regular |
+198|9|6878|587.41|y even accounts poach carefully about the asymptotes. deposits haggle slyly. finally unusual requests run silently regular, bold packages: instructions after the |
+198|10|6493|673.99|y express excuses use blithely among the pending accounts. stealthy ide|
+198|1|8410|166.93|kages. blithely final theodolites dazzle fluffily. accounts boost furiously. furiously unu|
+198|2|6190|697.10|beans nag fluffily about the asymptotes. slyly bold escapades haggle quickly. fluffily special requests haggle above the ironic,|
+199|10|9343|79.70|ending accounts nag across the instructions. carefully express packages over the blithely even pac|
+199|1|8199|46.52|oost slyly. ironic platelets sleep blithely about the slyly silent foxes. furiously even pl|
+199|2|2742|890.63| the special deposits? carefully final deposits about the carefully regular sauternes |
+199|3|7167|884.56|onic platelets use carefully along the slowly stealthy ideas. slyly dogged instructions are quickly above the slyly u|
+200|1|3120|776.41|ntly final packages kindle furiously blithely ironic accounts. carefully final packages according to the carefully |
+200|2|5392|242.52|y unusual ideas. ruthlessly express asymptotes cajole. regular theodolites are. carefully silent deposits poach carefully across the fluffily even theodolites. carefully express realms hag|
+200|3|9408|307.79|oxes! fluffily regular requests use against the unusual, slow ideas. ironic accounts doze b|
+200|4|331|466.07| slyly even requests. fluffily final packages boost carefully express instructions. slyly regular forges are blithely unusual, regular |
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/region.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/region.tbl
new file mode 100644
index 0000000..c5ebb63
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/region.tbl
@@ -0,0 +1,5 @@
+0|AFRICA|lar deposits. blithely final packages cajole. regular waters are final requests. regular accounts are according to |
+1|AMERICA|hs use ironic, even requests. s|
+2|ASIA|ges. thinly even pinto beans ca|
+3|EUROPE|ly final courts cajole furiously final excuse|
+4|MIDDLE EAST|uickly special accounts cajole carefully blithely close requests. carefully final asymptotes haggle furiousl|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/supplier.tbl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/supplier.tbl
new file mode 100644
index 0000000..d9c0e9f
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/supplier.tbl
@@ -0,0 +1,10 @@
+1|Supplier#000000001| N kD4on9OM Ipw3,gf0JBoQDd7tgrzrddZ|17|27-918-335-1736|5755.94|each slyly above the careful|
+2|Supplier#000000002|89eJ5ksX3ImxJQBvxObC,|5|15-679-861-2259|4032.68| slyly bold instructions. idle dependen|
+3|Supplier#000000003|q1,G3Pj6OjIuUYfUoH18BFTKP5aU9bEV3|1|11-383-516-1199|4192.40|blithely silent requests after the express dependencies are sl|
+4|Supplier#000000004|Bk7ah4CK8SYQTepEmvMkkgMwg|15|25-843-787-7479|4641.08|riously even requests above the exp|
+5|Supplier#000000005|Gcdm2rJRzl5qlTVzc|11|21-151-690-3663|-283.84|. slyly regular pinto bea|
+6|Supplier#000000006|tQxuVm7s7CnK|14|24-696-997-4969|1365.79|final accounts. regular dolphins use against the furiously ironic decoys. |
+7|Supplier#000000007|s,4TicNGB4uO6PaSqNBUq|23|33-990-965-2201|6820.35|s unwind silently furiously regular courts. final requests are deposits. requests wake quietly blit|
+8|Supplier#000000008|9Sq4bBH2FQEmaFOocY45sRTxo6yuoG|17|27-498-742-3860|7627.85|al pinto beans. asymptotes haggl|
+9|Supplier#000000009|1KhUgZegwM3ua7dsYmekYBsK|10|20-403-398-8662|5302.37|s. unusual, even requests along the furiously regular pac|
+10|Supplier#000000010|Saygah3gYWMp72i PY|24|34-852-489-8585|3891.91|ing waters. regular requests ar|
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/tpch.ddl b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/tpch.ddl
new file mode 100644
index 0000000..89efe56
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/data/tpch0.001/tpch.ddl
@@ -0,0 +1,70 @@
+-- Sccsid: @(#)dss.ddl 2.1.8.1
+CREATE TABLE NATION ( N_NATIONKEY INTEGER NOT NULL,
+ N_NAME CHAR(25) NOT NULL,
+ N_REGIONKEY INTEGER NOT NULL,
+ N_COMMENT VARCHAR(152));
+
+CREATE TABLE REGION ( R_REGIONKEY INTEGER NOT NULL,
+ R_NAME CHAR(25) NOT NULL,
+ R_COMMENT VARCHAR(152));
+
+CREATE TABLE PART ( P_PARTKEY INTEGER NOT NULL,
+ P_NAME VARCHAR(55) NOT NULL,
+ P_MFGR CHAR(25) NOT NULL,
+ P_BRAND CHAR(10) NOT NULL,
+ P_TYPE VARCHAR(25) NOT NULL,
+ P_SIZE INTEGER NOT NULL,
+ P_CONTAINER CHAR(10) NOT NULL,
+ P_RETAILPRICE DECIMAL(15,2) NOT NULL,
+ P_COMMENT VARCHAR(23) NOT NULL );
+
+CREATE TABLE SUPPLIER ( S_SUPPKEY INTEGER NOT NULL,
+ S_NAME CHAR(25) NOT NULL,
+ S_ADDRESS VARCHAR(40) NOT NULL,
+ S_NATIONKEY INTEGER NOT NULL,
+ S_PHONE CHAR(15) NOT NULL,
+ S_ACCTBAL DECIMAL(15,2) NOT NULL,
+ S_COMMENT VARCHAR(101) NOT NULL);
+
+CREATE TABLE PARTSUPP ( PS_PARTKEY INTEGER NOT NULL,
+ PS_SUPPKEY INTEGER NOT NULL,
+ PS_AVAILQTY INTEGER NOT NULL,
+ PS_SUPPLYCOST DECIMAL(15,2) NOT NULL,
+ PS_COMMENT VARCHAR(199) NOT NULL );
+
+CREATE TABLE CUSTOMER ( C_CUSTKEY INTEGER NOT NULL,
+ C_NAME VARCHAR(25) NOT NULL,
+ C_ADDRESS VARCHAR(40) NOT NULL,
+ C_NATIONKEY INTEGER NOT NULL,
+ C_PHONE CHAR(15) NOT NULL,
+ C_ACCTBAL DECIMAL(15,2) NOT NULL,
+ C_MKTSEGMENT CHAR(10) NOT NULL,
+ C_COMMENT VARCHAR(117) NOT NULL);
+
+CREATE TABLE ORDERS ( O_ORDERKEY INTEGER NOT NULL,
+ O_CUSTKEY INTEGER NOT NULL,
+ O_ORDERSTATUS CHAR(1) NOT NULL,
+ O_TOTALPRICE DECIMAL(15,2) NOT NULL,
+ O_ORDERDATE DATE NOT NULL,
+ O_ORDERPRIORITY CHAR(15) NOT NULL,
+ O_CLERK CHAR(15) NOT NULL,
+ O_SHIPPRIORITY INTEGER NOT NULL,
+ O_COMMENT VARCHAR(79) NOT NULL);
+
+CREATE TABLE LINEITEM ( L_ORDERKEY INTEGER NOT NULL,
+ L_PARTKEY INTEGER NOT NULL,
+ L_SUPPKEY INTEGER NOT NULL,
+ L_LINENUMBER INTEGER NOT NULL,
+ L_QUANTITY DECIMAL(15,2) NOT NULL,
+ L_EXTENDEDPRICE DECIMAL(15,2) NOT NULL,
+ L_DISCOUNT DECIMAL(15,2) NOT NULL,
+ L_TAX DECIMAL(15,2) NOT NULL,
+ L_RETURNFLAG CHAR(1) NOT NULL,
+ L_LINESTATUS CHAR(1) NOT NULL,
+ L_SHIPDATE DATE NOT NULL,
+ L_COMMITDATE DATE NOT NULL,
+ L_RECEIPTDATE DATE NOT NULL,
+ L_SHIPINSTRUCT CHAR(25) NOT NULL,
+ L_SHIPMODE CHAR(10) NOT NULL,
+ L_COMMENT VARCHAR(44) NOT NULL);
+
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/pom.xml b/hyracks-algebricks/hyracks-algebricks-tests/pom.xml
new file mode 100644
index 0000000..d326539
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/pom.xml
@@ -0,0 +1,122 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-algebricks-tests</artifactId>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.7.2</version>
+ <configuration>
+ <forkMode>pertest</forkMode>
+ <argLine>-enableassertions -Djava.util.logging.config.file=src/test/resources/logging.properties</argLine>
+ <includes>
+ <include>**/*Test.java</include>
+ <include>**/*Suite.java</include>
+ </includes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>generate-sources</phase>
+ <configuration>
+ <tasks>
+ <ant antfile="build-script.xml" target="build">
+ <property name="main.class" value="edu.uci.ics.hyracks.algebricks.tests.script.IdentityStreamingScript" />
+ <property name="script.classpath" refid="maven.compile.classpath" />
+ <property name="jvm.params" value="" />
+ <property name="program.params" value="" />
+ <property name="source" value="${basedir}/src/main/scripts/run" />
+ <property name="target.dir" value="${basedir}/target/testscripts" />
+ <property name="target" value="idscript" />
+ </ant>
+ </tasks>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ <pluginManagement>
+ <plugins>
+ <!--This plugin's configuration is used to store Eclipse m2e settings only. It has no influence on the Maven build itself.-->
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>
+ org.apache.maven.plugins
+ </groupId>
+ <artifactId>
+ maven-antrun-plugin
+ </artifactId>
+ <versionRange>[1.3,)</versionRange>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore></ignore>
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-algebricks-compiler</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-cc</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-control-nc</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntArrayUnnester.java b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntArrayUnnester.java
new file mode 100644
index 0000000..ed98eec
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntArrayUnnester.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunction;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class IntArrayUnnester implements IUnnestingFunctionFactory {
+
+ private int[] x;
+
+ public IntArrayUnnester(int[] x) {
+ this.x = x;
+ }
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IUnnestingFunction createUnnestingFunction(IDataOutputProvider provider) throws AlgebricksException {
+
+ final DataOutput out = provider.getDataOutput();
+
+ return new IUnnestingFunction() {
+
+ private int pos;
+
+ @Override
+ public void init(IFrameTupleReference tuple) throws AlgebricksException {
+ pos = 0;
+ }
+
+ @Override
+ public boolean step() throws AlgebricksException {
+ try {
+ if (pos < x.length) {
+ // Writes one byte to distinguish between null
+ // values and end of sequence.
+ out.writeInt(x[pos]);
+ ++pos;
+ return true;
+ } else {
+ return false;
+ }
+
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ };
+
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerAddEvalFactory.java b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerAddEvalFactory.java
new file mode 100644
index 0000000..57c0789
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerAddEvalFactory.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+public class IntegerAddEvalFactory implements IEvaluatorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private IEvaluatorFactory evalLeftFactory;
+ private IEvaluatorFactory evalRightFactory;
+
+ public IntegerAddEvalFactory(IEvaluatorFactory evalLeftFactory, IEvaluatorFactory evalRightFactory) {
+ this.evalLeftFactory = evalLeftFactory;
+ this.evalRightFactory = evalRightFactory;
+ }
+
+ @Override
+ public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new IEvaluator() {
+
+ private DataOutput out = output.getDataOutput();
+ private ArrayBackedValueStorage argOut = new ArrayBackedValueStorage();
+
+ private IEvaluator evalLeft = evalLeftFactory.createEvaluator(argOut);
+ private IEvaluator evalRight = evalRightFactory.createEvaluator(argOut);
+
+ @SuppressWarnings("static-access")
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ argOut.reset();
+ evalLeft.evaluate(tuple);
+ int v1 = IntegerSerializerDeserializer.INSTANCE.getInt(argOut.getBytes(), 0);
+ argOut.reset();
+ evalRight.evaluate(tuple);
+ int v2 = IntegerSerializerDeserializer.INSTANCE.getInt(argOut.getBytes(), 0);
+ try {
+ out.writeInt(v1 + v2);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerConstantEvalFactory.java b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerConstantEvalFactory.java
new file mode 100644
index 0000000..880a4df
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerConstantEvalFactory.java
@@ -0,0 +1,71 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+public class IntegerConstantEvalFactory implements IEvaluatorFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final int value;
+
+ public IntegerConstantEvalFactory(int value) {
+ this.value = value;
+ }
+
+ @Override
+ public String toString() {
+ return "IntConstantEvalFactory " + value;
+ }
+
+ @Override
+ public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new IEvaluator() {
+
+ private DataOutput out = output.getDataOutput();
+ private ArrayBackedValueStorage buf = new ArrayBackedValueStorage();
+ boolean first = true;
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ if (first) {
+ first = false;
+ try {
+ IntegerSerializerDeserializer.INSTANCE.serialize(value, buf.getDataOutput());
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ try {
+ out.write(buf.getBytes(), 0, buf.getLength());
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerEqualsEvalFactory.java b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerEqualsEvalFactory.java
new file mode 100644
index 0000000..e15851b
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerEqualsEvalFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+public class IntegerEqualsEvalFactory implements IEvaluatorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private IEvaluatorFactory evalFact1, evalFact2;
+
+ public IntegerEqualsEvalFactory(IEvaluatorFactory evalFact1, IEvaluatorFactory evalFact2) {
+ this.evalFact1 = evalFact1;
+ this.evalFact2 = evalFact2;
+ }
+
+ @Override
+ public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new IEvaluator() {
+ private DataOutput dataout = output.getDataOutput();
+ private ArrayBackedValueStorage out1 = new ArrayBackedValueStorage();
+ private ArrayBackedValueStorage out2 = new ArrayBackedValueStorage();
+ private IEvaluator eval1 = evalFact1.createEvaluator(out1);
+ private IEvaluator eval2 = evalFact2.createEvaluator(out2);
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ out1.reset();
+ eval1.evaluate(tuple);
+ out2.reset();
+ eval2.evaluate(tuple);
+ int v1 = IntegerSerializerDeserializer.getInt(out1.getBytes(), 0);
+ int v2 = IntegerSerializerDeserializer.getInt(out2.getBytes(), 0);
+ boolean r = v1 == v2;
+ try {
+ dataout.writeBoolean(r);
+ } catch (IOException ioe) {
+ throw new AlgebricksException(ioe);
+ }
+ }
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerGreaterThanEvalFactory.java b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerGreaterThanEvalFactory.java
new file mode 100644
index 0000000..5c5d6d4
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/IntegerGreaterThanEvalFactory.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluator;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.api.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+public class IntegerGreaterThanEvalFactory implements IEvaluatorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private IEvaluatorFactory evalFact1, evalFact2;
+
+ public IntegerGreaterThanEvalFactory(IEvaluatorFactory evalFact1, IEvaluatorFactory evalFact2) {
+ this.evalFact1 = evalFact1;
+ this.evalFact2 = evalFact2;
+ }
+
+ @Override
+ public IEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new IEvaluator() {
+ private DataOutput dataout = output.getDataOutput();
+ private ArrayBackedValueStorage out1 = new ArrayBackedValueStorage();
+ private ArrayBackedValueStorage out2 = new ArrayBackedValueStorage();
+ private IEvaluator eval1 = evalFact1.createEvaluator(out1);
+ private IEvaluator eval2 = evalFact2.createEvaluator(out2);
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ out1.reset();
+ eval1.evaluate(tuple);
+ out2.reset();
+ eval2.evaluate(tuple);
+ int v1 = IntegerSerializerDeserializer.getInt(out1.getBytes(), 0);
+ int v2 = IntegerSerializerDeserializer.getInt(out2.getBytes(), 0);
+ boolean r = v1 > v2;
+ try {
+ dataout.writeBoolean(r);
+ } catch (IOException ioe) {
+ throw new AlgebricksException(ioe);
+ }
+ }
+ };
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/script/IdentityStreamingScript.java b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/script/IdentityStreamingScript.java
new file mode 100644
index 0000000..c914c7c
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/main/java/edu/uci/ics/hyracks/algebricks/tests/script/IdentityStreamingScript.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.tests.script;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+public class IdentityStreamingScript {
+
+ public static void main(String args[]) throws IOException {
+ BufferedReader r = new BufferedReader(new InputStreamReader(System.in));
+ String s;
+ while ((s = r.readLine()) != null) {
+ System.out.println(s);
+ }
+ r.close();
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/main/scripts/run.cmd b/hyracks-algebricks/hyracks-algebricks-tests/src/main/scripts/run.cmd
new file mode 100644
index 0000000..b8eb4a0
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/main/scripts/run.cmd
@@ -0,0 +1,63 @@
+@ECHO OFF
+SETLOCAL
+
+:: Licensed to the Apache Software Foundation (ASF) under one or more
+:: contributor license agreements. See the NOTICE file distributed with
+:: this work for additional information regarding copyright ownership.
+:: The ASF licenses this file to You under the Apache License, Version 2.0
+:: (the "License"); you may not use this file except in compliance with
+:: the License. You may obtain a copy of the License at
+::
+:: http://www.apache.org/licenses/LICENSE-2.0
+::
+:: Unless required by applicable law or agreed to in writing, software
+:: distributed under the License is distributed on an "AS IS" BASIS,
+:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+:: See the License for the specific language governing permissions and
+:: limitations under the License.
+
+:: JAVA classpath
+:: Use the local variable CLASSPATH to add custom entries (e.g. JDBC drivers) to
+:: the classpath. Separate multiple paths with ":". Enclose the value
+:: in double quotes. Adding additional files or locations on separate
+:: lines makes things clearer.
+:: Note: If under running under cygwin use "/cygdrive/c/..." for "C:/..."
+:: Example:
+::
+:: Set the CLASSPATH to a jar file and a directory. Note that
+:: "classes dir" is a directory of class files with a space in the name.
+::
+:: CLASSPATH="usr/local/Product1/lib/product.jar"
+:: CLASSPATH="${CLASSPATH}:../MyProject/classes dir"
+::
+SET CLASSPATH="@classpath@"
+
+:: JVM parameters
+:: If you want to modify the default parameters (e.g. maximum heap size -Xmx)
+:: for the Java virtual machine set the local variable JVM_PARAMETERS below
+:: Example:
+:: JVM_PARAMETERS=-Xms100M -Xmx200M
+::
+:: Below are the JVM parameters needed to do remote debugging using Intellij
+:: IDEA. Uncomment and then do: JVM_PARAMETERS="$IDEA_REMOTE_DEBUG_PARAMS"
+:: IDEA_REMOTE_DEBUG_PARAMS="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"
+::
+:: JVM_PARAMETERS=
+
+:: ---------------------------------------------------------------------------
+:: Default configuration. Do not modify below this line.
+:: ---------------------------------------------------------------------------
+:: Application specific parameters
+
+SET MAIN_CLASS=@main.class@
+SET JVM_PARAMS=@jvm.params@
+SET PROGRAM_PARAMS=@program.params@
+
+:: Try to find java virtual machine
+IF NOT DEFINED JAVA (
+ IF NOT DEFINED JAVA_HOME SET JAVA="java.exe"
+ IF DEFINED JAVA_HOME SET JAVA="%JAVA_HOME%\bin\java.exe"
+)
+
+:: Run program
+%JAVA% %JVM_PARAMS% %JVM_PARAMETERS% -classpath %CLASSPATH% %MAIN_CLASS% %PROGRAM_PARAMS% %*
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/main/scripts/run.sh b/hyracks-algebricks/hyracks-algebricks-tests/src/main/scripts/run.sh
new file mode 100644
index 0000000..a998626
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/main/scripts/run.sh
@@ -0,0 +1,81 @@
+#!/bin/sh
+# JAVA classpath
+# Use the local variable CLASSPATH to add custom entries (e.g. JDBC drivers) to
+# the classpath. Separate multiple paths with ":". Enclose the value
+# in double quotes. Adding additional files or locations on separate
+# lines makes things clearer.
+# Note: If under running under cygwin use "/cygdrive/c/..." for "C:/..."
+# Example:
+#
+# Set the CLASSPATH to a jar file and a directory. Note that
+# "classes dir" is a directory of class files with a space in the name.
+#
+# CLASSPATH="usr/local/Product1/lib/product.jar"
+# CLASSPATH="${CLASSPATH}:../MyProject/classes dir"
+#
+CLASSPATH="@classpath@"
+
+# JVM parameters
+# If you want to modify the default parameters (e.g. maximum heap size -Xmx)
+# for the Java virtual machine set the local variable JVM_PARAMETERS below
+# Example:
+# JVM_PARAMETERS=-Xms100M -Xmx200M
+#
+# Below are the JVM parameters needed to do remote debugging using Intellij
+# IDEA. Uncomment and then do: JVM_PARAMETERS="$IDEA_REMOTE_DEBUG_PARAMS"
+# IDEA_REMOTE_DEBUG_PARAMS="-Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=5005"
+#
+# JVM_PARAMETERS=
+
+#run with shared memory setup
+#if [ -n "${RUN_SHARED_MEM}"]; then
+# JVM_PARAMETERS="${JVM_PARAMETERS} -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_shmem,server=n,address=javadebug,suspend=y"
+#fi
+
+# ---------------------------------------------------------------------------
+# Default configuration. Do not modify below this line.
+# ---------------------------------------------------------------------------
+# Application specific parameters
+
+MAIN_CLASS="@main.class@"
+JVM_PARAMS="@jvm.params@"
+PROGRAM_PARAMS="@program.params@"
+
+# Cygwin support. $cygwin _must_ be set to either true or false.
+case "`uname`" in
+ CYGWIN*) cygwin=true ;;
+ *) cygwin=false ;;
+esac
+
+# For Cygwin, ensure paths are in UNIX format before anything is touched
+if $cygwin; then
+ [ -n "$JAVA_HOME" ] &&
+ JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
+ [ -n "$CLASSPATH" ] &&
+ CLASSPATH=`cygpath --path --unix "$CLASSPATH"`
+fi
+
+# Try to find java virtual machine
+if [ -z "${JAVA}" ]; then
+ if [ -z "${JAVA_HOME}" ]; then
+ JAVA=java
+ else
+ JAVA=${JAVA_HOME}/bin/java
+ fi
+fi
+
+# Try to find directory where this script is located
+COMMAND="${PWD}/$0"
+if [ ! -f "${COMMAND}" ]; then
+ COMMAND="$0"
+fi
+BASEDIR=`expr "${COMMAND}" : '\(.*\)/\.*'`
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin; then
+# JAVA=`cygpath --path --windows "$JAVA"`
+ CLASSPATH=`cygpath --path --windows "$CLASSPATH"`
+fi
+
+# Run program
+${JAVA} ${JVM_PARAMS} ${JVM_PARAMETERS} -classpath "${CLASSPATH}" ${MAIN_CLASS} ${PROGRAM_PARAMS} $*
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java b/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
new file mode 100644
index 0000000..0370632
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/pushruntime/PushRuntimeTest.java
@@ -0,0 +1,898 @@
+package edu.uci.ics.hyracks.algebricks.tests.pushruntime;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+
+import junit.framework.Assert;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.algebricks.core.algebra.data.IPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.aggregators.TupleCountAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.aggregators.TupleCountRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.AlgebricksPipeline;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IEvaluatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IPushRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IRunningAggregateFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.base.IUnnestingFunctionFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.evaluators.ColumnAccessEvalFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.BinaryBooleanInspectorImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.BinaryIntegerInspectorImpl;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.IntegerPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.NoopNullWriterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.jobgen.data.UTF8StringPrinterFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.AggregateRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.NestedPlansAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.aggreg.SimpleAlgebricksAccumulatingAggregatorFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.group.MicroPreClusteredGroupRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.AlgebricksMetaOperatorDescriptor;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.meta.SubplanRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.sort.InMemorySortRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.AssignRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.EmptyTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.NestedTupleSourceRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.PrinterRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.RunningAggregateRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.SinkWriterRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamLimitRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamProjectRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StreamSelectRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.StringStreamingRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.operators.std.UnnestRuntimeFactory;
+import edu.uci.ics.hyracks.algebricks.core.algebra.runtime.writers.PrinterBasedWriterFactory;
+import edu.uci.ics.hyracks.algebricks.tests.util.AlgebricksHyracksIntegrationUtil;
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.FloatParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.LineFileWriteOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.misc.SplitOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
+
+public class PushRuntimeTest {
+
+ private static final String SEPARATOR = System.getProperty("file.separator");
+ private static final String PATH_ACTUAL = "rttest";
+ private static final String PATH_BASE = "src" + SEPARATOR + "test" + SEPARATOR + "resources";
+ private static final String PATH_EXPECTED = PATH_BASE + SEPARATOR + "results";
+
+ private static final String[] DEFAULT_NODES = new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID };
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ File outdir = new File(PATH_ACTUAL);
+ outdir.mkdirs();
+ AlgebricksHyracksIntegrationUtil.init();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ AlgebricksHyracksIntegrationUtil.deinit();
+ File outdir = new File(PATH_ACTUAL);
+ File[] files = outdir.listFiles();
+ if (files == null || files.length == 0) {
+ outdir.delete();
+ }
+ }
+
+ @Test
+ public void etsAssignPrint() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
+ IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
+
+ EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
+ RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
+ AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IEvaluatorFactory[] { const1,
+ const2 }, new int[] { 0, 1 });
+ RecordDescriptor assignDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+
+ PrinterRuntimeFactory printer = new PrinterRuntimeFactory(new int[] { 0, 1 }, new IPrinterFactory[] {
+ IntegerPrinterFactory.INSTANCE, IntegerPrinterFactory.INSTANCE }, assignDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 0, 0,
+ new IPushRuntimeFactory[] { ets, assign, printer },
+ new RecordDescriptor[] { etsDesc, assignDesc, null });
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp, DEFAULT_NODES);
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+ }
+
+ @Test
+ public void etsAssignWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
+ IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
+
+ EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
+ RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
+ AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IEvaluatorFactory[] { const1,
+ const2 }, new int[] { 0, 1 });
+ RecordDescriptor assignDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "etsAssignWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0, 1 }, new IPrinterFactory[] {
+ IntegerPrinterFactory.INSTANCE, IntegerPrinterFactory.INSTANCE }, outFile,
+ PrinterBasedWriterFactory.INSTANCE, assignDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 0, 0,
+ new IPushRuntimeFactory[] { ets, assign, writer }, new RecordDescriptor[] { etsDesc, assignDesc, null });
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp, DEFAULT_NODES);
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("400; 3", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void scanSelectWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ // the scanner
+ FileSplit[] intFileSplits = new FileSplit[1];
+ intFileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(new File(
+ "data/simple/int-part1.tbl")));
+ IFileSplitProvider intSplitProvider = new ConstantFileSplitProvider(intFileSplits);
+ RecordDescriptor intScannerDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+ IValueParserFactory[] valueParsers = new IValueParserFactory[] { IntegerParserFactory.INSTANCE };
+ FileScanOperatorDescriptor intScanner = new FileScanOperatorDescriptor(spec, intSplitProvider,
+ new DelimitedDataTupleParserFactory(valueParsers, '|'), intScannerDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, intScanner, DEFAULT_NODES);
+
+ // the algebricks op.
+ IEvaluatorFactory cond = new IntegerGreaterThanEvalFactory(new IntegerConstantEvalFactory(2),
+ new ColumnAccessEvalFactory(0));
+ StreamSelectRuntimeFactory select = new StreamSelectRuntimeFactory(cond, new int[] { 0 },
+ BinaryBooleanInspectorImpl.INSTANCE);
+ RecordDescriptor selectDesc = intScannerDesc;
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "scanSelectWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ selectDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ new IPushRuntimeFactory[] { select, writer }, new RecordDescriptor[] { selectDesc, null });
+
+ PartitionConstraintHelper.addPartitionCountConstraint(spec, algebricksOp, 1);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), intScanner, 0, algebricksOp, 0);
+
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("0", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void etsAssignProjectWrite() throws Exception {
+
+ JobSpecification spec = new JobSpecification();
+ IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
+ IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
+
+ EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
+ RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
+ AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IEvaluatorFactory[] { const1,
+ const2 }, new int[] { 0, 1 });
+ RecordDescriptor assignDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+ StreamProjectRuntimeFactory project = new StreamProjectRuntimeFactory(new int[] { 1 });
+ RecordDescriptor projectDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "etsAssignProjectWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ projectDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 0, 0,
+ new IPushRuntimeFactory[] { ets, assign, project, writer }, new RecordDescriptor[] { etsDesc,
+ assignDesc, projectDesc, null });
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp, DEFAULT_NODES);
+
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("3", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void scanLimitWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ // the scanner
+ FileSplit[] fileSplits = new FileSplit[1];
+ fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(new File(
+ "data/tpch0.001/customer.tbl")));
+ IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
+
+ RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+ IValueParserFactory[] valueParsers = new IValueParserFactory[] { IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, FloatParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE };
+ FileScanOperatorDescriptor scanner = new FileScanOperatorDescriptor(spec, splitProvider,
+ new DelimitedDataTupleParserFactory(valueParsers, '|'), scannerDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, scanner, DEFAULT_NODES);
+
+ // the algebricks op.
+ StreamLimitRuntimeFactory limit = new StreamLimitRuntimeFactory(new IntegerConstantEvalFactory(2), null,
+ new int[] { 0 }, BinaryIntegerInspectorImpl.INSTANCE);
+ RecordDescriptor limitDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "scanLimitWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ limitDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ new IPushRuntimeFactory[] { limit, writer }, new RecordDescriptor[] { limitDesc, null });
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), scanner, 0, algebricksOp, 0);
+
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("12", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void etsUnnestWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
+ RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
+ IUnnestingFunctionFactory aggregFactory = new IntArrayUnnester(new int[] { 100, 200, 300 });
+ UnnestRuntimeFactory unnest = new UnnestRuntimeFactory(0, aggregFactory, new int[] { 0 });
+ RecordDescriptor unnestDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "etsUnnestWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ unnestDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 0, 0,
+ new IPushRuntimeFactory[] { ets, unnest, writer }, new RecordDescriptor[] { etsDesc, unnestDesc, null });
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("100200300", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void scanAggregateWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ // the scanner
+ FileSplit[] fileSplits = new FileSplit[1];
+ fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(new File(
+ "data/tpch0.001/customer-part1.tbl")));
+ IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
+ RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+ IValueParserFactory[] valueParsers = new IValueParserFactory[] { IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, FloatParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE };
+ FileScanOperatorDescriptor scanner = new FileScanOperatorDescriptor(spec, splitProvider,
+ new DelimitedDataTupleParserFactory(valueParsers, '|'), scannerDesc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, scanner,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ // the algebricks op.
+ AggregateRuntimeFactory agg = new AggregateRuntimeFactory(
+ new IAggregateFunctionFactory[] { new TupleCountAggregateFunctionFactory() });
+ RecordDescriptor aggDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "scanAggregateWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ aggDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ new IPushRuntimeFactory[] { agg, writer }, new RecordDescriptor[] { aggDesc, null });
+
+ PartitionConstraintHelper.addPartitionCountConstraint(spec, algebricksOp, 1);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), scanner, 0, algebricksOp, 0);
+
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("75", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void scanSortGbySelectWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ // the scanner
+ FileSplit[] fileSplits = new FileSplit[1];
+ fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(new File(
+ "data/tpch0.001/customer.tbl")));
+ IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
+ RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+ IValueParserFactory[] valueParsers = new IValueParserFactory[] { IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, FloatParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE };
+ FileScanOperatorDescriptor scanner = new FileScanOperatorDescriptor(spec, splitProvider,
+ new DelimitedDataTupleParserFactory(valueParsers, '|'), scannerDesc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, scanner,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ // the sort (by nation id)
+ RecordDescriptor sortDesc = scannerDesc;
+ InMemorySortOperatorDescriptor sort = new InMemorySortOperatorDescriptor(spec, new int[] { 3 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
+ sortDesc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sort,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ // the group-by
+ NestedTupleSourceRuntimeFactory nts = new NestedTupleSourceRuntimeFactory();
+ RecordDescriptor ntsDesc = sortDesc;
+ AggregateRuntimeFactory agg = new AggregateRuntimeFactory(
+ new IAggregateFunctionFactory[] { new TupleCountAggregateFunctionFactory() });
+ RecordDescriptor aggDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+ AlgebricksPipeline pipeline = new AlgebricksPipeline(new IPushRuntimeFactory[] { nts, agg },
+ new RecordDescriptor[] { ntsDesc, aggDesc });
+ NestedPlansAccumulatingAggregatorFactory npaaf = new NestedPlansAccumulatingAggregatorFactory(
+ new AlgebricksPipeline[] { pipeline }, new int[] { 3 }, new int[] {});
+ RecordDescriptor gbyDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+ PreclusteredGroupOperatorDescriptor gby = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 3 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
+ npaaf, gbyDesc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, gby,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ // the algebricks op.
+ IEvaluatorFactory cond = new IntegerEqualsEvalFactory(new IntegerConstantEvalFactory(3),
+ new ColumnAccessEvalFactory(0)); // Canadian customers
+ StreamSelectRuntimeFactory select = new StreamSelectRuntimeFactory(cond, new int[] { 1 },
+ BinaryBooleanInspectorImpl.INSTANCE);
+ RecordDescriptor selectDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "scanSortGbySelectWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ selectDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ new IPushRuntimeFactory[] { select, writer }, new RecordDescriptor[] { selectDesc, null });
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), scanner, 0, sort, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sort, 0, gby, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), gby, 0, algebricksOp, 0);
+ spec.addRoot(algebricksOp);
+
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("9", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void scanHashGbySelectWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ // the scanner
+ FileSplit[] fileSplits = new FileSplit[1];
+ fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(new File(
+ "data/tpch0.001/customer.tbl")));
+ IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
+ RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+ IValueParserFactory[] valueParsers = new IValueParserFactory[] { IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, FloatParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE };
+ FileScanOperatorDescriptor scanner = new FileScanOperatorDescriptor(spec, splitProvider,
+ new DelimitedDataTupleParserFactory(valueParsers, '|'), scannerDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, scanner,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ // the group-by
+ RecordDescriptor gbyDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+ ITuplePartitionComputerFactory tpcf = new FieldHashPartitionComputerFactory(new int[] { 3 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) });
+ IAggregateFunctionFactory[] aggFuns = new IAggregateFunctionFactory[] { new TupleCountAggregateFunctionFactory() };
+ IAggregatorDescriptorFactory aggFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(aggFuns,
+ new int[] { 3 }, new int[] {});
+ HashGroupOperatorDescriptor gby = new HashGroupOperatorDescriptor(spec, new int[] { 3 }, tpcf,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
+ aggFactory, gbyDesc, 1024);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, gby,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ // the algebricks op.
+ IEvaluatorFactory cond = new IntegerEqualsEvalFactory(new IntegerConstantEvalFactory(3),
+ new ColumnAccessEvalFactory(0)); // Canadian customers
+ StreamSelectRuntimeFactory select = new StreamSelectRuntimeFactory(cond, new int[] { 1 },
+ BinaryBooleanInspectorImpl.INSTANCE);
+ RecordDescriptor selectDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "scanHashGbySelectWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ selectDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ new IPushRuntimeFactory[] { select, writer }, new RecordDescriptor[] { selectDesc, null });
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), scanner, 0, gby, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), gby, 0, algebricksOp, 0);
+ spec.addRoot(algebricksOp);
+
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("9", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void etsUnnestRunningaggregateWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
+ RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
+ IUnnestingFunctionFactory aggregFactory = new IntArrayUnnester(new int[] { 100, 200, 300 });
+ UnnestRuntimeFactory unnest = new UnnestRuntimeFactory(0, aggregFactory, new int[] { 0 });
+ RecordDescriptor unnestDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ RunningAggregateRuntimeFactory ragg = new RunningAggregateRuntimeFactory(new int[] { 1 },
+ new IRunningAggregateFunctionFactory[] { new TupleCountRunningAggregateFunctionFactory() }, new int[] {
+ 0, 1 });
+ RecordDescriptor raggDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "etsUnnestRunningaggregateWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 1 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ raggDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 0, 0,
+ new IPushRuntimeFactory[] { ets, unnest, ragg, writer }, new RecordDescriptor[] { etsDesc, unnestDesc,
+ raggDesc, null });
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("123", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void etsAssignScriptWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
+ IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
+
+ EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
+ RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
+ AssignRuntimeFactory assign = new AssignRuntimeFactory(new int[] { 0, 1 }, new IEvaluatorFactory[] { const1,
+ const2 }, new int[] { 0, 1 });
+ RecordDescriptor assignDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+
+ IValueParserFactory[] valueParsers = { IntegerParserFactory.INSTANCE, IntegerParserFactory.INSTANCE };
+
+ String osname = System.getProperty("os.name");
+ String command;
+ if (osname.equals("Linux")) {
+ command = "bash target/testscripts/idscript";
+ } else if (osname.startsWith("Windows")) {
+ command = "target\\testscripts\\idscript.cmd";
+ } else {
+ // don't know how to test
+ return;
+ }
+
+ StringStreamingRuntimeFactory script = new StringStreamingRuntimeFactory(command, new IPrinterFactory[] {
+ IntegerPrinterFactory.INSTANCE, IntegerPrinterFactory.INSTANCE }, ' ',
+ new DelimitedDataTupleParserFactory(valueParsers, ' '));
+ RecordDescriptor scriptDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "etsAssignScriptWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0, 1 }, new IPrinterFactory[] {
+ IntegerPrinterFactory.INSTANCE, IntegerPrinterFactory.INSTANCE }, outFile,
+ PrinterBasedWriterFactory.INSTANCE, scriptDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 0, 0,
+ new IPushRuntimeFactory[] { ets, assign, script, writer }, new RecordDescriptor[] { etsDesc,
+ assignDesc, scriptDesc, null });
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("400; 3", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void scanSplitWrite() throws Exception {
+ final int outputArity = 2;
+
+ JobSpecification spec = new JobSpecification();
+
+ String inputFileName = "data/tpch0.001/customer.tbl";
+ File inputFile = new File(inputFileName);
+ File[] outputFile = new File[outputArity];
+ for (int i = 0; i < outputArity; i++) {
+ outputFile[i] = File.createTempFile("splitop", null);
+ }
+
+ FileSplit[] inputSplits = new FileSplit[] { new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID,
+ new FileReference(inputFile)) };
+
+ DelimitedDataTupleParserFactory stringParser = new DelimitedDataTupleParserFactory(
+ new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE }, '\u0000');
+ RecordDescriptor stringRec = new RecordDescriptor(
+ new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE, });
+
+ FileScanOperatorDescriptor scanOp = new FileScanOperatorDescriptor(spec, new ConstantFileSplitProvider(
+ inputSplits), stringParser, stringRec);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, scanOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ SplitOperatorDescriptor splitOp = new SplitOperatorDescriptor(spec, stringRec, outputArity);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, splitOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ IOperatorDescriptor outputOp[] = new IOperatorDescriptor[outputFile.length];
+ for (int i = 0; i < outputArity; i++) {
+ outputOp[i] = new LineFileWriteOperatorDescriptor(spec, new FileSplit[] { new FileSplit(
+ AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(outputFile[i])) });
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, outputOp[i],
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+ }
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), scanOp, 0, splitOp, 0);
+ for (int i = 0; i < outputArity; i++) {
+ spec.connect(new OneToOneConnectorDescriptor(spec), splitOp, i, outputOp[i], 0);
+ }
+
+ for (int i = 0; i < outputArity; i++) {
+ spec.addRoot(outputOp[i]);
+ }
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ for (int i = 0; i < outputArity; i++) {
+ compareFiles(inputFileName, outputFile[i].getAbsolutePath());
+ }
+ }
+
+ @Test
+ public void scanMicroSortWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ // the scanner
+ FileSplit[] fileSplits = new FileSplit[1];
+ fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(new File(
+ "data/tpch0.001/nation.tbl")));
+ IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
+ RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+ IValueParserFactory[] valueParsers = new IValueParserFactory[] { IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, IntegerParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE };
+ FileScanOperatorDescriptor scanner = new FileScanOperatorDescriptor(spec, splitProvider,
+ new DelimitedDataTupleParserFactory(valueParsers, '|'), scannerDesc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, scanner,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ // the algebricks op.
+ InMemorySortRuntimeFactory sort = new InMemorySortRuntimeFactory(new int[] { 1 }, null,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ null);
+ RecordDescriptor sortDesc = scannerDesc;
+
+ String fileName = "scanMicroSortWrite.out";
+ String filePath = PATH_ACTUAL + SEPARATOR + fileName;
+ String resultFilePath = PATH_EXPECTED + SEPARATOR + fileName;
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0, 1, 2, 3 }, new IPrinterFactory[] {
+ IntegerPrinterFactory.INSTANCE, UTF8StringPrinterFactory.INSTANCE, IntegerPrinterFactory.INSTANCE,
+ UTF8StringPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE, sortDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ new IPushRuntimeFactory[] { sort, writer }, new RecordDescriptor[] { sortDesc, null });
+
+ PartitionConstraintHelper.addPartitionCountConstraint(spec, algebricksOp, 1);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), scanner, 0, algebricksOp, 0);
+
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ compareFiles(filePath, resultFilePath);
+ outFile.delete();
+ }
+
+ @Test
+ public void etsAssignSubplanProjectWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ IntegerConstantEvalFactory const1 = new IntegerConstantEvalFactory(400);
+ IntegerConstantEvalFactory const2 = new IntegerConstantEvalFactory(3);
+
+ EmptyTupleSourceRuntimeFactory ets = new EmptyTupleSourceRuntimeFactory();
+ RecordDescriptor etsDesc = new RecordDescriptor(new ISerializerDeserializer[] {});
+
+ AssignRuntimeFactory assign1 = new AssignRuntimeFactory(new int[] { 0 }, new IEvaluatorFactory[] { const1 },
+ new int[] { 0 });
+ RecordDescriptor assign1Desc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ NestedTupleSourceRuntimeFactory nts = new NestedTupleSourceRuntimeFactory();
+
+ AssignRuntimeFactory assign2 = new AssignRuntimeFactory(new int[] { 1 },
+ new IEvaluatorFactory[] { new IntegerAddEvalFactory(new ColumnAccessEvalFactory(0), const2) },
+ new int[] { 0, 1 });
+ RecordDescriptor assign2Desc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+
+ StreamProjectRuntimeFactory project1 = new StreamProjectRuntimeFactory(new int[] { 1 });
+ RecordDescriptor project1Desc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ AlgebricksPipeline pipeline = new AlgebricksPipeline(new IPushRuntimeFactory[] { nts, assign2, project1 },
+ new RecordDescriptor[] { assign1Desc, assign2Desc, project1Desc });
+
+ SubplanRuntimeFactory subplan = new SubplanRuntimeFactory(pipeline,
+ new INullWriterFactory[] { NoopNullWriterFactory.INSTANCE }, assign1Desc, null);
+
+ RecordDescriptor subplanDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+
+ StreamProjectRuntimeFactory project2 = new StreamProjectRuntimeFactory(new int[] { 1 });
+ RecordDescriptor project2Desc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "etsAssignSubplanProjectWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ project2Desc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 0, 0,
+ new IPushRuntimeFactory[] { ets, assign1, subplan, project2, writer }, new RecordDescriptor[] {
+ etsDesc, assign1Desc, subplanDesc, project2Desc, null });
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp, DEFAULT_NODES);
+
+ spec.addRoot(algebricksOp);
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("403", buf.toString());
+ outFile.delete();
+ }
+
+ @Test
+ public void scanMicroSortGbySelectWrite() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ // the scanner
+ FileSplit[] fileSplits = new FileSplit[1];
+ fileSplits[0] = new FileSplit(AlgebricksHyracksIntegrationUtil.NC1_ID, new FileReference(new File(
+ "data/tpch0.001/customer.tbl")));
+ IFileSplitProvider splitProvider = new ConstantFileSplitProvider(fileSplits);
+ RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+ IValueParserFactory[] valueParsers = new IValueParserFactory[] { IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, FloatParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE };
+ FileScanOperatorDescriptor scanner = new FileScanOperatorDescriptor(spec, splitProvider,
+ new DelimitedDataTupleParserFactory(valueParsers, '|'), scannerDesc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, scanner,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ // the sort (by nation id)
+ RecordDescriptor sortDesc = scannerDesc;
+ InMemorySortRuntimeFactory sort = new InMemorySortRuntimeFactory(new int[] { 3 }, null,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, null);
+
+ // the group-by
+ NestedTupleSourceRuntimeFactory nts = new NestedTupleSourceRuntimeFactory();
+ RecordDescriptor ntsDesc = sortDesc;
+ AggregateRuntimeFactory agg = new AggregateRuntimeFactory(
+ new IAggregateFunctionFactory[] { new TupleCountAggregateFunctionFactory() });
+ RecordDescriptor aggDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+ AlgebricksPipeline pipeline = new AlgebricksPipeline(new IPushRuntimeFactory[] { nts, agg },
+ new RecordDescriptor[] { ntsDesc, aggDesc });
+ NestedPlansAccumulatingAggregatorFactory npaaf = new NestedPlansAccumulatingAggregatorFactory(
+ new AlgebricksPipeline[] { pipeline }, new int[] { 3 }, new int[] {});
+ RecordDescriptor gbyDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+ MicroPreClusteredGroupRuntimeFactory gby = new MicroPreClusteredGroupRuntimeFactory(new int[] { 3 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
+ npaaf, sortDesc, gbyDesc, null);
+
+ // the algebricks op.
+ IEvaluatorFactory cond = new IntegerEqualsEvalFactory(new IntegerConstantEvalFactory(3),
+ new ColumnAccessEvalFactory(0)); // Canadian customers
+ StreamSelectRuntimeFactory select = new StreamSelectRuntimeFactory(cond, new int[] { 1 },
+ BinaryBooleanInspectorImpl.INSTANCE);
+ RecordDescriptor selectDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ String filePath = PATH_ACTUAL + SEPARATOR + "scanSortGbySelectWrite.out";
+ File outFile = new File(filePath);
+ SinkWriterRuntimeFactory writer = new SinkWriterRuntimeFactory(new int[] { 0 },
+ new IPrinterFactory[] { IntegerPrinterFactory.INSTANCE }, outFile, PrinterBasedWriterFactory.INSTANCE,
+ selectDesc);
+
+ AlgebricksMetaOperatorDescriptor algebricksOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ new IPushRuntimeFactory[] { sort, gby, select, writer }, new RecordDescriptor[] { sortDesc, gbyDesc,
+ selectDesc, null });
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, algebricksOp,
+ new String[] { AlgebricksHyracksIntegrationUtil.NC1_ID });
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), scanner, 0, algebricksOp, 0);
+ spec.addRoot(algebricksOp);
+
+ AlgebricksHyracksIntegrationUtil.runJob(spec);
+ StringBuilder buf = new StringBuilder();
+ readFileToString(outFile, buf);
+ Assert.assertEquals("9", buf.toString());
+ outFile.delete();
+ }
+
+ private static void readFileToString(File file, StringBuilder buf) throws Exception {
+ BufferedReader result = new BufferedReader(new FileReader(file));
+ boolean first = true;
+ while (true) {
+ String s = result.readLine();
+ if (s == null) {
+ break;
+ } else {
+ if (!first) {
+ first = false;
+ buf.append('\n');
+ }
+ buf.append(s);
+ }
+ }
+ result.close();
+ }
+
+ public void compareFiles(String fileNameA, String fileNameB) throws IOException {
+ BufferedReader fileA = new BufferedReader(new FileReader(fileNameA));
+ BufferedReader fileB = new BufferedReader(new FileReader(fileNameB));
+
+ String lineA, lineB;
+ while ((lineA = fileA.readLine()) != null) {
+ lineB = fileB.readLine();
+ Assert.assertEquals(lineA, lineB);
+ }
+ Assert.assertNull(fileB.readLine());
+ }
+
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/tools/WriteValueTest.java b/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/tools/WriteValueTest.java
new file mode 100644
index 0000000..18f0e75
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/tools/WriteValueTest.java
@@ -0,0 +1,97 @@
+package edu.uci.ics.hyracks.algebricks.tests.tools;
+
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.algebricks.core.utils.WriteValueTools;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ByteArrayAccessibleOutputStream;
+
+public class WriteValueTest {
+
+ private ByteArrayAccessibleOutputStream baaos = new ByteArrayAccessibleOutputStream();
+
+ @Test
+ public void writeIntegers() throws Exception {
+ writeIntTest(6);
+ writeIntTest(1234);
+ writeIntTest(-1234);
+ writeIntTest(Integer.MAX_VALUE);
+ writeIntTest(Integer.MAX_VALUE - 1);
+ writeIntTest(Integer.MIN_VALUE);
+ writeIntTest(Integer.MIN_VALUE + 1);
+ }
+
+ @Test
+ public void writeLongs() throws Exception {
+ writeLongTest(Integer.MAX_VALUE);
+ writeLongTest(Integer.MAX_VALUE - 1);
+ writeLongTest(Integer.MIN_VALUE);
+ writeLongTest(Integer.MIN_VALUE + 1);
+ writeLongTest(0L);
+ writeLongTest(1234567890L);
+ writeLongTest(-1234567890L);
+ }
+
+ @Test
+ public void writeUTF8Strings() throws Exception {
+ ByteArrayAccessibleOutputStream interm = new ByteArrayAccessibleOutputStream();
+ DataOutput dout = new DataOutputStream(interm);
+ writeUTF8Test("abcdefABCDEF", dout, interm);
+ writeUTF8Test("šťžľčěďňřůĺ", dout, interm);
+ writeUTF8Test("Ă㪺Ţţ", dout, interm);
+ }
+
+ private void writeIntTest(int i) throws Exception {
+ baaos.reset();
+ WriteValueTools.writeInt(i, baaos);
+ byte[] goal = Integer.toString(i).getBytes();
+ if (baaos.size() != goal.length) {
+ throw new Exception("Expecting to write " + i + " in " + goal.length + " bytes, but found " + baaos.size()
+ + " bytes.");
+ }
+ for (int k = 0; k < goal.length; k++) {
+ if (goal[k] != baaos.getByteArray()[k]) {
+ throw new Exception("Expecting to write " + i + " as " + goal + ", but found " + baaos.getByteArray()
+ + " instead.");
+ }
+ }
+ }
+
+ private void writeLongTest(long x) throws Exception {
+ baaos.reset();
+ WriteValueTools.writeLong(x, baaos);
+ byte[] goal = Long.toString(x).getBytes();
+ if (baaos.size() != goal.length) {
+ throw new Exception("Expecting to write " + x + " in " + goal.length + " bytes, but found " + baaos.size()
+ + " bytes.");
+ }
+ for (int k = 0; k < goal.length; k++) {
+ if (goal[k] != baaos.getByteArray()[k]) {
+ throw new Exception("Expecting to write " + x + " as " + goal + ", but found " + baaos.getByteArray()
+ + " instead.");
+ }
+ }
+ }
+
+ private void writeUTF8Test(String str, DataOutput dout, ByteArrayAccessibleOutputStream interm) throws Exception {
+ interm.reset();
+ dout.writeUTF(str);
+ baaos.reset();
+ WriteValueTools.writeUTF8String(interm.getByteArray(), 0, interm.size(), baaos);
+ byte[] b = str.getBytes("UTF-8");
+ if (baaos.size() != b.length + 2) {
+ throw new Exception("Expecting to write " + b + " in " + b.length + " bytes, but found " + baaos.size()
+ + " bytes.");
+ }
+ if (baaos.getByteArray()[0] != '\"' || baaos.getByteArray()[baaos.size() - 1] != '\"') {
+ throw new Exception("Missing quotes.");
+ }
+ for (int k = 0; k < b.length; k++) {
+ if (b[k] != baaos.getByteArray()[k + 1]) {
+ throw new Exception("Expecting to write " + b + ", but found " + baaos.getByteArray() + " instead.");
+ }
+ }
+ }
+}
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java b/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java
new file mode 100644
index 0000000..922b0fd
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/test/java/edu/uci/ics/hyracks/algebricks/tests/util/AlgebricksHyracksIntegrationUtil.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.algebricks.tests.util;
+
+import java.util.EnumSet;
+
+import edu.uci.ics.hyracks.algebricks.core.config.AlgebricksConfig;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
+import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.common.controllers.CCConfig;
+import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+
+public class AlgebricksHyracksIntegrationUtil {
+
+ public static final String NC1_ID = "nc1";
+ public static final String NC2_ID = "nc2";
+
+ public static final int TEST_HYRACKS_CC_CLUSTER_NET_PORT = 4322;
+ public static final int TEST_HYRACKS_CC_CLIENT_NET_PORT = 4321;
+
+ private static ClusterControllerService cc;
+ private static NodeControllerService nc1;
+ private static NodeControllerService nc2;
+ private static IHyracksClientConnection hcc;
+
+ public static void init() throws Exception {
+ CCConfig ccConfig = new CCConfig();
+ ccConfig.clientNetIpAddress = "127.0.0.1";
+ ccConfig.clientNetPort = TEST_HYRACKS_CC_CLIENT_NET_PORT;
+ ccConfig.clusterNetIpAddress = "127.0.0.1";
+ ccConfig.clusterNetPort = TEST_HYRACKS_CC_CLUSTER_NET_PORT;
+ // ccConfig.useJOL = true;
+ cc = new ClusterControllerService(ccConfig);
+ cc.start();
+
+ NCConfig ncConfig1 = new NCConfig();
+ ncConfig1.ccHost = "localhost";
+ ncConfig1.ccPort = TEST_HYRACKS_CC_CLUSTER_NET_PORT;
+ ncConfig1.clusterNetIPAddress = "127.0.0.1";
+ ncConfig1.dataIPAddress = "127.0.0.1";
+ ncConfig1.nodeId = NC1_ID;
+ nc1 = new NodeControllerService(ncConfig1);
+ nc1.start();
+
+ NCConfig ncConfig2 = new NCConfig();
+ ncConfig2.ccHost = "localhost";
+ ncConfig2.ccPort = TEST_HYRACKS_CC_CLUSTER_NET_PORT;
+ ncConfig2.clusterNetIPAddress = "127.0.0.1";
+ ncConfig2.dataIPAddress = "127.0.0.1";
+ ncConfig2.nodeId = NC2_ID;
+ nc2 = new NodeControllerService(ncConfig2);
+ nc2.start();
+
+ hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
+ hcc.createApplication(AlgebricksConfig.HYRACKS_APP_NAME, null);
+ }
+
+ public static void deinit() throws Exception {
+ nc2.stop();
+ nc1.stop();
+ cc.stop();
+ }
+
+ public static void runJob(JobSpecification spec) throws Exception {
+ JobId jobId = hcc.createJob(AlgebricksConfig.HYRACKS_APP_NAME, spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
+ AlgebricksConfig.ALGEBRICKS_LOGGER.info(spec.toJSON().toString());
+ hcc.start(jobId);
+ AlgebricksConfig.ALGEBRICKS_LOGGER.info(jobId.toString());
+ hcc.waitForCompletion(jobId);
+ }
+
+}
\ No newline at end of file
diff --git a/hyracks-algebricks/hyracks-algebricks-tests/src/test/resources/results/scanMicroSortWrite.out b/hyracks-algebricks/hyracks-algebricks-tests/src/test/resources/results/scanMicroSortWrite.out
new file mode 100644
index 0000000..1c0fd6a
--- /dev/null
+++ b/hyracks-algebricks/hyracks-algebricks-tests/src/test/resources/results/scanMicroSortWrite.out
@@ -0,0 +1,25 @@
+0; "ALGERIA"; 0; " haggle. carefully final deposits detect slyly agai"
+1; "ARGENTINA"; 1; "al foxes promise slyly according to the regular accounts. bold requests alon"
+2; "BRAZIL"; 1; "y alongside of the pending deposits. carefully special packages are about the ironic forges. slyly special "
+3; "CANADA"; 1; "eas hang ironic, silent packages. slyly regular packages are furiously over the tithes. fluffily bold"
+18; "CHINA"; 2; "c dependencies. furiously express notornis sleep slyly regular accounts. ideas sleep. depos"
+4; "EGYPT"; 4; "y above the carefully unusual theodolites. final dugouts are quickly across the furiously regular d"
+5; "ETHIOPIA"; 0; "ven packages wake quickly. regu"
+6; "FRANCE"; 3; "refully final requests. regular, ironi"
+7; "GERMANY"; 3; "l platelets. regular accounts x-ray: unusual, regular acco"
+8; "INDIA"; 2; "ss excuses cajole slyly across the packages. deposits print aroun"
+9; "INDONESIA"; 2; " slyly express asymptotes. regular deposits haggle slyly. carefully ironic hockey players sleep blithely. carefull"
+10; "IRAN"; 4; "efully alongside of the slyly final dependencies. "
+11; "IRAQ"; 4; "nic deposits boost atop the quickly final requests? quickly regula"
+12; "JAPAN"; 2; "ously. final, express gifts cajole a"
+13; "JORDAN"; 4; "ic deposits are blithely about the carefully regular pa"
+14; "KENYA"; 0; " pending excuses haggle furiously deposits. pending, express pinto beans wake fluffily past t"
+15; "MOROCCO"; 0; "rns. blithely bold courts among the closely regular packages use furiously bold platelets?"
+16; "MOZAMBIQUE"; 0; "s. ironic, unusual asymptotes wake blithely r"
+17; "PERU"; 1; "platelets. blithely pending dependencies use fluffily across the even pinto beans. carefully silent accoun"
+19; "ROMANIA"; 3; "ular asymptotes are about the furious multipliers. express dependencies nag above the ironically ironic account"
+22; "RUSSIA"; 3; " requests against the platelets use never according to the quickly regular pint"
+20; "SAUDI ARABIA"; 4; "ts. silent requests haggle. closely express packages sleep across the blithely"
+23; "UNITED KINGDOM"; 3; "eans boost carefully special requests. accounts are. carefull"
+24; "UNITED STATES"; 1; "y final packages. slow foxes cajole quickly. quickly silent platelets breach ironic accounts. unusual pinto be"
+21; "VIETNAM"; 2; "hely enticingly express accounts. even, final "
diff --git a/hyracks-algebricks/pom.xml b/hyracks-algebricks/pom.xml
new file mode 100644
index 0000000..01987f5
--- /dev/null
+++ b/hyracks-algebricks/pom.xml
@@ -0,0 +1,19 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-algebricks</artifactId>
+ <packaging>pom</packaging>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <modules>
+ <module>hyracks-algebricks-compiler</module>
+ <module>hyracks-algebricks-core</module>
+ <module>hyracks-algebricks-rewriter</module>
+ <module>hyracks-algebricks-tests</module>
+ <module>hyracks-algebricks-examples</module>
+ </modules>
+</project>
diff --git a/hyracks-api/.classpath b/hyracks-api/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-api/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-api/.project b/hyracks-api/.project
deleted file mode 100644
index 4ae9e54..0000000
--- a/hyracks-api/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-api</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-api/.settings/org.eclipse.jdt.core.prefs b/hyracks-api/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 6b7a0fc..0000000
--- a/hyracks-api/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Thu Jun 02 12:55:19 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-api/.settings/org.maven.ide.eclipse.prefs b/hyracks-api/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 930b4a1..0000000
--- a/hyracks-api/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Jul 29 01:10:05 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-api/pom.xml b/hyracks-api/pom.xml
index 2301a93..c18995d 100644
--- a/hyracks-api/pom.xml
+++ b/hyracks-api/pom.xml
@@ -1,13 +1,10 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -45,5 +42,15 @@
<type>jar</type>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-ipc</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-lang3</artifactId>
+ <version>3.1</version>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IApplicationContext.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IApplicationContext.java
index c739a5e..c07de41 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IApplicationContext.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IApplicationContext.java
@@ -16,8 +16,27 @@
import java.io.Serializable;
+/**
+ * Base class of the {@link ICCApplicationContext} and the
+ * {@link INCApplicationContext}.
+ *
+ * @author vinayakb
+ *
+ */
public interface IApplicationContext {
+ /**
+ * Provides the Class Loader that loads classes for this Hyracks Application
+ * at the CC.
+ *
+ * @return the application {@link ClassLoader}.
+ */
public ClassLoader getClassLoader();
- public Serializable getDestributedState();
+ /**
+ * Gets the distributed state that is made available to all the Application
+ * Contexts of this application in the cluster.
+ *
+ * @return
+ */
+ public Serializable getDistributedState();
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IBootstrap.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IBootstrap.java
index efbf0df..19f97b4 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IBootstrap.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/IBootstrap.java
@@ -14,8 +14,23 @@
*/
package edu.uci.ics.hyracks.api.application;
+/**
+ * Base class of {@link ICCBootstrap} and {@link INCBootstrap}.
+ *
+ * @author vinayakb
+ */
public interface IBootstrap {
+ /**
+ * Method called to start the application at a Hyracks CC or NC node.
+ *
+ * @throws Exception
+ */
public void start() throws Exception;
+ /**
+ * Method called to shutdown the application at a Hyracks CC or NC node.
+ *
+ * @throws Exception
+ */
public void stop() throws Exception;
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCApplicationContext.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCApplicationContext.java
index 0987bd2..2e02101 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCApplicationContext.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCApplicationContext.java
@@ -19,13 +19,51 @@
import edu.uci.ics.hyracks.api.context.ICCContext;
import edu.uci.ics.hyracks.api.job.IJobLifecycleListener;
import edu.uci.ics.hyracks.api.job.IJobSpecificationFactory;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+/**
+ * Application Context at the Cluster Controller for an application.
+ *
+ * @author vinayakb
+ *
+ */
public interface ICCApplicationContext extends IApplicationContext {
+ /**
+ * Sets the state that must be distributed by the infrastructure to all the
+ * NC application contects. Any state set by calling thsi method in the
+ * {@link ICCBootstrap#start()} call is made available to all the
+ * {@link INCApplicationContext} objects at each Node Controller. The state
+ * is then available to be inspected by the application at the NC during or
+ * after the {@link INCBootstrap#start()} call.
+ *
+ * @param state
+ * The distributed state
+ */
public void setDistributedState(Serializable state);
+ /**
+ * A factory class specific to this application that may accept incoming
+ * {@link JobSpecification} and produce modified {@link JobSpecification}
+ * that is executed on the cluster. If a {@link IJobSpecificationFactory} is
+ * not set, the incoming {@link JobSpecification} is executed unmodified.
+ *
+ * @param jobSpecFactory
+ * - The Job Specification Factory.
+ */
public void setJobSpecificationFactory(IJobSpecificationFactory jobSpecFactory);
+ /**
+ * A listener that listens to Job Lifecycle events at the Cluster
+ * Controller.
+ *
+ * @param jobLifecycleListener
+ */
public void addJobLifecycleListener(IJobLifecycleListener jobLifecycleListener);
-
+
+ /**
+ * Get the Cluster Controller Context.
+ *
+ * @return The Cluster Controller Context.
+ */
public ICCContext getCCContext();
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCBootstrap.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCBootstrap.java
index 3acbe5e..e3906ea 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCBootstrap.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/ICCBootstrap.java
@@ -1,5 +1,19 @@
package edu.uci.ics.hyracks.api.application;
+/**
+ * Implemented by the bootstrap class of the application that will manage its
+ * life cycle at the Cluster Controller.
+ *
+ * @author vinayakb
+ *
+ */
public interface ICCBootstrap extends IBootstrap {
+ /**
+ * Called by the infrastructure to set the CC Application Context for the
+ * application. The infrastructure makes this call prior to calling start().
+ *
+ * @param appCtx
+ * - The CC application context
+ */
public void setApplicationContext(ICCApplicationContext appCtx);
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/INCApplicationContext.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/INCApplicationContext.java
index eed5bb3..24a598b 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/INCApplicationContext.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/INCApplicationContext.java
@@ -14,18 +14,43 @@
*/
package edu.uci.ics.hyracks.api.application;
-import java.io.Serializable;
-
import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+/**
+ * Application Context at the Node Controller for an application.
+ *
+ * @author vinayakb
+ *
+ */
public interface INCApplicationContext extends IApplicationContext {
+ /**
+ * Gets the node Id of the Node Congtroller.
+ *
+ * @return the Node Id.
+ */
public String getNodeId();
+ /**
+ * Get the Hyracks Root Context.
+ *
+ * @return The Hyracks Root Context
+ */
public IHyracksRootContext getRootContext();
- public void setDistributedState(Serializable state);
-
+ /**
+ * Set an object that can be later retrieved by the
+ * {@link #getApplicationObject()} call.
+ *
+ * @param object
+ * Application Object
+ */
public void setApplicationObject(Object object);
+ /**
+ * Get the application object previously set by the
+ * {@link #setApplicationObject(Object)} call.
+ *
+ * @return Application Object
+ */
public Object getApplicationObject();
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/INCBootstrap.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/INCBootstrap.java
index 4aa1ac3..300f7c7 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/INCBootstrap.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/application/INCBootstrap.java
@@ -1,5 +1,19 @@
package edu.uci.ics.hyracks.api.application;
+/**
+ * Implemented by the bootstrap class of the application that will manage its
+ * life cycle at a Node Controller.
+ *
+ * @author vinayakb
+ *
+ */
public interface INCBootstrap extends IBootstrap {
+ /**
+ * Called by the infrastructure to set the NC Application Context for the
+ * application. The infrastructure makes this call prior to calling start().
+ *
+ * @param appCtx
+ * - The NC application context
+ */
public void setApplicationContext(INCApplicationContext appCtx);
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/channels/IInputChannel.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/channels/IInputChannel.java
index e621a09..f79a464 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/channels/IInputChannel.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/channels/IInputChannel.java
@@ -17,10 +17,9 @@
import java.nio.ByteBuffer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
public interface IInputChannel {
- public void registerMonitor(IInputChannelMonitor monitor) throws HyracksException;
+ public void registerMonitor(IInputChannelMonitor monitor);
public void setAttachment(Object attachment);
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/AbstractHyracksConnection.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/AbstractHyracksConnection.java
deleted file mode 100644
index 1377e37..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/AbstractHyracksConnection.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.client;
-
-import java.io.File;
-import java.util.EnumSet;
-
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpPut;
-import org.apache.http.entity.FileEntity;
-import org.apache.http.impl.client.DefaultHttpClient;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobFlag;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
-
-abstract class AbstractHyracksConnection implements IHyracksClientConnection {
- private final String ccHost;
-
- private final IHyracksClientInterface hci;
-
- private final ClusterControllerInfo ccInfo;
-
- public AbstractHyracksConnection(String ccHost, IHyracksClientInterface hci) throws Exception {
- this.ccHost = ccHost;
- this.hci = hci;
- ccInfo = hci.getClusterControllerInfo();
- }
-
- @Override
- public void createApplication(String appName, File harFile) throws Exception {
- hci.createApplication(appName);
- if (harFile != null) {
- HttpClient hc = new DefaultHttpClient();
- HttpPut put = new HttpPut("http://" + ccHost + ":" + ccInfo.getWebPort() + "/applications/" + appName);
- put.setEntity(new FileEntity(harFile, "application/octet-stream"));
- HttpResponse response = hc.execute(put);
- if (response.getStatusLine().getStatusCode() != 200) {
- hci.destroyApplication(appName);
- throw new HyracksException(response.getStatusLine().toString());
- }
- }
- hci.startApplication(appName);
- }
-
- @Override
- public void destroyApplication(String appName) throws Exception {
- hci.destroyApplication(appName);
- }
-
- @Override
- public JobId createJob(String appName, JobSpecification jobSpec) throws Exception {
- return createJob(appName, jobSpec, EnumSet.noneOf(JobFlag.class));
- }
-
- @Override
- public JobId createJob(String appName, JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
- return hci.createJob(appName, JavaSerializationUtils.serialize(jobSpec), jobFlags);
- }
-
- @Override
- public JobStatus getJobStatus(JobId jobId) throws Exception {
- return hci.getJobStatus(jobId);
- }
-
- @Override
- public void start(JobId jobId) throws Exception {
- hci.start(jobId);
- }
-
- @Override
- public void waitForCompletion(JobId jobId) throws Exception {
- hci.waitForCompletion(jobId);
- }
-}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/ClusterControllerInfo.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/ClusterControllerInfo.java
index 315f9dc..8dd4d6c 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/ClusterControllerInfo.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/ClusterControllerInfo.java
@@ -19,16 +19,27 @@
public class ClusterControllerInfo implements Serializable {
private static final long serialVersionUID = 1L;
- private int webPort;
+ private final String clientNetAddress;
- public ClusterControllerInfo() {
- }
+ private final int clientNetPort;
- public void setWebPort(int webPort) {
+ private final int webPort;
+
+ public ClusterControllerInfo(String clientNetAddress, int clientNetPort, int webPort) {
+ this.clientNetAddress = clientNetAddress;
+ this.clientNetPort = clientNetPort;
this.webPort = webPort;
}
public int getWebPort() {
return webPort;
}
+
+ public String getClientNetAddress() {
+ return clientNetAddress;
+ }
+
+ public int getClientNetPort() {
+ return clientNetPort;
+ }
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java
new file mode 100644
index 0000000..a7f324d
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceFunctions.java
@@ -0,0 +1,204 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.client;
+
+import java.io.Serializable;
+import java.util.EnumSet;
+
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+
+public class HyracksClientInterfaceFunctions {
+ public enum FunctionId {
+ GET_CLUSTER_CONTROLLER_INFO,
+ CREATE_APPLICATION,
+ START_APPLICATION,
+ DESTROY_APPLICATION,
+ CREATE_JOB,
+ GET_JOB_STATUS,
+ START_JOB,
+ WAIT_FOR_COMPLETION,
+ GET_NODE_CONTROLLERS_INFO
+ }
+
+ public abstract static class Function implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ public abstract FunctionId getFunctionId();
+ }
+
+ public static class GetClusterControllerInfoFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.GET_CLUSTER_CONTROLLER_INFO;
+ }
+ }
+
+ public static class CreateApplicationFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String appName;
+
+ public CreateApplicationFunction(String appName) {
+ this.appName = appName;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.CREATE_APPLICATION;
+ }
+
+ public String getAppName() {
+ return appName;
+ }
+ }
+
+ public static class StartApplicationFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String appName;
+
+ public StartApplicationFunction(String appName) {
+ this.appName = appName;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.START_APPLICATION;
+ }
+
+ public String getAppName() {
+ return appName;
+ }
+ }
+
+ public static class DestroyApplicationFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String appName;
+
+ public DestroyApplicationFunction(String appName) {
+ this.appName = appName;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.DESTROY_APPLICATION;
+ }
+
+ public String getAppName() {
+ return appName;
+ }
+ }
+
+ public static class CreateJobFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String appName;
+ private final byte[] jobSpec;
+ private final EnumSet<JobFlag> jobFlags;
+
+ public CreateJobFunction(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) {
+ this.appName = appName;
+ this.jobSpec = jobSpec;
+ this.jobFlags = jobFlags;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.CREATE_JOB;
+ }
+
+ public String getAppName() {
+ return appName;
+ }
+
+ public byte[] getJobSpec() {
+ return jobSpec;
+ }
+
+ public EnumSet<JobFlag> getJobFlags() {
+ return jobFlags;
+ }
+ }
+
+ public static class GetJobStatusFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final JobId jobId;
+
+ public GetJobStatusFunction(JobId jobId) {
+ this.jobId = jobId;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.GET_JOB_STATUS;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+ }
+
+ public static class StartJobFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final JobId jobId;
+
+ public StartJobFunction(JobId jobId) {
+ this.jobId = jobId;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.START_JOB;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+ }
+
+ public static class WaitForCompletionFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final JobId jobId;
+
+ public WaitForCompletionFunction(JobId jobId) {
+ this.jobId = jobId;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.WAIT_FOR_COMPLETION;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+ }
+
+ public static class GetNodeControllersInfoFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.GET_NODE_CONTROLLERS_INFO;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
new file mode 100644
index 0000000..602288c
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksClientInterfaceRemoteProxy.java
@@ -0,0 +1,96 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.client;
+
+import java.util.EnumSet;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.RPCInterface;
+
+public class HyracksClientInterfaceRemoteProxy implements IHyracksClientInterface {
+ private final IIPCHandle ipcHandle;
+
+ private final RPCInterface rpci;
+
+ public HyracksClientInterfaceRemoteProxy(IIPCHandle ipcHandle, RPCInterface rpci) {
+ this.ipcHandle = ipcHandle;
+ this.rpci = rpci;
+ }
+
+ @Override
+ public ClusterControllerInfo getClusterControllerInfo() throws Exception {
+ HyracksClientInterfaceFunctions.GetClusterControllerInfoFunction gccif = new HyracksClientInterfaceFunctions.GetClusterControllerInfoFunction();
+ return (ClusterControllerInfo) rpci.call(ipcHandle, gccif);
+ }
+
+ @Override
+ public void createApplication(String appName) throws Exception {
+ HyracksClientInterfaceFunctions.CreateApplicationFunction caf = new HyracksClientInterfaceFunctions.CreateApplicationFunction(
+ appName);
+ rpci.call(ipcHandle, caf);
+ }
+
+ @Override
+ public void startApplication(String appName) throws Exception {
+ HyracksClientInterfaceFunctions.StartApplicationFunction saf = new HyracksClientInterfaceFunctions.StartApplicationFunction(
+ appName);
+ rpci.call(ipcHandle, saf);
+ }
+
+ @Override
+ public void destroyApplication(String appName) throws Exception {
+ HyracksClientInterfaceFunctions.DestroyApplicationFunction daf = new HyracksClientInterfaceFunctions.DestroyApplicationFunction(
+ appName);
+ rpci.call(ipcHandle, daf);
+ }
+
+ @Override
+ public JobId createJob(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
+ HyracksClientInterfaceFunctions.CreateJobFunction cjf = new HyracksClientInterfaceFunctions.CreateJobFunction(
+ appName, jobSpec, jobFlags);
+ return (JobId) rpci.call(ipcHandle, cjf);
+ }
+
+ @Override
+ public JobStatus getJobStatus(JobId jobId) throws Exception {
+ HyracksClientInterfaceFunctions.GetJobStatusFunction gjsf = new HyracksClientInterfaceFunctions.GetJobStatusFunction(
+ jobId);
+ return (JobStatus) rpci.call(ipcHandle, gjsf);
+ }
+
+ @Override
+ public void startJob(JobId jobId) throws Exception {
+ HyracksClientInterfaceFunctions.StartJobFunction sjf = new HyracksClientInterfaceFunctions.StartJobFunction(
+ jobId);
+ rpci.call(ipcHandle, sjf);
+ }
+
+ @Override
+ public void waitForCompletion(JobId jobId) throws Exception {
+ HyracksClientInterfaceFunctions.WaitForCompletionFunction wfcf = new HyracksClientInterfaceFunctions.WaitForCompletionFunction(
+ jobId);
+ rpci.call(ipcHandle, wfcf);
+ }
+
+ @Override
+ public Map<String, NodeControllerInfo> getNodeControllersInfo() throws Exception {
+ HyracksClientInterfaceFunctions.GetNodeControllersInfoFunction gncif = new HyracksClientInterfaceFunctions.GetNodeControllersInfoFunction();
+ return (Map<String, NodeControllerInfo>) rpci.call(ipcHandle, gncif);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java
new file mode 100644
index 0000000..8c9409c
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksConnection.java
@@ -0,0 +1,126 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.client;
+
+import java.io.File;
+import java.net.InetSocketAddress;
+import java.util.EnumSet;
+import java.util.Map;
+
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpPut;
+import org.apache.http.entity.FileEntity;
+import org.apache.http.impl.client.DefaultHttpClient;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.RPCInterface;
+import edu.uci.ics.hyracks.ipc.impl.IPCSystem;
+import edu.uci.ics.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer;
+
+/**
+ * Connection Class used by a Hyracks Client to interact with a Hyracks Cluster
+ * Controller.
+ *
+ * @author vinayakb
+ *
+ */
+public final class HyracksConnection implements IHyracksClientConnection {
+ private final String ccHost;
+
+ private final IPCSystem ipc;
+
+ private final IHyracksClientInterface hci;
+
+ private final ClusterControllerInfo ccInfo;
+
+ /**
+ * Constructor to create a connection to the Hyracks Cluster Controller.
+ *
+ * @param ccHost
+ * Host name (or IP Address) where the Cluster Controller can be
+ * reached.
+ * @param ccPort
+ * Port to reach the Hyracks Cluster Controller at the specified
+ * host name.
+ * @throws Exception
+ */
+ public HyracksConnection(String ccHost, int ccPort) throws Exception {
+ this.ccHost = ccHost;
+ RPCInterface rpci = new RPCInterface();
+ ipc = new IPCSystem(new InetSocketAddress(0), rpci, new JavaSerializationBasedPayloadSerializerDeserializer());
+ ipc.start();
+ IIPCHandle ccIpchandle = ipc.getHandle(new InetSocketAddress(ccHost, ccPort));
+ this.hci = new HyracksClientInterfaceRemoteProxy(ccIpchandle, rpci);
+ ccInfo = hci.getClusterControllerInfo();
+ }
+
+ @Override
+ public void createApplication(String appName, File harFile) throws Exception {
+ hci.createApplication(appName);
+ if (harFile != null) {
+ HttpClient hc = new DefaultHttpClient();
+ HttpPut put = new HttpPut("http://" + ccHost + ":" + ccInfo.getWebPort() + "/applications/" + appName);
+ put.setEntity(new FileEntity(harFile, "application/octet-stream"));
+ HttpResponse response = hc.execute(put);
+ if (response.getStatusLine().getStatusCode() != 200) {
+ hci.destroyApplication(appName);
+ throw new HyracksException(response.getStatusLine().toString());
+ }
+ }
+ hci.startApplication(appName);
+ }
+
+ @Override
+ public void destroyApplication(String appName) throws Exception {
+ hci.destroyApplication(appName);
+ }
+
+ @Override
+ public JobId createJob(String appName, JobSpecification jobSpec) throws Exception {
+ return createJob(appName, jobSpec, EnumSet.noneOf(JobFlag.class));
+ }
+
+ @Override
+ public JobId createJob(String appName, JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
+ return hci.createJob(appName, JavaSerializationUtils.serialize(jobSpec), jobFlags);
+ }
+
+ @Override
+ public JobStatus getJobStatus(JobId jobId) throws Exception {
+ return hci.getJobStatus(jobId);
+ }
+
+ @Override
+ public void start(JobId jobId) throws Exception {
+ hci.startJob(jobId);
+ }
+
+ @Override
+ public void waitForCompletion(JobId jobId) throws Exception {
+ hci.waitForCompletion(jobId);
+ }
+
+ @Override
+ public Map<String, NodeControllerInfo> getNodeControllerInfos() throws Exception {
+ return hci.getNodeControllersInfo();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksLocalConnection.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksLocalConnection.java
deleted file mode 100644
index b41e92c..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksLocalConnection.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.client;
-
-public final class HyracksLocalConnection extends AbstractHyracksConnection {
- public HyracksLocalConnection(IHyracksClientInterface hci) throws Exception {
- super("localhost", hci);
- }
-}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksRMIConnection.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksRMIConnection.java
deleted file mode 100644
index 930126d..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/HyracksRMIConnection.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.client;
-
-import java.rmi.NotBoundException;
-import java.rmi.RemoteException;
-import java.rmi.registry.LocateRegistry;
-import java.rmi.registry.Registry;
-
-public final class HyracksRMIConnection extends AbstractHyracksConnection {
- public HyracksRMIConnection(String host, int port) throws Exception {
- super(host, lookupHCI(host, port));
- }
-
- private static IHyracksClientInterface lookupHCI(String host, int port) throws RemoteException, NotBoundException {
- Registry registry = LocateRegistry.getRegistry(host, port);
- return (IHyracksClientInterface) registry.lookup(IHyracksClientInterface.class.getName());
- }
-}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientConnection.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientConnection.java
index 659a181..37a746b 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientConnection.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientConnection.java
@@ -16,24 +16,102 @@
import java.io.File;
import java.util.EnumSet;
+import java.util.Map;
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.api.job.JobStatus;
+/**
+ * Interface used by clients to communicate with the Hyracks Cluster Controller.
+ *
+ * @author vinayakb
+ *
+ */
public interface IHyracksClientConnection {
+ /**
+ * Create a Hyracks Application
+ *
+ * @param appName
+ * Name of the application
+ * @param harFile
+ * Archive that contains deployable code for the application
+ * @throws Exception
+ */
public void createApplication(String appName, File harFile) throws Exception;
+ /**
+ * Destroy an already-deployed Hyracks application
+ *
+ * @param appName
+ * Name of the application
+ * @throws Exception
+ */
public void destroyApplication(String appName) throws Exception;
+ /**
+ * Creates a Job Instance in the specified Hyracks application using the
+ * specified {@link JobSpecification}.
+ *
+ * @param appName
+ * Name of the application
+ * @param jobSpec
+ * Job Specification
+ * @return
+ * @throws Exception
+ */
public JobId createJob(String appName, JobSpecification jobSpec) throws Exception;
+ /**
+ * Creates a Job Instance in the specified Hyracks application using the
+ * specified {@link JobSpecification}. The specified flags are used to
+ * configure the Job creation process.
+ *
+ * @param appName
+ * Name of the application
+ * @param jobSpec
+ * Job Specification
+ * @param jobFlags
+ * Flags
+ * @return
+ * @throws Exception
+ */
public JobId createJob(String appName, JobSpecification jobSpec, EnumSet<JobFlag> jobFlags) throws Exception;
+ /**
+ * Gets the status of the specified Job.
+ *
+ * @param jobId
+ * JobId of the Job
+ * @return {@link JobStatus}
+ * @throws Exception
+ */
public JobStatus getJobStatus(JobId jobId) throws Exception;
+ /**
+ * Start the specified Job.
+ *
+ * @param jobId
+ * JobId of the Job.
+ * @throws Exception
+ */
public void start(JobId jobId) throws Exception;
+ /**
+ * Waits until the specified job has completed, either successfully or has
+ * encountered a permanent failure.
+ *
+ * @param jobId
+ * JobId of the Job
+ * @throws Exception
+ */
public void waitForCompletion(JobId jobId) throws Exception;
+
+ /**
+ * Gets a map of node controller names to node information.
+ *
+ * @return Map of node name to node information.
+ */
+ public Map<String, NodeControllerInfo> getNodeControllerInfos() throws Exception;
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java
index f57ac97..866d307 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/IHyracksClientInterface.java
@@ -14,14 +14,14 @@
*/
package edu.uci.ics.hyracks.api.client;
-import java.rmi.Remote;
import java.util.EnumSet;
+import java.util.Map;
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
-public interface IHyracksClientInterface extends Remote {
+public interface IHyracksClientInterface {
public ClusterControllerInfo getClusterControllerInfo() throws Exception;
public void createApplication(String appName) throws Exception;
@@ -34,7 +34,9 @@
public JobStatus getJobStatus(JobId jobId) throws Exception;
- public void start(JobId jobId) throws Exception;
+ public void startJob(JobId jobId) throws Exception;
public void waitForCompletion(JobId jobId) throws Exception;
+
+ public Map<String, NodeControllerInfo> getNodeControllersInfo() throws Exception;
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/NodeControllerInfo.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/NodeControllerInfo.java
new file mode 100644
index 0000000..fd9218a
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/NodeControllerInfo.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.client;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
+
+public class NodeControllerInfo implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final String nodeId;
+
+ private final NodeStatus status;
+
+ private final NetworkAddress netAddress;
+
+ public NodeControllerInfo(String nodeId, NodeStatus status, NetworkAddress netAddress) {
+ this.nodeId = nodeId;
+ this.status = status;
+ this.netAddress = netAddress;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public NodeStatus getStatus() {
+ return status;
+ }
+
+ public NetworkAddress getNetworkAddress() {
+ return netAddress;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/NodeStatus.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/NodeStatus.java
new file mode 100644
index 0000000..d6b99d0
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/client/NodeStatus.java
@@ -0,0 +1,20 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.client;
+
+public enum NodeStatus {
+ ALIVE,
+ DEAD
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/NetworkAddress.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/NetworkAddress.java
index 868221d..d176c3c 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/NetworkAddress.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/comm/NetworkAddress.java
@@ -15,21 +15,20 @@
package edu.uci.ics.hyracks.api.comm;
import java.io.Serializable;
-import java.net.InetAddress;
public final class NetworkAddress implements Serializable {
private static final long serialVersionUID = 1L;
- private final InetAddress ipAddress;
+ private final byte[] ipAddress;
private final int port;
- public NetworkAddress(InetAddress ipAddress, int port) {
+ public NetworkAddress(byte[] ipAddress, int port) {
this.ipAddress = ipAddress;
this.port = port;
}
- public InetAddress getIpAddress() {
+ public byte[] getIpAddress() {
return ipAddress;
}
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java
index 266ebe9..7466276 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/ICCContext.java
@@ -17,6 +17,10 @@
import java.util.Map;
import java.util.Set;
+import edu.uci.ics.hyracks.api.client.ClusterControllerInfo;
+
public interface ICCContext {
- public Map<String, Set<String>> getIPAddressNodeMap();
+ public ClusterControllerInfo getClusterControllerInfo();
+
+ public void getIPAddressNodeMap(Map<String, Set<String>> map) throws Exception;
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksTaskContext.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksTaskContext.java
index 7609951..042d341 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksTaskContext.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/context/IHyracksTaskContext.java
@@ -14,25 +14,17 @@
*/
package edu.uci.ics.hyracks.api.context;
-import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.IWorkspaceFileFactory;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.profiling.counters.ICounterContext;
import edu.uci.ics.hyracks.api.resources.IDeallocatableRegistry;
-public interface IHyracksTaskContext extends IHyracksCommonContext, IWorkspaceFileFactory, IDeallocatableRegistry {
+public interface IHyracksTaskContext extends IHyracksCommonContext, IWorkspaceFileFactory, IDeallocatableRegistry,
+ IOperatorEnvironment {
public IHyracksJobletContext getJobletContext();
public TaskAttemptId getTaskAttemptId();
public ICounterContext getCounterContext();
-
- public Object lookupGlobalVariable(ActivityId producerActivity, int partition, String varName)
- throws HyracksDataException;
-
- public Object lookupLocalVariable(ActivityId producerActivity, int partition, String varName)
- throws HyracksDataException;
-
- public void setVariable(String name, Object value) throws HyracksDataException;
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityId.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityId.java
index c6ca51c..41b4c23 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityId.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/ActivityId.java
@@ -19,9 +19,9 @@
public final class ActivityId implements Serializable {
private static final long serialVersionUID = 1L;
private final OperatorDescriptorId odId;
- private final long id;
+ private final int id;
- public ActivityId(OperatorDescriptorId odId, long id) {
+ public ActivityId(OperatorDescriptorId odId, int id) {
this.odId = odId;
this.id = id;
}
@@ -30,7 +30,7 @@
return odId;
}
- public long getLocalId() {
+ public int getLocalId() {
return id;
}
@@ -52,6 +52,16 @@
}
public String toString() {
- return "ANID:[" + odId + "]:" + id;
+ return "ANID:" + odId + ":" + id;
+ }
+
+ public static ActivityId parse(String str) {
+ if (str.startsWith("ANID:")) {
+ str = str.substring(5);
+ int idIdx = str.lastIndexOf(':');
+ return new ActivityId(OperatorDescriptorId.parse(str.substring(0, idIdx)), Integer.parseInt(str
+ .substring(idIdx + 1)));
+ }
+ throw new IllegalArgumentException("Unable to parse: " + str);
}
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivity.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivity.java
index 6b519d6..df8fd53 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivity.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IActivity.java
@@ -19,11 +19,10 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
public interface IActivity extends Serializable {
public ActivityId getActivityId();
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException;
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java
index ec8ab76..0ac3658 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IConnectorDescriptor.java
@@ -44,7 +44,8 @@
public ConnectorDescriptorId getConnectorId();
/**
- * Factory method to create the send side writer that writes into this connector.
+ * Factory method to create the send side writer that writes into this
+ * connector.
*
* @param ctx
* Context
@@ -66,7 +67,8 @@
throws HyracksDataException;
/**
- * Factory metod to create the receive side reader that reads data from this connector.
+ * Factory metod to create the receive side reader that reads data from this
+ * connector.
*
* @param ctx
* Context
@@ -96,12 +98,29 @@
ICCApplicationContext appCtx);
/**
- * Indicate which consumer partitions may receive data from the given producer partition.
+ * Indicate which consumer partitions may receive data from the given
+ * producer partition.
*/
public void indicateTargetPartitions(int nProducerPartitions, int nConsumerPartitions, int producerIndex,
BitSet targetBitmap);
/**
+ * Indicate which producer partitions are required for the given receiver.
+ */
+ public void indicateSourcePartitions(int nProducerPartitions, int nConsumerPartitions, int consumerIndex,
+ BitSet sourceBitmap);
+
+ /**
+ * Gets the display name.
+ */
+ public String getDisplayName();
+
+ /**
+ * Sets the display name.
+ */
+ public void setDisplayName(String displayName);
+
+ /**
* Translate this connector descriptor to JSON.
*
* @return
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java
index 78847eb..c37a530 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/IOperatorDescriptor.java
@@ -59,7 +59,8 @@
public RecordDescriptor[] getOutputRecordDescriptors();
/**
- * Contributes the activity graph that describes the behavior of this operator.
+ * Contributes the activity graph that describes the behavior of this
+ * operator.
*
* @param builder
* - graph builder
@@ -78,6 +79,16 @@
ICCApplicationContext appCtx);
/**
+ * Gets the display name.
+ */
+ public String getDisplayName();
+
+ /**
+ * Sets the display name.
+ */
+ public void setDisplayName(String displayName);
+
+ /**
* Translates this operator descriptor to JSON.
*/
public JSONObject toJSON() throws JSONException;
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java
index b858736..8794e09 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/OperatorDescriptorId.java
@@ -49,4 +49,12 @@
public String toString() {
return "ODID:" + id;
}
+
+ public static OperatorDescriptorId parse(String str) {
+ if (str.startsWith("ODID:")) {
+ str = str.substring(5);
+ return new OperatorDescriptorId(Integer.parseInt(str));
+ }
+ throw new IllegalArgumentException("Unable to parse: " + str);
+ }
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskAttemptId.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskAttemptId.java
index d4e6972..0fb44c1 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskAttemptId.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskAttemptId.java
@@ -52,6 +52,15 @@
@Override
public String toString() {
- return "TAID:[" + taskId + "]:" + attempt;
+ return "TAID:" + taskId + ":" + attempt;
+ }
+
+ public static TaskAttemptId parse(String str) {
+ if (str.startsWith("TAID:")) {
+ str = str.substring(5);
+ int idIdx = str.lastIndexOf(':');
+ return new TaskAttemptId(TaskId.parse(str.substring(0, idIdx)), Integer.parseInt(str.substring(idIdx + 1)));
+ }
+ throw new IllegalArgumentException("Unable to parse: " + str);
}
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskId.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskId.java
index ee63355..7e0b22d 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskId.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/TaskId.java
@@ -52,6 +52,15 @@
@Override
public String toString() {
- return "TID:[" + activityId + "]:" + partition;
+ return "TID:" + activityId + ":" + partition;
+ }
+
+ public static TaskId parse(String str) {
+ if (str.startsWith("TID:")) {
+ str = str.substring(4);
+ int idIdx = str.lastIndexOf(':');
+ return new TaskId(ActivityId.parse(str.substring(0, idIdx)), Integer.parseInt(str.substring(idIdx + 1)));
+ }
+ throw new IllegalArgumentException("Unable to parse: " + str);
}
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/connectors/PipeliningConnectorPolicy.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/connectors/PipeliningConnectorPolicy.java
index 39b3904..b55947e 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/connectors/PipeliningConnectorPolicy.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/connectors/PipeliningConnectorPolicy.java
@@ -24,7 +24,7 @@
@Override
public boolean consumerWaitsForProducerToFinish() {
- return true;
+ return false;
}
@Override
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunctionFamily.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunctionFamily.java
new file mode 100644
index 0000000..5cb27d6
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunctionFamily.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.Serializable;
+
+public interface IBinaryHashFunctionFamily extends Serializable {
+ public IBinaryHashFunction createBinaryHashFunction(int seed);
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunctionGeneratorFactory.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunctionGeneratorFactory.java
deleted file mode 100644
index 953f61c..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/IBinaryHashFunctionGeneratorFactory.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package edu.uci.ics.hyracks.api.dataflow.value;
-
-import java.io.Serializable;
-
-public interface IBinaryHashFunctionGeneratorFactory extends Serializable{
- public IBinaryHashFunction createBinaryHashFunction(int seed);
-}
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputerFamily.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputerFamily.java
new file mode 100644
index 0000000..1839dc1
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputerFamily.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.Serializable;
+
+public interface ITuplePartitionComputerFamily extends Serializable {
+ public ITuplePartitionComputer createPartitioner(int seed);
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputerGeneratorFactory.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputerGeneratorFactory.java
deleted file mode 100644
index ed83986..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITuplePartitionComputerGeneratorFactory.java
+++ /dev/null
@@ -1,7 +0,0 @@
-package edu.uci.ics.hyracks.api.dataflow.value;
-
-import java.io.Serializable;
-
-public interface ITuplePartitionComputerGeneratorFactory extends Serializable {
- public ITuplePartitionComputer createPartitioner(int seed);
-}
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITypeTrait.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITypeTrait.java
deleted file mode 100644
index 0f46f57..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITypeTrait.java
+++ /dev/null
@@ -1,8 +0,0 @@
-package edu.uci.ics.hyracks.api.dataflow.value;
-
-import java.io.Serializable;
-
-public interface ITypeTrait extends Serializable {
- public static final int VARIABLE_LENGTH = -1;
- int getStaticallyKnownDataLength();
-}
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITypeTraits.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITypeTraits.java
new file mode 100644
index 0000000..4a6f826
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/ITypeTraits.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.dataflow.value;
+
+import java.io.Serializable;
+
+public interface ITypeTraits extends Serializable {
+ public boolean isFixedLength();
+
+ public int getFixedLength();
+}
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java
index 4922723..b04fe86 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/RecordDescriptor.java
@@ -21,25 +21,29 @@
private static final long serialVersionUID = 1L;
private final ISerializerDeserializer[] fields;
- private final ITypeTrait[] typeTraits;
-
+ private final ITypeTraits[] typeTraits;
+
// leaving this constructor for backwards-compatibility
public RecordDescriptor(ISerializerDeserializer[] fields) {
this.fields = fields;
this.typeTraits = null;
}
-
+
// temporarily adding constructor to include type traits
- public RecordDescriptor(ISerializerDeserializer[] fields, ITypeTrait[] typeTraits) {
- this.fields = fields;
+ public RecordDescriptor(ISerializerDeserializer[] fields, ITypeTraits[] typeTraits) {
+ this.fields = fields;
this.typeTraits = typeTraits;
}
+ public int getFieldCount() {
+ return fields.length;
+ }
+
public ISerializerDeserializer[] getFields() {
return fields;
}
-
- public ITypeTrait[] getTypeTraits() {
- return typeTraits;
+
+ public ITypeTraits[] getTypeTraits() {
+ return typeTraits;
}
}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/TypeTrait.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/TypeTrait.java
deleted file mode 100644
index 6be9552..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/dataflow/value/TypeTrait.java
+++ /dev/null
@@ -1,16 +0,0 @@
-package edu.uci.ics.hyracks.api.dataflow.value;
-
-public class TypeTrait implements ITypeTrait {
-
- private static final long serialVersionUID = 1L;
- private int length;
-
- public TypeTrait(int length) {
- this.length = length;
- }
-
- @Override
- public int getStaticallyKnownDataLength() {
- return length;
- }
-}
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobletEventListener.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobletEventListener.java
new file mode 100644
index 0000000..c83e333
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobletEventListener.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+public interface IJobletEventListener {
+ public void jobletStart();
+
+ public void jobletFinish(JobStatus status);
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobletEventListenerFactory.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobletEventListenerFactory.java
new file mode 100644
index 0000000..3b3cacb
--- /dev/null
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/IJobletEventListenerFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.api.job;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.context.IHyracksJobletContext;
+
+public interface IJobletEventListenerFactory extends Serializable {
+ public IJobletEventListener createListener(IHyracksJobletContext ctx);
+}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java
index 930d299..2b6d361 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobActivityGraph.java
@@ -22,6 +22,7 @@
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang3.tuple.Pair;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -33,7 +34,6 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.util.Pair;
public class JobActivityGraph implements Serializable {
private static final long serialVersionUID = 1L;
@@ -148,8 +148,8 @@
Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connEdge = jobSpec
.getConnectorOperatorMap().get(cdId);
- OperatorDescriptorId consumerOpId = connEdge.second.first.getOperatorId();
- int consumerInputIdx = connEdge.second.second;
+ OperatorDescriptorId consumerOpId = connEdge.getRight().getLeft().getOperatorId();
+ int consumerInputIdx = connEdge.getRight().getRight();
for (ActivityId anId : operatorActivityMap.get(consumerOpId)) {
List<Integer> anInputs = activityInputMap.get(anId);
@@ -168,8 +168,8 @@
Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connEdge = jobSpec
.getConnectorOperatorMap().get(cdId);
- OperatorDescriptorId producerOpId = connEdge.first.first.getOperatorId();
- int producerInputIdx = connEdge.first.second;
+ OperatorDescriptorId producerOpId = connEdge.getLeft().getLeft().getOperatorId();
+ int producerInputIdx = connEdge.getLeft().getRight();
for (ActivityId anId : operatorActivityMap.get(producerOpId)) {
List<Integer> anOutputs = activityOutputMap.get(anId);
@@ -209,13 +209,11 @@
public JSONObject toJSON() throws JSONException {
JSONObject jplan = new JSONObject();
- jplan.put("type", "plan");
jplan.put("flags", jobFlags.toString());
JSONArray jans = new JSONArray();
for (IActivity an : activityNodes.values()) {
JSONObject jan = new JSONObject();
- jan.put("type", "activity");
jan.put("id", an.getActivityId().toString());
jan.put("java-class", an.getClass().getName());
jan.put("operator-id", an.getActivityId().getOperatorDescriptorId().toString());
@@ -225,7 +223,6 @@
JSONArray jInputs = new JSONArray();
for (int i = 0; i < inputs.size(); ++i) {
JSONObject jInput = new JSONObject();
- jInput.put("type", "activity-input");
jInput.put("input-port", i);
jInput.put("connector-id", inputs.get(i).getConnectorId().toString());
jInputs.put(jInput);
@@ -238,7 +235,6 @@
JSONArray jOutputs = new JSONArray();
for (int i = 0; i < outputs.size(); ++i) {
JSONObject jOutput = new JSONObject();
- jOutput.put("type", "activity-output");
jOutput.put("output-port", i);
jOutput.put("connector-id", outputs.get(i).getConnectorId().toString());
jOutputs.put(jOutput);
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
index ef44033..90c799d 100644
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
+++ b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/job/JobSpecification.java
@@ -22,6 +22,7 @@
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang3.tuple.Pair;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -33,7 +34,6 @@
import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicyAssignmentPolicy;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.util.Pair;
public class JobSpecification implements Serializable {
private static final long serialVersionUID = 1L;
@@ -56,7 +56,9 @@
private IConnectorPolicyAssignmentPolicy connectorPolicyAssignmentPolicy;
- private int maxAttempts;
+ private int maxReattempts;
+
+ private IJobletEventListenerFactory jobletEventListenerFactory;
private transient int operatorIdCounter;
@@ -73,7 +75,7 @@
userConstraints = new HashSet<Constraint>();
operatorIdCounter = 0;
connectorIdCounter = 0;
- maxAttempts = 5;
+ maxReattempts = 2;
}
public OperatorDescriptorId createOperatorDescriptorId() {
@@ -92,10 +94,11 @@
IOperatorDescriptor consumerOp, int consumerPort) {
insertIntoIndexedMap(opInputMap, consumerOp.getOperatorId(), consumerPort, conn);
insertIntoIndexedMap(opOutputMap, producerOp.getOperatorId(), producerPort, conn);
- connectorOpMap.put(conn.getConnectorId(),
- new Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>>(
- new Pair<IOperatorDescriptor, Integer>(producerOp, producerPort),
- new Pair<IOperatorDescriptor, Integer>(consumerOp, consumerPort)));
+ connectorOpMap.put(
+ conn.getConnectorId(),
+ Pair.<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> of(
+ Pair.<IOperatorDescriptor, Integer> of(producerOp, producerPort),
+ Pair.<IOperatorDescriptor, Integer> of(consumerOp, consumerPort)));
}
public void setProperty(String name, Serializable value) {
@@ -124,19 +127,19 @@
public RecordDescriptor getConnectorRecordDescriptor(IConnectorDescriptor conn) {
Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
.getConnectorId());
- return connInfo.first.first.getOutputRecordDescriptors()[connInfo.first.second];
+ return connInfo.getLeft().getLeft().getOutputRecordDescriptors()[connInfo.getLeft().getRight()];
}
public IOperatorDescriptor getConsumer(IConnectorDescriptor conn) {
Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
.getConnectorId());
- return connInfo.second.first;
+ return connInfo.getRight().getLeft();
}
public int getConsumerInputIndex(IConnectorDescriptor conn) {
Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
.getConnectorId());
- return connInfo.second.second;
+ return connInfo.getRight().getRight();
}
public IConnectorDescriptor getInputConnectorDescriptor(IOperatorDescriptor op, int inputIndex) {
@@ -178,13 +181,13 @@
public IOperatorDescriptor getProducer(IConnectorDescriptor conn) {
Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
.getConnectorId());
- return connInfo.first.first;
+ return connInfo.getLeft().getLeft();
}
public int getProducerOutputIndex(IConnectorDescriptor conn) {
Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connInfo = connectorOpMap.get(conn
.getConnectorId());
- return connInfo.first.second;
+ return connInfo.getLeft().getRight();
}
public List<OperatorDescriptorId> getRoots() {
@@ -199,12 +202,12 @@
this.connectorPolicyAssignmentPolicy = connectorPolicyAssignmentPolicy;
}
- public void setMaxAttempts(int maxAttempts) {
- this.maxAttempts = maxAttempts;
+ public void setMaxReattempts(int maxReattempts) {
+ this.maxReattempts = maxReattempts;
}
- public int getMaxAttempts() {
- return maxAttempts;
+ public int getMaxReattempts() {
+ return maxReattempts;
}
public void addUserConstraint(Constraint constraint) {
@@ -215,6 +218,14 @@
return userConstraints;
}
+ public IJobletEventListenerFactory getJobletEventListenerFactory() {
+ return jobletEventListenerFactory;
+ }
+
+ public void setJobletEventListenerFactory(IJobletEventListenerFactory jobletEventListenerFactory) {
+ this.jobletEventListenerFactory = jobletEventListenerFactory;
+ }
+
private <K, V> void insertIntoIndexedMap(Map<K, List<V>> map, K key, int index, V value) {
List<V> vList = map.get(key);
if (vList == null) {
@@ -254,8 +265,6 @@
public JSONObject toJSON() throws JSONException {
JSONObject jjob = new JSONObject();
- jjob.put("type", "job");
-
JSONArray jopArray = new JSONArray();
for (Map.Entry<OperatorDescriptorId, IOperatorDescriptor> e : opMap.entrySet()) {
jopArray.put(e.getValue().toJSON());
@@ -267,12 +276,11 @@
JSONObject conn = new JSONObject();
Pair<Pair<IOperatorDescriptor, Integer>, Pair<IOperatorDescriptor, Integer>> connection = connectorOpMap
.get(e.getKey());
- conn.put("type", "connector-info");
if (connection != null) {
- conn.put("in-operator-id", connection.first.first.getOperatorId().toString());
- conn.put("in-operator-port", connection.first.second.intValue());
- conn.put("out-operator-id", connection.second.first.getOperatorId().toString());
- conn.put("out-operator-port", connection.second.second.intValue());
+ conn.put("in-operator-id", connection.getLeft().getLeft().getOperatorId().toString());
+ conn.put("in-operator-port", connection.getLeft().getRight().intValue());
+ conn.put("out-operator-id", connection.getRight().getLeft().getOperatorId().toString());
+ conn.put("out-operator-port", connection.getRight().getRight().intValue());
}
conn.put("connector", e.getValue().toJSON());
jcArray.put(conn);
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
deleted file mode 100644
index 2364d74..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/util/Pair.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.util;
-
-import java.io.Serializable;
-
-public final class Pair<F, S> implements Serializable {
- private static final long serialVersionUID = 1L;
-
- public F first;
- public S second;
-
- public Pair(F first, S second) {
- this.first = first;
- this.second = second;
- }
-}
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/workflow/variables/WorkflowVariableDescriptor.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/workflow/variables/WorkflowVariableDescriptor.java
deleted file mode 100644
index b13afe4..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/workflow/variables/WorkflowVariableDescriptor.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.workflow.variables;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.naming.MultipartName;
-
-public final class WorkflowVariableDescriptor implements Serializable {
- private static final long serialVersionUID = 1L;
-
- private final MultipartName name;
-
- private final WorkflowVariableValueScope scope;
-
- public WorkflowVariableDescriptor(MultipartName name, WorkflowVariableValueScope scope) {
- this.name = name;
- this.scope = scope;
- }
-
- public MultipartName getName() {
- return name;
- }
-
- public WorkflowVariableValueScope getScope() {
- return scope;
- }
-}
\ No newline at end of file
diff --git a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/workflow/variables/WorkflowVariableValueScope.java b/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/workflow/variables/WorkflowVariableValueScope.java
deleted file mode 100644
index 44ed129..0000000
--- a/hyracks-api/src/main/java/edu/uci/ics/hyracks/api/workflow/variables/WorkflowVariableValueScope.java
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.api.workflow.variables;
-
-public enum WorkflowVariableValueScope {
- LOCAL,
- GLOBAL,
-}
\ No newline at end of file
diff --git a/hyracks-cli/.classpath b/hyracks-cli/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-cli/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-cli/.project b/hyracks-cli/.project
deleted file mode 100644
index 9741f63..0000000
--- a/hyracks-cli/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-cli</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-cli/.settings/org.eclipse.jdt.core.prefs b/hyracks-cli/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index a80ec7b..0000000
--- a/hyracks-cli/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:08 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-cli/.settings/org.maven.ide.eclipse.prefs b/hyracks-cli/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index a4b7f25..0000000
--- a/hyracks-cli/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Aug 05 10:16:22 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-cli/pom.xml b/hyracks-cli/pom.xml
index ba2c796..1fdc644 100644
--- a/hyracks-cli/pom.xml
+++ b/hyracks-cli/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-cli</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -89,7 +89,7 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
diff --git a/hyracks-cli/src/main/java/edu/uci/ics/hyracks/cli/commands/ConnectCommand.java b/hyracks-cli/src/main/java/edu/uci/ics/hyracks/cli/commands/ConnectCommand.java
index e1a1fd2..168280b 100644
--- a/hyracks-cli/src/main/java/edu/uci/ics/hyracks/cli/commands/ConnectCommand.java
+++ b/hyracks-cli/src/main/java/edu/uci/ics/hyracks/cli/commands/ConnectCommand.java
@@ -1,6 +1,6 @@
package edu.uci.ics.hyracks.cli.commands;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.cli.Session;
@@ -12,7 +12,7 @@
public ConnectCommand(String hostPortStr) {
int idx = hostPortStr.indexOf(':');
host = hostPortStr;
- port = 1099;
+ port = 1098;
if (idx != -1) {
host = hostPortStr.substring(0, idx);
port = Integer.valueOf(hostPortStr.substring(idx + 1));
@@ -22,7 +22,7 @@
@Override
public void run(Session session) throws Exception {
System.err.println("Connecting to host: " + host + ", port: " + port);
- IHyracksClientConnection conn = new HyracksRMIConnection(host, port);
+ IHyracksClientConnection conn = new HyracksConnection(host, port);
session.setConnection(conn);
}
}
\ No newline at end of file
diff --git a/hyracks-control-cc/.classpath b/hyracks-control-cc/.classpath
deleted file mode 100644
index fb2f7c1..0000000
--- a/hyracks-control-cc/.classpath
+++ /dev/null
@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-control-cc/.project b/hyracks-control-cc/.project
deleted file mode 100644
index 271840b..0000000
--- a/hyracks-control-cc/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-control-cc</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-control-cc/.settings/org.eclipse.jdt.core.prefs b/hyracks-control-cc/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 375e12e..0000000
--- a/hyracks-control-cc/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:07 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-control-cc/.settings/org.maven.ide.eclipse.prefs b/hyracks-control-cc/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index d783847..0000000
--- a/hyracks-control-cc/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Fri Jul 30 17:52:26 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-control-cc/pom.xml b/hyracks-control-cc/pom.xml
index 3fed82f..203bc49 100644
--- a/hyracks-control-cc/pom.xml
+++ b/hyracks-control-cc/pom.xml
@@ -1,13 +1,10 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-cc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,7 +24,7 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
@@ -45,5 +42,15 @@
<type>jar</type>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.wicket</groupId>
+ <artifactId>wicket-core</artifactId>
+ <version>1.5.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-jcl</artifactId>
+ <version>1.6.3</version>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/CCClientInterface.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/CCClientInterface.java
deleted file mode 100644
index 77694b4..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/CCClientInterface.java
+++ /dev/null
@@ -1,61 +0,0 @@
-package edu.uci.ics.hyracks.control.cc;
-
-import java.rmi.RemoteException;
-import java.rmi.server.UnicastRemoteObject;
-import java.util.EnumSet;
-
-import edu.uci.ics.hyracks.api.client.ClusterControllerInfo;
-import edu.uci.ics.hyracks.api.client.IHyracksClientInterface;
-import edu.uci.ics.hyracks.api.job.JobFlag;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-
-public class CCClientInterface extends UnicastRemoteObject implements IHyracksClientInterface {
- private static final long serialVersionUID = 1L;
-
- private final ClusterControllerService ccs;
-
- public CCClientInterface(ClusterControllerService ccs) throws RemoteException {
- this.ccs = ccs;
- }
-
- @Override
- public ClusterControllerInfo getClusterControllerInfo() throws Exception {
- return ccs.getClusterControllerInfo();
- }
-
- @Override
- public void createApplication(String appName) throws Exception {
- ccs.createApplication(appName);
- }
-
- @Override
- public void startApplication(String appName) throws Exception {
- ccs.startApplication(appName);
- }
-
- @Override
- public void destroyApplication(String appName) throws Exception {
- ccs.destroyApplication(appName);
- }
-
- @Override
- public JobId createJob(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
- return ccs.createJob(appName, jobSpec, jobFlags);
- }
-
- @Override
- public JobStatus getJobStatus(JobId jobId) throws Exception {
- return ccs.getJobStatus(jobId);
- }
-
- @Override
- public void start(JobId jobId) throws Exception {
- ccs.start(jobId);
- }
-
- @Override
- public void waitForCompletion(JobId jobId) throws Exception {
- ccs.waitForCompletion(jobId);
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
index cbbef2d..ebeb8a6 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/ClusterControllerService.java
@@ -15,13 +15,10 @@
package edu.uci.ics.hyracks.control.cc;
import java.io.File;
-import java.rmi.registry.LocateRegistry;
-import java.rmi.registry.Registry;
-import java.util.EnumSet;
+import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.Hashtable;
import java.util.LinkedHashMap;
-import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Timer;
@@ -32,55 +29,59 @@
import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.client.ClusterControllerInfo;
-import edu.uci.ics.hyracks.api.client.IHyracksClientInterface;
+import edu.uci.ics.hyracks.api.client.HyracksClientInterfaceFunctions;
+import edu.uci.ics.hyracks.api.client.NodeControllerInfo;
import edu.uci.ics.hyracks.api.context.ICCContext;
-import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
-import edu.uci.ics.hyracks.control.cc.job.IJobStatusConditionVariable;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.ApplicationDestroyEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.ApplicationStartEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.GetJobStatusConditionVariableEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.GetJobStatusEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.JobCreateEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.JobStartEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.NodeHeartbeatEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.RegisterNodeEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.RegisterPartitionAvailibilityEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.RegisterPartitionRequestEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.RemoveDeadNodesEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.ReportProfilesEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.TaskCompleteEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.TaskFailureEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.UnregisterNodeEvent;
-import edu.uci.ics.hyracks.control.cc.jobqueue.FutureValue;
-import edu.uci.ics.hyracks.control.cc.jobqueue.JobQueue;
import edu.uci.ics.hyracks.control.cc.web.WebServer;
+import edu.uci.ics.hyracks.control.cc.work.ApplicationCreateWork;
+import edu.uci.ics.hyracks.control.cc.work.ApplicationDestroyWork;
+import edu.uci.ics.hyracks.control.cc.work.ApplicationStartWork;
+import edu.uci.ics.hyracks.control.cc.work.ApplicationStateChangeWork;
+import edu.uci.ics.hyracks.control.cc.work.GetIpAddressNodeNameMapWork;
+import edu.uci.ics.hyracks.control.cc.work.GetJobStatusWork;
+import edu.uci.ics.hyracks.control.cc.work.GetNodeControllersInfoWork;
+import edu.uci.ics.hyracks.control.cc.work.JobCreateWork;
+import edu.uci.ics.hyracks.control.cc.work.JobStartWork;
+import edu.uci.ics.hyracks.control.cc.work.JobletCleanupNotificationWork;
+import edu.uci.ics.hyracks.control.cc.work.NodeHeartbeatWork;
+import edu.uci.ics.hyracks.control.cc.work.RegisterNodeWork;
+import edu.uci.ics.hyracks.control.cc.work.RegisterPartitionAvailibilityWork;
+import edu.uci.ics.hyracks.control.cc.work.RegisterPartitionRequestWork;
+import edu.uci.ics.hyracks.control.cc.work.RemoveDeadNodesWork;
+import edu.uci.ics.hyracks.control.cc.work.ReportProfilesWork;
+import edu.uci.ics.hyracks.control.cc.work.TaskCompleteWork;
+import edu.uci.ics.hyracks.control.cc.work.TaskFailureWork;
+import edu.uci.ics.hyracks.control.cc.work.UnregisterNodeWork;
+import edu.uci.ics.hyracks.control.cc.work.WaitForJobCompletionWork;
import edu.uci.ics.hyracks.control.common.AbstractRemoteService;
-import edu.uci.ics.hyracks.control.common.base.IClusterController;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
import edu.uci.ics.hyracks.control.common.context.ServerContext;
import edu.uci.ics.hyracks.control.common.controllers.CCConfig;
-import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
-import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
-import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
-import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
-import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
-import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
-import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
+import edu.uci.ics.hyracks.control.common.ipc.CCNCFunctions;
+import edu.uci.ics.hyracks.control.common.ipc.CCNCFunctions.Function;
+import edu.uci.ics.hyracks.control.common.logs.LogFile;
+import edu.uci.ics.hyracks.control.common.work.IPCResponder;
+import edu.uci.ics.hyracks.control.common.work.WorkQueue;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.IIPCI;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+import edu.uci.ics.hyracks.ipc.impl.IPCSystem;
+import edu.uci.ics.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer;
-public class ClusterControllerService extends AbstractRemoteService implements IClusterController,
- IHyracksClientInterface {
- private static final long serialVersionUID = 1L;
-
- private CCConfig ccConfig;
-
+public class ClusterControllerService extends AbstractRemoteService {
private static Logger LOGGER = Logger.getLogger(ClusterControllerService.class.getName());
+ private final CCConfig ccConfig;
+
+ private IPCSystem clusterIPC;
+
+ private IPCSystem clientIPC;
+
+ private final LogFile jobLog;
+
private final Map<String, NodeControllerState> nodeRegistry;
private final Map<String, Set<String>> ipAddressNodeNameMap;
@@ -93,53 +94,76 @@
private ClusterControllerInfo info;
- private final Map<JobId, JobRun> runMap;
+ private final Map<JobId, JobRun> activeRunMap;
- private final JobQueue jobQueue;
+ private final Map<JobId, JobRun> runMapArchive;
- private final Executor taskExecutor;
+ private final WorkQueue workQueue;
+
+ private final Executor executor;
private final Timer timer;
- private final CCClientInterface ccci;
-
private final ICCContext ccContext;
+ private final DeadNodeSweeper sweeper;
+
private long jobCounter;
- public ClusterControllerService(CCConfig ccConfig) throws Exception {
+ public ClusterControllerService(final CCConfig ccConfig) throws Exception {
this.ccConfig = ccConfig;
+ File jobLogFolder = new File(ccConfig.ccRoot, "logs/jobs");
+ jobLog = new LogFile(jobLogFolder);
nodeRegistry = new LinkedHashMap<String, NodeControllerState>();
ipAddressNodeNameMap = new HashMap<String, Set<String>>();
applications = new Hashtable<String, CCApplicationContext>();
- serverCtx = new ServerContext(ServerContext.ServerType.CLUSTER_CONTROLLER, new File(
- ClusterControllerService.class.getName()));
- taskExecutor = Executors.newCachedThreadPool();
+ serverCtx = new ServerContext(ServerContext.ServerType.CLUSTER_CONTROLLER, new File(ccConfig.ccRoot));
+ executor = Executors.newCachedThreadPool();
+ IIPCI ccIPCI = new ClusterControllerIPCI();
+ clusterIPC = new IPCSystem(new InetSocketAddress(ccConfig.clusterNetPort), ccIPCI,
+ new CCNCFunctions.SerializerDeserializer());
+ IIPCI ciIPCI = new HyracksClientInterfaceIPCI();
+ clientIPC = new IPCSystem(new InetSocketAddress(ccConfig.clientNetIpAddress, ccConfig.clientNetPort), ciIPCI,
+ new JavaSerializationBasedPayloadSerializerDeserializer());
webServer = new WebServer(this);
- runMap = new HashMap<JobId, JobRun>();
- jobQueue = new JobQueue();
- this.timer = new Timer(true);
- ccci = new CCClientInterface(this);
- ccContext = new ICCContext() {
- @Override
- public Map<String, Set<String>> getIPAddressNodeMap() {
- return ipAddressNodeNameMap;
+ activeRunMap = new HashMap<JobId, JobRun>();
+ runMapArchive = new LinkedHashMap<JobId, JobRun>() {
+ private static final long serialVersionUID = 1L;
+
+ protected boolean removeEldestEntry(Map.Entry<JobId, JobRun> eldest) {
+ return size() > ccConfig.jobHistorySize;
}
};
+ workQueue = new WorkQueue();
+ this.timer = new Timer(true);
+ ccContext = new ICCContext() {
+ @Override
+ public void getIPAddressNodeMap(Map<String, Set<String>> map) throws Exception {
+ GetIpAddressNodeNameMapWork ginmw = new GetIpAddressNodeNameMapWork(ClusterControllerService.this, map);
+ workQueue.scheduleAndSync(ginmw);
+ }
+
+ @Override
+ public ClusterControllerInfo getClusterControllerInfo() {
+ return info;
+ }
+ };
+ sweeper = new DeadNodeSweeper();
jobCounter = 0;
}
@Override
public void start() throws Exception {
- LOGGER.log(Level.INFO, "Starting ClusterControllerService");
- Registry registry = LocateRegistry.createRegistry(ccConfig.port);
- registry.rebind(IHyracksClientInterface.class.getName(), ccci);
- registry.rebind(IClusterController.class.getName(), this);
+ LOGGER.log(Level.INFO, "Starting ClusterControllerService: " + this);
+ clusterIPC.start();
+ clientIPC.start();
webServer.setPort(ccConfig.httpPort);
webServer.start();
- info = new ClusterControllerInfo();
- info.setWebPort(webServer.getListeningPort());
- timer.schedule(new DeadNodeSweeper(), 0, ccConfig.heartbeatPeriod);
+ workQueue.start();
+ info = new ClusterControllerInfo(ccConfig.clientNetIpAddress, ccConfig.clientNetPort,
+ webServer.getListeningPort());
+ timer.schedule(sweeper, 0, ccConfig.heartbeatPeriod);
+ jobLog.open();
LOGGER.log(Level.INFO, "Started ClusterControllerService");
}
@@ -147,33 +171,52 @@
public void stop() throws Exception {
LOGGER.log(Level.INFO, "Stopping ClusterControllerService");
webServer.stop();
+ sweeper.cancel();
+ workQueue.stop();
+ jobLog.close();
LOGGER.log(Level.INFO, "Stopped ClusterControllerService");
}
+ public ServerContext getServerContext() {
+ return serverCtx;
+ }
+
+ public ICCContext getCCContext() {
+ return ccContext;
+ }
+
public Map<String, CCApplicationContext> getApplicationMap() {
return applications;
}
- public Map<JobId, JobRun> getRunMap() {
- return runMap;
+ public Map<JobId, JobRun> getActiveRunMap() {
+ return activeRunMap;
}
- public JobQueue getJobQueue() {
- return jobQueue;
+ public Map<JobId, JobRun> getRunMapArchive() {
+ return runMapArchive;
+ }
+
+ public Map<String, Set<String>> getIpAddressNodeNameMap() {
+ return ipAddressNodeNameMap;
+ }
+
+ public LogFile getJobLogFile() {
+ return jobLog;
+ }
+
+ public WorkQueue getWorkQueue() {
+ return workQueue;
}
public Executor getExecutor() {
- return taskExecutor;
+ return executor;
}
public Map<String, NodeControllerState> getNodeMap() {
return nodeRegistry;
}
- public Map<String, Set<String>> getIPAddressNodeNameMap() {
- return ipAddressNodeNameMap;
- }
-
public CCConfig getConfig() {
return ccConfig;
}
@@ -182,128 +225,174 @@
return new JobId(jobCounter++);
}
- @Override
- public JobId createJob(String appName, byte[] jobSpec, EnumSet<JobFlag> jobFlags) throws Exception {
- JobId jobId = createJobId();
- JobCreateEvent jce = new JobCreateEvent(this, jobId, appName, jobSpec, jobFlags);
- jobQueue.schedule(jce);
- jce.sync();
- return jobId;
- }
-
- @Override
- public NodeParameters registerNode(NodeRegistration reg) throws Exception {
- INodeController nodeController = reg.getNodeController();
- String id = reg.getNodeId();
- NodeControllerState state = new NodeControllerState(nodeController, reg);
- jobQueue.scheduleAndSync(new RegisterNodeEvent(this, id, state));
- nodeController.notifyRegistration(this);
- LOGGER.log(Level.INFO, "Registered INodeController: id = " + id);
- NodeParameters params = new NodeParameters();
- params.setClusterControllerInfo(info);
- params.setHeartbeatPeriod(ccConfig.heartbeatPeriod);
- params.setProfileDumpPeriod(ccConfig.profileDumpPeriod);
- return params;
- }
-
- @Override
- public void unregisterNode(INodeController nodeController) throws Exception {
- String id = nodeController.getId();
- jobQueue.scheduleAndSync(new UnregisterNodeEvent(this, id));
- LOGGER.log(Level.INFO, "Unregistered INodeController");
- }
-
- @Override
- public void notifyTaskComplete(JobId jobId, TaskAttemptId taskId, String nodeId, TaskProfile statistics)
- throws Exception {
- TaskCompleteEvent sce = new TaskCompleteEvent(this, jobId, taskId, nodeId);
- jobQueue.schedule(sce);
- }
-
- @Override
- public void notifyTaskFailure(JobId jobId, TaskAttemptId taskId, String nodeId, Exception exception)
- throws Exception {
- TaskFailureEvent tfe = new TaskFailureEvent(this, jobId, taskId, nodeId, exception);
- jobQueue.schedule(tfe);
- }
-
- @Override
- public JobStatus getJobStatus(JobId jobId) throws Exception {
- GetJobStatusEvent gse = new GetJobStatusEvent(this, jobId);
- jobQueue.scheduleAndSync(gse);
- return gse.getStatus();
- }
-
- @Override
- public void start(JobId jobId) throws Exception {
- JobStartEvent jse = new JobStartEvent(this, jobId);
- jobQueue.schedule(jse);
- }
-
- @Override
- public void waitForCompletion(JobId jobId) throws Exception {
- GetJobStatusConditionVariableEvent e = new GetJobStatusConditionVariableEvent(this, jobId);
- jobQueue.scheduleAndSync(e);
- IJobStatusConditionVariable var = e.getConditionVariable();
- if (var != null) {
- var.waitForCompletion();
- }
- }
-
- @Override
- public void reportProfile(String id, List<JobProfile> profiles) throws Exception {
- jobQueue.schedule(new ReportProfilesEvent(this, profiles));
- }
-
- @Override
- public synchronized void nodeHeartbeat(String id, HeartbeatData hbData) throws Exception {
- jobQueue.schedule(new NodeHeartbeatEvent(this, id, hbData));
- }
-
- @Override
- public void createApplication(String appName) throws Exception {
- synchronized (applications) {
- if (applications.containsKey(appName)) {
- throw new HyracksException("Duplicate application with name: " + appName + " being created.");
- }
- CCApplicationContext appCtx = new CCApplicationContext(serverCtx, ccContext, appName);
- applications.put(appName, appCtx);
- }
- }
-
- @Override
- public void destroyApplication(String appName) throws Exception {
- FutureValue fv = new FutureValue();
- jobQueue.schedule(new ApplicationDestroyEvent(this, appName, fv));
- fv.get();
- }
-
- @Override
- public void startApplication(final String appName) throws Exception {
- FutureValue fv = new FutureValue();
- jobQueue.schedule(new ApplicationStartEvent(this, appName, fv));
- fv.get();
- }
-
- @Override
- public ClusterControllerInfo getClusterControllerInfo() throws Exception {
+ public ClusterControllerInfo getClusterControllerInfo() {
return info;
}
- @Override
- public void registerPartitionProvider(PartitionDescriptor partitionDescriptor) {
- jobQueue.schedule(new RegisterPartitionAvailibilityEvent(this, partitionDescriptor));
+ public CCConfig getCCConfig() {
+ return ccConfig;
}
- @Override
- public void registerPartitionRequest(PartitionRequest partitionRequest) {
- jobQueue.schedule(new RegisterPartitionRequestEvent(this, partitionRequest));
+ public IPCSystem getClusterIPC() {
+ return clusterIPC;
}
private class DeadNodeSweeper extends TimerTask {
@Override
public void run() {
- jobQueue.schedule(new RemoveDeadNodesEvent(ClusterControllerService.this));
+ workQueue.schedule(new RemoveDeadNodesWork(ClusterControllerService.this));
+ }
+ }
+
+ private class HyracksClientInterfaceIPCI implements IIPCI {
+ @Override
+ public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
+ HyracksClientInterfaceFunctions.Function fn = (HyracksClientInterfaceFunctions.Function) payload;
+ switch (fn.getFunctionId()) {
+ case GET_CLUSTER_CONTROLLER_INFO: {
+ try {
+ handle.send(mid, info, null);
+ } catch (IPCException e) {
+ e.printStackTrace();
+ }
+ return;
+ }
+
+ case CREATE_APPLICATION: {
+ HyracksClientInterfaceFunctions.CreateApplicationFunction caf = (HyracksClientInterfaceFunctions.CreateApplicationFunction) fn;
+ workQueue.schedule(new ApplicationCreateWork(ClusterControllerService.this, caf.getAppName(),
+ new IPCResponder<Object>(handle, mid)));
+ return;
+ }
+
+ case START_APPLICATION: {
+ HyracksClientInterfaceFunctions.StartApplicationFunction saf = (HyracksClientInterfaceFunctions.StartApplicationFunction) fn;
+ workQueue.schedule(new ApplicationStartWork(ClusterControllerService.this, saf.getAppName(),
+ new IPCResponder<Object>(handle, mid)));
+ return;
+ }
+
+ case DESTROY_APPLICATION: {
+ HyracksClientInterfaceFunctions.DestroyApplicationFunction daf = (HyracksClientInterfaceFunctions.DestroyApplicationFunction) fn;
+ workQueue.schedule(new ApplicationDestroyWork(ClusterControllerService.this, daf.getAppName(),
+ new IPCResponder<Object>(handle, mid)));
+ return;
+ }
+
+ case CREATE_JOB: {
+ HyracksClientInterfaceFunctions.CreateJobFunction cjf = (HyracksClientInterfaceFunctions.CreateJobFunction) fn;
+ JobId jobId = createJobId();
+ workQueue.schedule(new JobCreateWork(ClusterControllerService.this, jobId, cjf.getAppName(), cjf
+ .getJobSpec(), cjf.getJobFlags(), new IPCResponder<JobId>(handle, mid)));
+ return;
+ }
+
+ case GET_JOB_STATUS: {
+ HyracksClientInterfaceFunctions.GetJobStatusFunction gjsf = (HyracksClientInterfaceFunctions.GetJobStatusFunction) fn;
+ workQueue.schedule(new GetJobStatusWork(ClusterControllerService.this, gjsf.getJobId(),
+ new IPCResponder<JobStatus>(handle, mid)));
+ return;
+ }
+
+ case START_JOB: {
+ HyracksClientInterfaceFunctions.StartJobFunction sjf = (HyracksClientInterfaceFunctions.StartJobFunction) fn;
+ workQueue.schedule(new JobStartWork(ClusterControllerService.this, sjf.getJobId(),
+ new IPCResponder<Object>(handle, mid)));
+ return;
+ }
+
+ case WAIT_FOR_COMPLETION: {
+ HyracksClientInterfaceFunctions.WaitForCompletionFunction wfcf = (HyracksClientInterfaceFunctions.WaitForCompletionFunction) fn;
+ workQueue.schedule(new WaitForJobCompletionWork(ClusterControllerService.this, wfcf.getJobId(),
+ new IPCResponder<Object>(handle, mid)));
+ return;
+ }
+
+ case GET_NODE_CONTROLLERS_INFO: {
+ workQueue.schedule(new GetNodeControllersInfoWork(ClusterControllerService.this,
+ new IPCResponder<Map<String, NodeControllerInfo>>(handle, mid)));
+ return;
+ }
+ }
+ try {
+ handle.send(mid, null, new IllegalArgumentException("Unknown function " + fn.getFunctionId()));
+ } catch (IPCException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private class ClusterControllerIPCI implements IIPCI {
+ @Override
+ public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
+ CCNCFunctions.Function fn = (Function) payload;
+ switch (fn.getFunctionId()) {
+ case REGISTER_NODE: {
+ CCNCFunctions.RegisterNodeFunction rnf = (CCNCFunctions.RegisterNodeFunction) fn;
+ workQueue.schedule(new RegisterNodeWork(ClusterControllerService.this, rnf.getNodeRegistration()));
+ return;
+ }
+
+ case UNREGISTER_NODE: {
+ CCNCFunctions.UnregisterNodeFunction unf = (CCNCFunctions.UnregisterNodeFunction) fn;
+ workQueue.schedule(new UnregisterNodeWork(ClusterControllerService.this, unf.getNodeId()));
+ return;
+ }
+
+ case NODE_HEARTBEAT: {
+ CCNCFunctions.NodeHeartbeatFunction nhf = (CCNCFunctions.NodeHeartbeatFunction) fn;
+ workQueue.schedule(new NodeHeartbeatWork(ClusterControllerService.this, nhf.getNodeId(), nhf
+ .getHeartbeatData()));
+ return;
+ }
+
+ case NOTIFY_JOBLET_CLEANUP: {
+ CCNCFunctions.NotifyJobletCleanupFunction njcf = (CCNCFunctions.NotifyJobletCleanupFunction) fn;
+ workQueue.schedule(new JobletCleanupNotificationWork(ClusterControllerService.this,
+ njcf.getJobId(), njcf.getNodeId()));
+ return;
+ }
+
+ case REPORT_PROFILE: {
+ CCNCFunctions.ReportProfileFunction rpf = (CCNCFunctions.ReportProfileFunction) fn;
+ workQueue.schedule(new ReportProfilesWork(ClusterControllerService.this, rpf.getProfiles()));
+ return;
+ }
+
+ case NOTIFY_TASK_COMPLETE: {
+ CCNCFunctions.NotifyTaskCompleteFunction ntcf = (CCNCFunctions.NotifyTaskCompleteFunction) fn;
+ workQueue.schedule(new TaskCompleteWork(ClusterControllerService.this, ntcf.getJobId(), ntcf
+ .getTaskId(), ntcf.getNodeId(), ntcf.getStatistics()));
+ return;
+ }
+ case NOTIFY_TASK_FAILURE: {
+ CCNCFunctions.NotifyTaskFailureFunction ntff = (CCNCFunctions.NotifyTaskFailureFunction) fn;
+ workQueue.schedule(new TaskFailureWork(ClusterControllerService.this, ntff.getJobId(), ntff
+ .getTaskId(), ntff.getDetails(), ntff.getDetails()));
+ return;
+ }
+
+ case REGISTER_PARTITION_PROVIDER: {
+ CCNCFunctions.RegisterPartitionProviderFunction rppf = (CCNCFunctions.RegisterPartitionProviderFunction) fn;
+ workQueue.schedule(new RegisterPartitionAvailibilityWork(ClusterControllerService.this, rppf
+ .getPartitionDescriptor()));
+ return;
+ }
+
+ case REGISTER_PARTITION_REQUEST: {
+ CCNCFunctions.RegisterPartitionRequestFunction rprf = (CCNCFunctions.RegisterPartitionRequestFunction) fn;
+ workQueue.schedule(new RegisterPartitionRequestWork(ClusterControllerService.this, rprf
+ .getPartitionRequest()));
+ return;
+ }
+
+ case APPLICATION_STATE_CHANGE_RESPONSE: {
+ CCNCFunctions.ApplicationStateChangeResponseFunction astrf = (CCNCFunctions.ApplicationStateChangeResponseFunction) fn;
+ workQueue.schedule(new ApplicationStateChangeWork(ClusterControllerService.this, astrf));
+ return;
+ }
+ }
+ LOGGER.warning("Unknown function: " + fn.getFunctionId());
}
}
}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java
index 21fac2f..c17acd0 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/NodeControllerState.java
@@ -15,8 +15,11 @@
package edu.uci.ics.hyracks.control.cc;
import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
import java.util.Set;
+import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -26,6 +29,8 @@
import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema.GarbageCollectorInfo;
public class NodeControllerState {
private static final int RRD_SIZE = 720;
@@ -46,6 +51,24 @@
private final int nProcessors;
+ private final String vmName;
+
+ private final String vmVersion;
+
+ private final String vmVendor;
+
+ private final String classpath;
+
+ private final String libraryPath;
+
+ private final String bootClasspath;
+
+ private final List<String> inputArguments;
+
+ private final Map<String, String> systemProperties;
+
+ private final HeartbeatSchema hbSchema;
+
private final long[] hbTime;
private final long[] heapInitSize;
@@ -70,6 +93,28 @@
private final double[] systemLoadAverage;
+ private final String[] gcNames;
+
+ private final long[][] gcCollectionCounts;
+
+ private final long[][] gcCollectionTimes;
+
+ private final long[] netPayloadBytesRead;
+
+ private final long[] netPayloadBytesWritten;
+
+ private final long[] netSignalingBytesRead;
+
+ private final long[] netSignalingBytesWritten;
+
+ private final long[] ipcMessagesSent;
+
+ private final long[] ipcMessageBytesSent;
+
+ private final long[] ipcMessagesReceived;
+
+ private final long[] ipcMessageBytesReceived;
+
private int rrdPtr;
private int lastHeartbeatDuration;
@@ -84,6 +129,16 @@
arch = reg.getArch();
osVersion = reg.getOSVersion();
nProcessors = reg.getNProcessors();
+ vmName = reg.getVmName();
+ vmVersion = reg.getVmVersion();
+ vmVendor = reg.getVmVendor();
+ classpath = reg.getClasspath();
+ libraryPath = reg.getLibraryPath();
+ bootClasspath = reg.getBootClasspath();
+ inputArguments = reg.getInputArguments();
+ systemProperties = reg.getSystemProperties();
+
+ hbSchema = reg.getHeartbeatSchema();
hbTime = new long[RRD_SIZE];
heapInitSize = new long[RRD_SIZE];
@@ -97,6 +152,23 @@
threadCount = new int[RRD_SIZE];
peakThreadCount = new int[RRD_SIZE];
systemLoadAverage = new double[RRD_SIZE];
+ GarbageCollectorInfo[] gcInfos = hbSchema.getGarbageCollectorInfos();
+ int gcN = gcInfos.length;
+ gcNames = new String[gcN];
+ for (int i = 0; i < gcN; ++i) {
+ gcNames[i] = gcInfos[i].getName();
+ }
+ gcCollectionCounts = new long[gcN][RRD_SIZE];
+ gcCollectionTimes = new long[gcN][RRD_SIZE];
+ netPayloadBytesRead = new long[RRD_SIZE];
+ netPayloadBytesWritten = new long[RRD_SIZE];
+ netSignalingBytesRead = new long[RRD_SIZE];
+ netSignalingBytesWritten = new long[RRD_SIZE];
+ ipcMessagesSent = new long[RRD_SIZE];
+ ipcMessageBytesSent = new long[RRD_SIZE];
+ ipcMessagesReceived = new long[RRD_SIZE];
+ ipcMessageBytesReceived = new long[RRD_SIZE];
+
rrdPtr = 0;
}
@@ -115,6 +187,19 @@
threadCount[rrdPtr] = hbData.threadCount;
peakThreadCount[rrdPtr] = hbData.peakThreadCount;
systemLoadAverage[rrdPtr] = hbData.systemLoadAverage;
+ int gcN = hbSchema.getGarbageCollectorInfos().length;
+ for (int i = 0; i < gcN; ++i) {
+ gcCollectionCounts[i][rrdPtr] = hbData.gcCollectionCounts[i];
+ gcCollectionTimes[i][rrdPtr] = hbData.gcCollectionTimes[i];
+ }
+ netPayloadBytesRead[rrdPtr] = hbData.netPayloadBytesRead;
+ netPayloadBytesWritten[rrdPtr] = hbData.netPayloadBytesWritten;
+ netSignalingBytesRead[rrdPtr] = hbData.netSignalingBytesRead;
+ netSignalingBytesWritten[rrdPtr] = hbData.netSignalingBytesWritten;
+ ipcMessagesSent[rrdPtr] = hbData.ipcMessagesSent;
+ ipcMessageBytesSent[rrdPtr] = hbData.ipcMessageBytesSent;
+ ipcMessagesReceived[rrdPtr] = hbData.ipcMessagesReceived;
+ ipcMessageBytesReceived[rrdPtr] = hbData.ipcMessageBytesReceived;
rrdPtr = (rrdPtr + 1) % RRD_SIZE;
}
@@ -159,6 +244,13 @@
o.put("arch", arch);
o.put("os-version", osVersion);
o.put("num-processors", nProcessors);
+ o.put("vm-name", vmName);
+ o.put("vm-version", vmVersion);
+ o.put("vm-vendor", vmVendor);
+ o.put("classpath", classpath);
+ o.put("library-path", libraryPath);
+ o.put("boot-classpath", bootClasspath);
+ o.put("input-arguments", new JSONArray(inputArguments));
o.put("rrd-ptr", rrdPtr);
o.put("heartbeat-times", hbTime);
o.put("heap-init-sizes", heapInitSize);
@@ -172,6 +264,17 @@
o.put("thread-counts", threadCount);
o.put("peak-thread-counts", peakThreadCount);
o.put("system-load-averages", systemLoadAverage);
+ o.put("gc-names", gcNames);
+ o.put("gc-collection-counts", gcCollectionCounts);
+ o.put("gc-collection-times", gcCollectionTimes);
+ o.put("net-payload-bytes-read", netPayloadBytesRead);
+ o.put("net-payload-bytes-written", netPayloadBytesWritten);
+ o.put("net-signaling-bytes-read", netSignalingBytesRead);
+ o.put("net-signaling-bytes-written", netSignalingBytesWritten);
+ o.put("ipc-messages-sent", ipcMessagesSent);
+ o.put("ipc-message-bytes-sent", ipcMessageBytesSent);
+ o.put("ipc-messages-received", ipcMessagesReceived);
+ o.put("ipc-message-bytes-received", ipcMessageBytesReceived);
return o;
}
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/HyracksAdminConsoleApplication.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/HyracksAdminConsoleApplication.java
new file mode 100644
index 0000000..a6113de
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/HyracksAdminConsoleApplication.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.adminconsole;
+
+import org.apache.wicket.Page;
+import org.apache.wicket.protocol.http.WebApplication;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.adminconsole.pages.IndexPage;
+
+public class HyracksAdminConsoleApplication extends WebApplication {
+ private ClusterControllerService ccs;
+
+ @Override
+ public void init() {
+ ccs = (ClusterControllerService) getServletContext().getAttribute(ClusterControllerService.class.getName());
+ }
+
+ @Override
+ public Class<? extends Page> getHomePage() {
+ return IndexPage.class;
+ }
+
+ public ClusterControllerService getClusterControllerService() {
+ return ccs;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/AbstractPage.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/AbstractPage.java
new file mode 100644
index 0000000..cc51228
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/AbstractPage.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.adminconsole.pages;
+
+import org.apache.wicket.markup.html.WebPage;
+
+import edu.uci.ics.hyracks.control.cc.adminconsole.HyracksAdminConsoleApplication;
+
+public class AbstractPage extends WebPage {
+ private static final long serialVersionUID = 1L;
+
+ public HyracksAdminConsoleApplication getAdminConsoleApplication() {
+ return (HyracksAdminConsoleApplication) getApplication();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/IndexPage.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/IndexPage.java
new file mode 100644
index 0000000..3b9be36
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/IndexPage.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.adminconsole.pages;
+
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.markup.html.link.BookmarkablePageLink;
+import org.apache.wicket.markup.html.list.ListItem;
+import org.apache.wicket.markup.html.list.ListView;
+import org.apache.wicket.request.mapper.parameter.PageParameters;
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.web.util.JSONUtils;
+import edu.uci.ics.hyracks.control.cc.work.GetJobSummariesJSONWork;
+import edu.uci.ics.hyracks.control.cc.work.GetNodeSummariesJSONWork;
+
+public class IndexPage extends AbstractPage {
+ private static final long serialVersionUID = 1L;
+
+ public IndexPage() throws Exception {
+ ClusterControllerService ccs = getAdminConsoleApplication().getClusterControllerService();
+
+ GetNodeSummariesJSONWork gnse = new GetNodeSummariesJSONWork(ccs);
+ ccs.getWorkQueue().scheduleAndSync(gnse);
+ JSONArray nodeSummaries = gnse.getSummaries();
+ add(new Label("node-count", String.valueOf(nodeSummaries.length())));
+ ListView<JSONObject> nodeList = new ListView<JSONObject>("node-list", JSONUtils.toList(nodeSummaries)) {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected void populateItem(ListItem<JSONObject> item) {
+ JSONObject o = item.getModelObject();
+ try {
+ item.add(new Label("node-id", o.getString("node-id")));
+ item.add(new Label("heap-used", o.getString("heap-used")));
+ item.add(new Label("system-load-average", o.getString("system-load-average")));
+ PageParameters params = new PageParameters();
+ params.add("node-id", o.getString("node-id"));
+ item.add(new BookmarkablePageLink<Object>("node-details", NodeDetailsPage.class, params));
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ add(nodeList);
+
+ GetJobSummariesJSONWork gjse = new GetJobSummariesJSONWork(ccs);
+ ccs.getWorkQueue().scheduleAndSync(gjse);
+ JSONArray jobSummaries = gjse.getSummaries();
+ ListView<JSONObject> jobList = new ListView<JSONObject>("jobs-list", JSONUtils.toList(jobSummaries)) {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ protected void populateItem(ListItem<JSONObject> item) {
+ JSONObject o = item.getModelObject();
+ try {
+ item.add(new Label("job-id", o.getString("job-id")));
+ item.add(new Label("application-name", o.getString("application-name")));
+ item.add(new Label("status", o.getString("status")));
+ item.add(new Label("create-time", o.getString("create-time")));
+ item.add(new Label("start-time", o.getString("start-time")));
+ item.add(new Label("end-time", o.getString("end-time")));
+ PageParameters params = new PageParameters();
+ params.add("job-id", o.getString("job-id"));
+ item.add(new BookmarkablePageLink<Object>("job-details", JobDetailsPage.class, params));
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ add(jobList);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java
new file mode 100644
index 0000000..1d96a71
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.java
@@ -0,0 +1,337 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.adminconsole.pages;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.wicket.markup.html.basic.Label;
+import org.apache.wicket.request.mapper.parameter.PageParameters;
+import org.apache.wicket.util.string.StringValue;
+import org.json.JSONArray;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.work.GetJobActivityGraphJSONWork;
+import edu.uci.ics.hyracks.control.cc.work.GetJobRunJSONWork;
+import edu.uci.ics.hyracks.control.cc.work.GetJobSpecificationJSONWork;
+
+public class JobDetailsPage extends AbstractPage {
+ private static final long serialVersionUID = 1L;
+
+ private static final int HEIGHT = 29;
+
+ public JobDetailsPage(PageParameters params) throws Exception {
+ ClusterControllerService ccs = getAdminConsoleApplication().getClusterControllerService();
+
+ StringValue jobIdStr = params.get("job-id");
+
+ JobId jobId = JobId.parse(jobIdStr.toString());
+
+ GetJobSpecificationJSONWork gjsw = new GetJobSpecificationJSONWork(ccs, jobId);
+ ccs.getWorkQueue().scheduleAndSync(gjsw);
+ Label jobspec = new Label("job-specification", gjsw.getJSON().toString());
+ jobspec.setEscapeModelStrings(false);
+ add(jobspec);
+
+ GetJobActivityGraphJSONWork gjagw = new GetJobActivityGraphJSONWork(ccs, jobId);
+ ccs.getWorkQueue().scheduleAndSync(gjagw);
+ Label jag = new Label("job-activity-graph", gjagw.getJSON().toString());
+ jag.setEscapeModelStrings(false);
+ add(jag);
+
+ JSONObject jagO = gjagw.getJSON();
+
+ Map<ActivityId, String> activityMap = new HashMap<ActivityId, String>();
+ if (jagO.has("activities")) {
+ JSONArray aArray = jagO.getJSONArray("activities");
+ for (int i = 0; i < aArray.length(); ++i) {
+ JSONObject aO = aArray.getJSONObject(i);
+ ActivityId aid = ActivityId.parse(aO.getString("id"));
+ String className = aO.getString("java-class");
+
+ activityMap.put(aid, className);
+ }
+ }
+
+ GetJobRunJSONWork gjrw = new GetJobRunJSONWork(ccs, jobId);
+ ccs.getWorkQueue().scheduleAndSync(gjrw);
+ Label jobrun = new Label("job-run", gjrw.getJSON().toString());
+ jobrun.setEscapeModelStrings(false);
+ add(jobrun);
+
+ JSONObject jrO = gjrw.getJSON();
+
+ List<TaskClusterAttempt[]> tcList = new ArrayList<TaskClusterAttempt[]>();
+ long minTime = Long.MAX_VALUE;
+ long maxTime = Long.MIN_VALUE;
+ if (jrO.has("activity-clusters")) {
+ JSONArray acA = jrO.getJSONArray("activity-clusters");
+ for (int i = 0; i < acA.length(); ++i) {
+ JSONObject acO = acA.getJSONObject(i);
+ if (acO.has("plan")) {
+ JSONObject planO = acO.getJSONObject("plan");
+ if (planO.has("task-clusters")) {
+ JSONArray tcA = planO.getJSONArray("task-clusters");
+ for (int j = 0; j < tcA.length(); ++j) {
+ JSONObject tcO = tcA.getJSONObject(j);
+ String tcId = tcO.getString("task-cluster-id");
+ if (tcO.has("attempts")) {
+ JSONArray tcaA = tcO.getJSONArray("attempts");
+ TaskClusterAttempt[] tcAttempts = new TaskClusterAttempt[tcaA.length()];
+ for (int k = 0; k < tcaA.length(); ++k) {
+ JSONObject tcaO = tcaA.getJSONObject(k);
+ int attempt = tcaO.getInt("attempt");
+ long startTime = tcaO.getLong("start-time");
+ long endTime = tcaO.getLong("end-time");
+
+ tcAttempts[k] = new TaskClusterAttempt(tcId, attempt, startTime, endTime);
+ if (startTime < minTime) {
+ minTime = startTime;
+ }
+ if (endTime > maxTime) {
+ maxTime = endTime;
+ }
+ if (tcaO.has("task-attempts")) {
+ JSONArray taArray = tcaO.getJSONArray("task-attempts");
+ tcAttempts[k].tasks = new TaskAttempt[taArray.length()];
+ for (int l = 0; l < taArray.length(); ++l) {
+ JSONObject taO = taArray.getJSONObject(l);
+ TaskAttemptId taId = TaskAttemptId.parse(taO.getString("task-attempt-id"));
+ TaskAttempt ta = new TaskAttempt(taId, taO.getLong("start-time"),
+ taO.getLong("end-time"));
+ tcAttempts[k].tasks[l] = ta;
+ TaskId tid = taId.getTaskId();
+ ta.name = activityMap.get(tid.getActivityId());
+ ta.partition = tid.getPartition();
+ }
+ Arrays.sort(tcAttempts[k].tasks, new Comparator<TaskAttempt>() {
+ @Override
+ public int compare(TaskAttempt o1, TaskAttempt o2) {
+ return o1.startTime < o2.startTime ? -1
+ : (o1.startTime > o2.startTime ? 1 : 0);
+ }
+ });
+ }
+ }
+ Arrays.sort(tcAttempts, new Comparator<TaskClusterAttempt>() {
+ @Override
+ public int compare(TaskClusterAttempt o1, TaskClusterAttempt o2) {
+ return o1.startTime < o2.startTime ? -1 : (o1.startTime > o2.startTime ? 1 : 0);
+ }
+ });
+ tcList.add(tcAttempts);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ Map<TaskAttemptId, TaskProfile> tpMap = new HashMap<TaskAttemptId, TaskProfile>();
+ if (jrO.has("profile")) {
+ JSONObject pO = jrO.getJSONObject("profile");
+ if (pO.has("joblets")) {
+ JSONArray jobletsA = pO.getJSONArray("joblets");
+ for (int i = 0; i < jobletsA.length(); ++i) {
+ JSONObject jobletO = jobletsA.getJSONObject(i);
+ if (jobletO.has("tasks")) {
+ JSONArray tasksA = jobletO.getJSONArray("tasks");
+ for (int j = 0; j < tasksA.length(); ++j) {
+ JSONObject taskO = tasksA.getJSONObject(j);
+ ActivityId activityId = ActivityId.parse(taskO.getString("activity-id"));
+ int partition = taskO.getInt("partition");
+ int attempt = taskO.getInt("attempt");
+ TaskAttemptId taId = new TaskAttemptId(new TaskId(activityId, partition), attempt);
+ if (taskO.has("partition-send-profile")) {
+ JSONArray taskProfilesA = taskO.getJSONArray("partition-send-profile");
+ for (int k = 0; k < taskProfilesA.length(); ++k) {
+ JSONObject ppO = taskProfilesA.getJSONObject(k);
+ long openTime = ppO.getLong("open-time");
+ long closeTime = ppO.getLong("close-time");
+ int resolution = ppO.getInt("resolution");
+ long offset = ppO.getLong("offset");
+ JSONArray frameTimesA = ppO.getJSONArray("frame-times");
+ long[] frameTimes = new long[frameTimesA.length()];
+ for (int l = 0; l < frameTimes.length; ++l) {
+ frameTimes[l] = frameTimesA.getInt(l) + offset;
+ }
+ TaskProfile tp = new TaskProfile(taId, openTime, closeTime, frameTimes, resolution);
+ if (!tpMap.containsKey(tp.taId)) {
+ tpMap.put(tp.taId, tp);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (!tcList.isEmpty()) {
+ Collections.sort(tcList, new Comparator<TaskClusterAttempt[]>() {
+ @Override
+ public int compare(TaskClusterAttempt[] o1, TaskClusterAttempt[] o2) {
+ if (o1.length == 0) {
+ return o2.length == 0 ? 0 : -1;
+ } else if (o2.length == 0) {
+ return 1;
+ }
+ return o1[0].startTime < o2[0].startTime ? -1 : (o1[0].startTime > o2[0].startTime ? 1 : 0);
+ }
+ });
+ long range = maxTime - minTime;
+
+ double leftOffset = 20;
+
+ int xWidth = 1024;
+ double width = ((double) xWidth) / range;
+ StringBuilder buffer = new StringBuilder();
+ int y = 0;
+ for (TaskClusterAttempt[] tcAttempts : tcList) {
+ for (int i = 0; i < tcAttempts.length; ++i) {
+ TaskClusterAttempt tca = tcAttempts[i];
+ long startTime = tca.startTime - minTime;
+ long endTime = tca.endTime - minTime;
+ buffer.append("<rect x=\"").append(startTime * width + leftOffset).append("\" y=\"")
+ .append(y * (HEIGHT + 1)).append("\" width=\"").append(width * (endTime - startTime))
+ .append("\" height=\"").append(HEIGHT).append("\"/>\n");
+ buffer.append("<text x=\"").append(endTime * width + leftOffset + 20).append("\" y=\"")
+ .append(y * (HEIGHT + 1) + HEIGHT * 3 / 4).append("\">")
+ .append((endTime - startTime) + " ms").append("</text>\n");
+ ++y;
+ for (int j = 0; j < tca.tasks.length; ++j) {
+ TaskAttempt ta = tca.tasks[j];
+ long tStartTime = ta.startTime - minTime;
+ long tEndTime = ta.endTime - minTime;
+ buffer.append("<rect x=\"").append(tStartTime * width + leftOffset).append("\" y=\"")
+ .append(y * (HEIGHT + 1) + HEIGHT / 4).append("\" width=\"")
+ .append(width * (tEndTime - tStartTime)).append("\" height=\"").append(HEIGHT / 2)
+ .append("\" style=\"fill:rgb(255,255,255);stroke-width:1;stroke:rgb(0,0,0)\"/>\n");
+ buffer.append("<text x=\"").append(tEndTime * width + leftOffset + 20).append("\" y=\"")
+ .append(y * (HEIGHT + 1) + HEIGHT * 3 / 4).append("\">")
+ .append((tEndTime - tStartTime) + " ms (" + ta.name + ":" + ta.partition + ")")
+ .append("</text>\n");
+ TaskProfile tp = tpMap.get(ta.taId);
+ if (tp != null) {
+ for (int k = 0; k < tp.frameTimes.length; ++k) {
+ long taOpenTime = tp.openTime - minTime;
+ buffer.append("<rect x=\"")
+ .append(taOpenTime * width + leftOffset)
+ .append("\" y=\"")
+ .append(y * (HEIGHT + 1) + HEIGHT / 4)
+ .append("\" width=\"1\" height=\"")
+ .append(HEIGHT / 2)
+ .append("\" style=\"fill:rgb(255,0,0);stroke-width:1;stroke:rgb(255,0,0)\"/>\n");
+ for (int l = 0; l < tp.frameTimes.length; ++l) {
+ long ft = tp.frameTimes[l];
+ long ftn = l < tp.frameTimes.length - 1 ? tp.frameTimes[l + 1] : ft;
+ long taNextTime = ft - minTime;
+ long barWidth = ftn - ft;
+ buffer.append("<rect x=\"")
+ .append(taNextTime * width + leftOffset)
+ .append("\" y=\"")
+ .append(y * (HEIGHT + 1) + HEIGHT / 4)
+ .append("\" width=\"")
+ .append(barWidth == 0 ? 1 : (barWidth * width))
+ .append("\" height=\"")
+ .append(HEIGHT / 2)
+ .append("\" style=\"fill:rgb(0,255,0);stroke-width:1;stroke:rgb(0,255,0)\"/>\n");
+ }
+ long taCloseTime = tp.closeTime - minTime;
+ buffer.append("<rect x=\"")
+ .append(taCloseTime * width + leftOffset)
+ .append("\" y=\"")
+ .append(y * (HEIGHT + 1) + HEIGHT / 4)
+ .append("\" width=\"1\" height=\"")
+ .append(HEIGHT / 2)
+ .append("\" style=\"fill:rgb(0,0,255);stroke-width:1;stroke:rgb(0,0,255)\"/>\n");
+ }
+ }
+ ++y;
+ }
+ }
+ }
+ buffer.append("<rect x=\"").append(leftOffset).append("\" y=\"").append(0).append("\" width=\"").append(1)
+ .append("\" height=\"").append((y + 2) * (HEIGHT + 1)).append("\"/>\n");
+ buffer.append("<rect x=\"").append(0).append("\" y=\"").append((y + 1) * (HEIGHT + 1))
+ .append("\" width=\"").append(xWidth + 2 * leftOffset).append("\" height=\"").append(1)
+ .append("\"/>\n");
+ buffer.append("</svg>");
+ Label markup = new Label("job-timeline",
+ "<svg version=\"1.1\"\nxmlns=\"http://www.w3.org/2000/svg\" width=\"" + (xWidth * 1.5)
+ + "\" height=\"" + ((y + 2) * (HEIGHT + 1)) + "\">\n" + buffer.toString());
+ markup.setEscapeModelStrings(false);
+ add(markup);
+ } else {
+ Label markup = new Label("job-timeline", "No information available yet");
+ add(markup);
+ }
+ }
+
+ private static class TaskAttempt {
+ private TaskAttemptId taId;
+ private long startTime;
+ private long endTime;
+ private String name;
+ private int partition;
+
+ public TaskAttempt(TaskAttemptId taId, long startTime, long endTime) {
+ this.taId = taId;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+ }
+
+ private static class TaskClusterAttempt {
+ private String tcId;
+ private int attempt;
+ private long startTime;
+ private long endTime;
+ private TaskAttempt[] tasks;
+
+ public TaskClusterAttempt(String tcId, int attempt, long startTime, long endTime) {
+ this.tcId = tcId;
+ this.attempt = attempt;
+ this.startTime = startTime;
+ this.endTime = endTime;
+ }
+ }
+
+ private static class TaskProfile {
+ private TaskAttemptId taId;
+ private long openTime;
+ private long closeTime;
+ private long[] frameTimes;
+ private int resolution;
+
+ public TaskProfile(TaskAttemptId taId, long openTime, long closeTime, long[] frameTimes, int resolution) {
+ this.taId = taId;
+ this.openTime = openTime;
+ this.closeTime = closeTime;
+ this.frameTimes = frameTimes;
+ this.resolution = resolution;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.java
new file mode 100644
index 0000000..072e8fb
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.adminconsole.pages;
+
+import org.apache.wicket.request.mapper.parameter.PageParameters;
+
+public class NodeDetailsPage extends AbstractPage {
+ private static final long serialVersionUID = 1L;
+
+ public NodeDetailsPage(PageParameters params) throws Exception {
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java
index c1c0161..28f7c59 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/application/CCApplicationContext.java
@@ -17,7 +17,9 @@
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
import edu.uci.ics.hyracks.api.application.ICCBootstrap;
@@ -30,10 +32,17 @@
import edu.uci.ics.hyracks.control.cc.job.DeserializingJobSpecificationFactory;
import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
import edu.uci.ics.hyracks.control.common.context.ServerContext;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
public class CCApplicationContext extends ApplicationContext implements ICCApplicationContext {
private final ICCContext ccContext;
+ protected final Set<String> initPendingNodeIds;
+ protected final Set<String> deinitPendingNodeIds;
+
+ protected IResultCallback<Object> initializationCallback;
+ protected IResultCallback<Object> deinitializationCallback;
+
private IJobSpecificationFactory jobSpecFactory;
private List<IJobLifecycleListener> jobLifecycleListeners;
@@ -41,6 +50,8 @@
public CCApplicationContext(ServerContext serverCtx, ICCContext ccContext, String appName) throws IOException {
super(serverCtx, appName);
this.ccContext = ccContext;
+ initPendingNodeIds = new HashSet<String>();
+ deinitPendingNodeIds = new HashSet<String>();
jobSpecFactory = DeserializingJobSpecificationFactory.INSTANCE;
jobLifecycleListeners = new ArrayList<IJobLifecycleListener>();
}
@@ -98,4 +109,28 @@
l.notifyJobCreation(jobId, specification);
}
}
+
+ public Set<String> getInitializationPendingNodeIds() {
+ return initPendingNodeIds;
+ }
+
+ public Set<String> getDeinitializationPendingNodeIds() {
+ return deinitPendingNodeIds;
+ }
+
+ public IResultCallback<Object> getInitializationCallback() {
+ return initializationCallback;
+ }
+
+ public void setInitializationCallback(IResultCallback<Object> initializationCallback) {
+ this.initializationCallback = initializationCallback;
+ }
+
+ public IResultCallback<Object> getDeinitializationCallback() {
+ return deinitializationCallback;
+ }
+
+ public void setDeinitializationCallback(IResultCallback<Object> deinitializationCallback) {
+ this.deinitializationCallback = deinitializationCallback;
+ }
}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityCluster.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityCluster.java
index 14cebb2..13db627 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityCluster.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/ActivityCluster.java
@@ -71,8 +71,8 @@
this.id = id;
}
- public int getMaxTaskClusterAttempts() {
- return jobRun.getJobActivityGraph().getJobSpecification().getMaxAttempts();
+ public int getMaxTaskClusterReattempts() {
+ return jobRun.getJobActivityGraph().getJobSpecification().getMaxReattempts();
}
public ActivityClusterPlan getPlan() {
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java
index 8fdde7c..707ef1f 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/JobRun.java
@@ -14,8 +14,6 @@
*/
package edu.uci.ics.hyracks.control.cc.job;
-import java.io.ByteArrayOutputStream;
-import java.io.PrintWriter;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -49,6 +47,8 @@
private final Set<String> participatingNodeIds;
+ private final Set<String> cleanupPendingNodeIds;
+
private final JobProfile profile;
private Set<ActivityCluster> activityClusters;
@@ -69,11 +69,16 @@
private Exception exception;
+ private JobStatus pendingStatus;
+
+ private Exception pendingException;
+
public JobRun(JobId jobId, JobActivityGraph plan) {
this.jobId = jobId;
this.jag = plan;
pmm = new PartitionMatchMaker();
participatingNodeIds = new HashSet<String>();
+ cleanupPendingNodeIds = new HashSet<String>();
profile = new JobProfile(jobId);
activityClusterMap = new HashMap<ActivityId, ActivityCluster>();
connectorPolicyMap = new HashMap<ConnectorDescriptorId, IConnectorPolicy>();
@@ -101,6 +106,23 @@
return status;
}
+ public synchronized Exception getException() {
+ return exception;
+ }
+
+ public void setPendingStatus(JobStatus status, Exception exception) {
+ this.pendingStatus = status;
+ this.pendingException = exception;
+ }
+
+ public JobStatus getPendingStatus() {
+ return pendingStatus;
+ }
+
+ public synchronized Exception getPendingException() {
+ return pendingException;
+ }
+
public long getCreateTime() {
return createTime;
}
@@ -125,10 +147,6 @@
this.endTime = endTime;
}
- public synchronized Exception getException() {
- return exception;
- }
-
@Override
public synchronized void waitForCompletion() throws Exception {
while (status != JobStatus.TERMINATED && status != JobStatus.FAILURE) {
@@ -143,6 +161,10 @@
return participatingNodeIds;
}
+ public Set<String> getCleanupPendingNodeIds() {
+ return cleanupPendingNodeIds;
+ }
+
public JobProfile getJobProfile() {
return profile;
}
@@ -175,6 +197,8 @@
JSONObject result = new JSONObject();
result.put("job-id", jobId.toString());
+ result.put("application-name", jag.getApplicationName());
+ result.put("status", getStatus());
result.put("create-time", getCreateTime());
result.put("start-time", getCreateTime());
result.put("end-time", getCreateTime());
@@ -260,7 +284,7 @@
acTasks.put(entry);
}
- planJSON.put("task-map", acTasks);
+ planJSON.put("activities", acTasks);
JSONArray tClusters = new JSONArray();
for (TaskCluster tc : acp.getTaskClusters()) {
@@ -292,25 +316,21 @@
JSONObject attempt = new JSONObject();
attempt.put("attempt", tca.getAttempt());
attempt.put("status", tca.getStatus());
+ attempt.put("start-time", tca.getStartTime());
+ attempt.put("end-time", tca.getEndTime());
JSONArray taskAttempts = new JSONArray();
- for (TaskAttempt ta : tca.getTaskAttempts()) {
+ for (TaskAttempt ta : tca.getTaskAttempts().values()) {
JSONObject taskAttempt = new JSONObject();
+ taskAttempt.put("task-id", ta.getTaskAttemptId().getTaskId());
taskAttempt.put("task-attempt-id", ta.getTaskAttemptId());
taskAttempt.put("status", ta.getStatus());
taskAttempt.put("node-id", ta.getNodeId());
- Exception e = ta.getException();
- if (e != null) {
- JSONObject ex = new JSONObject();
- ex.put("exception-class", e.getClass().getName());
- ex.put("exception-message", e.getMessage());
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintWriter pw = new PrintWriter(baos);
- e.printStackTrace(pw);
- pw.close();
- ex.put("exception-stacktrace", new String(baos.toByteArray()));
-
- taskAttempt.put("exception", ex);
+ taskAttempt.put("start-time", ta.getStartTime());
+ taskAttempt.put("end-time", ta.getEndTime());
+ String failureDetails = ta.getFailureDetails();
+ if (failureDetails != null) {
+ taskAttempt.put("failure-details", failureDetails);
}
taskAttempts.put(taskAttempt);
}
@@ -331,6 +351,8 @@
}
result.put("activity-clusters", aClusters);
+ result.put("profile", profile.toJSON());
+
return result;
}
}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskAttempt.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskAttempt.java
index 22f5587..7c0dd57 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskAttempt.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskAttempt.java
@@ -29,18 +29,24 @@
private final TaskAttemptId taskId;
- private final Task taskState;
+ private final Task task;
private String nodeId;
private TaskStatus status;
- private Exception exception;
+ private String failureDetails;
- public TaskAttempt(TaskClusterAttempt tcAttempt, TaskAttemptId taskId, Task taskState) {
+ private long startTime;
+
+ private long endTime;
+
+ public TaskAttempt(TaskClusterAttempt tcAttempt, TaskAttemptId taskId, Task task) {
this.tcAttempt = tcAttempt;
this.taskId = taskId;
- this.taskState = taskState;
+ this.task = task;
+ startTime = -1;
+ endTime = -1;
}
public TaskClusterAttempt getTaskClusterAttempt() {
@@ -51,8 +57,8 @@
return taskId;
}
- public Task getTaskState() {
- return taskState;
+ public Task getTask() {
+ return task;
}
public String getNodeId() {
@@ -67,12 +73,28 @@
return status;
}
- public Exception getException() {
- return exception;
+ public String getFailureDetails() {
+ return failureDetails;
}
- public void setStatus(TaskStatus status, Exception exception) {
+ public void setStatus(TaskStatus status, String details) {
this.status = status;
- this.exception = exception;
+ this.failureDetails = details;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public long getEndTime() {
+ return endTime;
+ }
+
+ public void setEndTime(long endTime) {
+ this.endTime = endTime;
}
}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterAttempt.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterAttempt.java
index 9d33d12..2b74585 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterAttempt.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/TaskClusterAttempt.java
@@ -14,6 +14,10 @@
*/
package edu.uci.ics.hyracks.control.cc.job;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
+
public class TaskClusterAttempt {
public enum TaskClusterStatus {
RUNNING,
@@ -26,26 +30,32 @@
private final int attempt;
- private TaskAttempt[] taskAttempts;
+ private Map<TaskId, TaskAttempt> taskAttempts;
private TaskClusterStatus status;
private int pendingTaskCounter;
+ private long startTime;
+
+ private long endTime;
+
public TaskClusterAttempt(TaskCluster taskCluster, int attempt) {
this.taskCluster = taskCluster;
this.attempt = attempt;
+ startTime = -1;
+ endTime = -1;
}
public TaskCluster getTaskCluster() {
return taskCluster;
}
- public void setTaskAttempts(TaskAttempt[] taskAttempts) {
+ public void setTaskAttempts(Map<TaskId, TaskAttempt> taskAttempts) {
this.taskAttempts = taskAttempts;
}
- public TaskAttempt[] getTaskAttempts() {
+ public Map<TaskId, TaskAttempt> getTaskAttempts() {
return taskAttempts;
}
@@ -61,8 +71,24 @@
return status;
}
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
+
+ public long getEndTime() {
+ return endTime;
+ }
+
+ public void setEndTime(long endTime) {
+ this.endTime = endTime;
+ }
+
public void initializePendingTaskCounter() {
- pendingTaskCounter = taskAttempts.length;
+ pendingTaskCounter = taskAttempts.size();
}
public int getPendingTaskCounter() {
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/AbstractTaskLifecycleEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/AbstractTaskLifecycleEvent.java
deleted file mode 100644
index 0028b6f..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/AbstractTaskLifecycleEvent.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.List;
-import java.util.Map;
-
-import edu.uci.ics.hyracks.api.dataflow.ActivityId;
-import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
-import edu.uci.ics.hyracks.control.cc.job.ActivityPlan;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.job.Task;
-import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
-import edu.uci.ics.hyracks.control.cc.job.TaskCluster;
-import edu.uci.ics.hyracks.control.cc.job.TaskClusterAttempt;
-import edu.uci.ics.hyracks.control.cc.jobqueue.AbstractEvent;
-
-public abstract class AbstractTaskLifecycleEvent extends AbstractEvent {
- protected final ClusterControllerService ccs;
- protected final JobId jobId;
- protected final TaskAttemptId taId;
- protected final String nodeId;
-
- public AbstractTaskLifecycleEvent(ClusterControllerService ccs, JobId jobId, TaskAttemptId taId, String nodeId) {
- this.ccs = ccs;
- this.jobId = jobId;
- this.taId = taId;
- this.nodeId = nodeId;
- }
-
- @Override
- public final void run() {
- JobRun run = ccs.getRunMap().get(jobId);
- if (run != null) {
- TaskId tid = taId.getTaskId();
- Map<ActivityId, ActivityCluster> activityClusterMap = run.getActivityClusterMap();
- ActivityCluster ac = activityClusterMap.get(tid.getActivityId());
- if (ac != null) {
- Map<ActivityId, ActivityPlan> taskStateMap = ac.getPlan().getActivityPlanMap();
- Task[] taskStates = taskStateMap.get(tid.getActivityId()).getTasks();
- if (taskStates != null && taskStates.length > tid.getPartition()) {
- Task ts = taskStates[tid.getPartition()];
- TaskCluster tc = ts.getTaskCluster();
- List<TaskClusterAttempt> taskClusterAttempts = tc.getAttempts();
- if (taskClusterAttempts != null && taskClusterAttempts.size() > taId.getAttempt()) {
- TaskClusterAttempt tca = taskClusterAttempts.get(taId.getAttempt());
- for (TaskAttempt ta : tca.getTaskAttempts()) {
- if (ta.getTaskAttemptId().equals(taId)) {
- performEvent(ta);
- break;
- }
- }
- }
- }
- }
- }
- }
-
- protected abstract void performEvent(TaskAttempt ta);
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/ApplicationDestroyEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/ApplicationDestroyEvent.java
deleted file mode 100644
index 050c6ed..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/ApplicationDestroyEvent.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.jobqueue.AbstractEvent;
-import edu.uci.ics.hyracks.control.cc.jobqueue.FutureValue;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteRunner;
-import edu.uci.ics.hyracks.control.cc.remote.ops.ApplicationDestroyer;
-import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
-
-public class ApplicationDestroyEvent extends AbstractEvent {
- private final ClusterControllerService ccs;
- private final String appName;
- private FutureValue fv;
-
- public ApplicationDestroyEvent(ClusterControllerService ccs, String appName, FutureValue fv) {
- this.ccs = ccs;
- this.appName = appName;
- this.fv = fv;
- }
-
- @Override
- public void run() {
- final ApplicationContext appCtx = ccs.getApplicationMap().remove(appName);
- if (appCtx == null) {
- fv.setException(new HyracksException("No application with name: " + appName));
- return;
- }
- List<RemoteOp<Void>> opList = new ArrayList<RemoteOp<Void>>();
- for (final String nodeId : ccs.getNodeMap().keySet()) {
- opList.add(new ApplicationDestroyer(nodeId, appName));
- }
- final RemoteOp[] ops = opList.toArray(new RemoteOp[opList.size()]);
- ccs.getExecutor().execute(new Runnable() {
- @Override
- public void run() {
- try {
- RemoteRunner.runRemote(ccs, ops, null);
- } catch (Exception e) {
- fv.setException(e);
- return;
- }
- ccs.getJobQueue().schedule(new AbstractEvent() {
- @Override
- public void run() {
- try {
- appCtx.deinitialize();
- } catch (Exception e) {
- fv.setException(e);
- }
- fv.setValue(null);
- }
- });
- }
- });
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/ApplicationStartEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/ApplicationStartEvent.java
deleted file mode 100644
index ab05d0f..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/ApplicationStartEvent.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.jobqueue.AbstractEvent;
-import edu.uci.ics.hyracks.control.cc.jobqueue.FutureValue;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteRunner;
-import edu.uci.ics.hyracks.control.cc.remote.ops.ApplicationStarter;
-import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
-
-public class ApplicationStartEvent extends AbstractEvent {
- private final ClusterControllerService ccs;
- private final String appName;
- private final FutureValue fv;
-
- public ApplicationStartEvent(ClusterControllerService ccs, String appName, FutureValue fv) {
- this.ccs = ccs;
- this.appName = appName;
- this.fv = fv;
- }
-
- @Override
- public void run() {
- ApplicationContext appCtx = ccs.getApplicationMap().get(appName);
- if (appCtx == null) {
- fv.setException(new HyracksException("No application with name: " + appName));
- return;
- }
- try {
- appCtx.initializeClassPath();
- appCtx.initialize();
- final byte[] distributedState = JavaSerializationUtils.serialize(appCtx.getDestributedState());
- final boolean deployHar = appCtx.containsHar();
- List<RemoteOp<Void>> opList = new ArrayList<RemoteOp<Void>>();
- for (final String nodeId : ccs.getNodeMap().keySet()) {
- opList.add(new ApplicationStarter(nodeId, appName, deployHar, distributedState));
- }
- final RemoteOp[] ops = opList.toArray(new RemoteOp[opList.size()]);
- ccs.getExecutor().execute(new Runnable() {
- @Override
- public void run() {
- try {
- RemoteRunner.runRemote(ccs, ops, null);
- fv.setValue(null);
- } catch (Exception e) {
- fv.setException(e);
- }
- }
- });
- } catch (Exception e) {
- fv.setException(e);
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobActivityGraphJSONEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobActivityGraphJSONEvent.java
deleted file mode 100644
index 14b56df..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobActivityGraphJSONEvent.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import org.json.JSONObject;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class GetJobActivityGraphJSONEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final JobId jobId;
- private JSONObject json;
-
- public GetJobActivityGraphJSONEvent(ClusterControllerService ccs, JobId jobId) {
- this.ccs = ccs;
- this.jobId = jobId;
- }
-
- @Override
- protected void doRun() throws Exception {
- JobRun run = ccs.getRunMap().get(jobId);
- if (run == null) {
- json = new JSONObject();
- return;
- }
- json = run.getJobActivityGraph().toJSON();
- }
-
- public JSONObject getJSON() {
- return json;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobProfileJSONEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobProfileJSONEvent.java
deleted file mode 100644
index 4355962..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobProfileJSONEvent.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import org.json.JSONObject;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class GetJobProfileJSONEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final JobId jobId;
- private JSONObject json;
-
- public GetJobProfileJSONEvent(ClusterControllerService ccs, JobId jobId) {
- this.ccs = ccs;
- this.jobId = jobId;
- }
-
- @Override
- protected void doRun() throws Exception {
- json = new JSONObject();
- JobRun jobRun = ccs.getRunMap().get(jobId);
- if (jobRun == null) {
- json = new JSONObject();
- return;
- }
- json = jobRun.getJobProfile().toJSON();
- }
-
- public JSONObject getJSON() {
- return json;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobRunJSONEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobRunJSONEvent.java
deleted file mode 100644
index 3e19a70..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobRunJSONEvent.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import org.json.JSONObject;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class GetJobRunJSONEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final JobId jobId;
- private JSONObject json;
-
- public GetJobRunJSONEvent(ClusterControllerService ccs, JobId jobId) {
- this.ccs = ccs;
- this.jobId = jobId;
- }
-
- @Override
- protected void doRun() throws Exception {
- JobRun run = ccs.getRunMap().get(jobId);
- if (run == null) {
- json = new JSONObject();
- return;
- }
- json = run.toJSON();
- }
-
- public JSONObject getJSON() {
- return json;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobSpecificationJSONEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobSpecificationJSONEvent.java
deleted file mode 100644
index db7e5a7..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobSpecificationJSONEvent.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import org.json.JSONObject;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class GetJobSpecificationJSONEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final JobId jobId;
- private JSONObject json;
-
- public GetJobSpecificationJSONEvent(ClusterControllerService ccs, JobId jobId) {
- this.ccs = ccs;
- this.jobId = jobId;
- }
-
- @Override
- protected void doRun() throws Exception {
- JobRun run = ccs.getRunMap().get(jobId);
- if (run == null) {
- json = new JSONObject();
- return;
- }
- json = run.getJobActivityGraph().getJobSpecification().toJSON();
- }
-
- public JSONObject getJSON() {
- return json;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobStatusConditionVariableEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobStatusConditionVariableEvent.java
deleted file mode 100644
index 357d62b..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobStatusConditionVariableEvent.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.IJobStatusConditionVariable;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class GetJobStatusConditionVariableEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final JobId jobId;
- private IJobStatusConditionVariable cVar;
-
- public GetJobStatusConditionVariableEvent(ClusterControllerService ccs, JobId jobId) {
- this.ccs = ccs;
- this.jobId = jobId;
- }
-
- @Override
- protected void doRun() throws Exception {
- cVar = ccs.getRunMap().get(jobId);
- }
-
- public IJobStatusConditionVariable getConditionVariable() {
- return cVar;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobStatusEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobStatusEvent.java
deleted file mode 100644
index fbe6e4c..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobStatusEvent.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class GetJobStatusEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final JobId jobId;
- private JobStatus status;
-
- public GetJobStatusEvent(ClusterControllerService ccs, JobId jobId) {
- this.ccs = ccs;
- this.jobId = jobId;
- }
-
- @Override
- protected void doRun() throws Exception {
- JobRun run = ccs.getRunMap().get(jobId);
- status = run == null ? null : run.getStatus();
- }
-
- public JobStatus getStatus() {
- return status;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobSummariesJSONEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobSummariesJSONEvent.java
deleted file mode 100644
index eda4338..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetJobSummariesJSONEvent.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import org.json.JSONArray;
-import org.json.JSONObject;
-
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class GetJobSummariesJSONEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private JSONArray summaries;
-
- public GetJobSummariesJSONEvent(ClusterControllerService ccs) {
- this.ccs = ccs;
- }
-
- @Override
- protected void doRun() throws Exception {
- summaries = new JSONArray();
- for (JobRun run : ccs.getRunMap().values()) {
- JSONObject jo = new JSONObject();
- jo.put("type", "job-summary");
- jo.put("job-id", run.getJobId().toString());
- jo.put("application-name", run.getJobActivityGraph().getApplicationName());
- jo.put("create-time", run.getCreateTime());
- jo.put("start-time", run.getCreateTime());
- jo.put("end-time", run.getCreateTime());
- jo.put("status", run.getStatus().toString());
- summaries.put(jo);
- }
- }
-
- public JSONArray getSummaries() {
- return summaries;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetNodeDetailsJSONEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetNodeDetailsJSONEvent.java
deleted file mode 100644
index 284ec35..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetNodeDetailsJSONEvent.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import org.json.JSONObject;
-
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class GetNodeDetailsJSONEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final String nodeId;
- private JSONObject detail;
-
- public GetNodeDetailsJSONEvent(ClusterControllerService ccs, String nodeId) {
- this.ccs = ccs;
- this.nodeId = nodeId;
- }
-
- @Override
- protected void doRun() throws Exception {
- NodeControllerState ncs = ccs.getNodeMap().get(nodeId);
- if (ncs == null) {
- detail = new JSONObject();
- return;
- }
- detail = ncs.toDetailedJSON();
- }
-
- public JSONObject getDetail() {
- return detail;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetNodeSummariesJSONEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetNodeSummariesJSONEvent.java
deleted file mode 100644
index 512bce2..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/GetNodeSummariesJSONEvent.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import org.json.JSONArray;
-
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class GetNodeSummariesJSONEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private JSONArray summaries;
-
- public GetNodeSummariesJSONEvent(ClusterControllerService ccs) {
- this.ccs = ccs;
- }
-
- @Override
- protected void doRun() throws Exception {
- summaries = new JSONArray();
- for (NodeControllerState ncs : ccs.getNodeMap().values()) {
- summaries.put(ncs.toSummaryJSON());
- }
- }
-
- public JSONArray getSummaries() {
- return summaries;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/JobCleanupEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/JobCleanupEvent.java
deleted file mode 100644
index 4274914..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/JobCleanupEvent.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.Set;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.AbstractEvent;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteRunner;
-import edu.uci.ics.hyracks.control.cc.remote.ops.JobCompleteNotifier;
-
-public class JobCleanupEvent extends AbstractEvent {
- private ClusterControllerService ccs;
- private JobId jobId;
- private JobStatus status;
- private Exception exception;
-
- public JobCleanupEvent(ClusterControllerService ccs, JobId jobId, JobStatus status, Exception exception) {
- this.ccs = ccs;
- this.jobId = jobId;
- this.status = status;
- this.exception = exception;
- }
-
- @Override
- public void run() {
- final JobRun run = ccs.getRunMap().get(jobId);
- Set<String> targetNodes = run.getParticipatingNodeIds();
- final JobCompleteNotifier[] jcns = new JobCompleteNotifier[targetNodes.size()];
- int i = 0;
- for (String n : targetNodes) {
- jcns[i++] = new JobCompleteNotifier(n, jobId);
- NodeControllerState ncs = ccs.getNodeMap().get(n);
- if (ncs != null) {
- ncs.getActiveJobIds().remove(jobId);
- }
- }
- ccs.getExecutor().execute(new Runnable() {
- @Override
- public void run() {
- if (jcns.length > 0) {
- try {
- RemoteRunner.runRemote(ccs, jcns, null);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- ccs.getJobQueue().schedule(new AbstractEvent() {
- @Override
- public void run() {
- CCApplicationContext appCtx = ccs.getApplicationMap().get(
- run.getJobActivityGraph().getApplicationName());
- if (appCtx != null) {
- try {
- appCtx.notifyJobFinish(jobId);
- } catch (HyracksException e) {
- e.printStackTrace();
- }
- }
- run.setStatus(status, exception);
- }
- });
- }
- });
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/JobCreateEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/JobCreateEvent.java
deleted file mode 100644
index 5fadb00..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/JobCreateEvent.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.EnumSet;
-
-import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
-import edu.uci.ics.hyracks.api.job.JobFlag;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
-import edu.uci.ics.hyracks.control.cc.job.IOperatorDescriptorVisitor;
-import edu.uci.ics.hyracks.control.cc.job.JobActivityGraphBuilder;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.job.PlanUtils;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-import edu.uci.ics.hyracks.control.cc.scheduler.JobScheduler;
-
-public class JobCreateEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final byte[] jobSpec;
- private final EnumSet<JobFlag> jobFlags;
- private final JobId jobId;
- private final String appName;
-
- public JobCreateEvent(ClusterControllerService ccs, JobId jobId, String appName, byte[] jobSpec,
- EnumSet<JobFlag> jobFlags) {
- this.jobId = jobId;
- this.ccs = ccs;
- this.jobSpec = jobSpec;
- this.jobFlags = jobFlags;
- this.appName = appName;
- }
-
- @Override
- protected void doRun() throws Exception {
- CCApplicationContext appCtx = ccs.getApplicationMap().get(appName);
- if (appCtx == null) {
- throw new HyracksException("No application with id " + appName + " found");
- }
- JobSpecification spec = appCtx.createJobSpecification(jobSpec);
-
- final JobActivityGraphBuilder builder = new JobActivityGraphBuilder();
- builder.init(appName, spec, jobFlags);
- PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {
- @Override
- public void visit(IOperatorDescriptor op) {
- op.contributeActivities(builder);
- }
- });
- final JobActivityGraph jag = builder.getActivityGraph();
-
- JobRun run = new JobRun(jobId, jag);
-
- run.setStatus(JobStatus.INITIALIZED, null);
-
- ccs.getRunMap().put(jobId, run);
- JobScheduler jrs = new JobScheduler(ccs, run);
- run.setScheduler(jrs);
- appCtx.notifyJobCreation(jobId, spec);
- }
-
- public JobId getJobId() {
- return jobId;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/JobStartEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/JobStartEvent.java
deleted file mode 100644
index 8f6357c..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/JobStartEvent.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobStatus;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class JobStartEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final JobId jobId;
-
- public JobStartEvent(ClusterControllerService ccs, JobId jobId) {
- this.ccs = ccs;
- this.jobId = jobId;
- }
-
- @Override
- protected void doRun() throws Exception {
- JobRun run = ccs.getRunMap().get(jobId);
- if (run == null) {
- throw new Exception("Unable to find job with id = " + jobId);
- }
- if (run.getStatus() != JobStatus.INITIALIZED) {
- throw new Exception("Job already started");
- }
- run.setStatus(JobStatus.RUNNING, null);
- try {
- run.getScheduler().startJob();
- } catch (Exception e) {
- ccs.getJobQueue().schedule(new JobCleanupEvent(ccs, run.getJobId(), JobStatus.FAILURE, e));
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/NodeHeartbeatEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/NodeHeartbeatEvent.java
deleted file mode 100644
index 98d17ad..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/NodeHeartbeatEvent.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.Map;
-import java.util.logging.Level;
-
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
-
-public class NodeHeartbeatEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final String nodeId;
- private final HeartbeatData hbData;
-
- public NodeHeartbeatEvent(ClusterControllerService ccs, String nodeId, HeartbeatData hbData) {
- this.ccs = ccs;
- this.nodeId = nodeId;
- this.hbData = hbData;
- }
-
- @Override
- protected void doRun() throws Exception {
- Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
- NodeControllerState state = nodeMap.get(nodeId);
- if (state != null) {
- state.notifyHeartbeat(hbData);
- }
- }
-
- @Override
- public Level logLevel() {
- return Level.FINEST;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RegisterNodeEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RegisterNodeEvent.java
deleted file mode 100644
index 5f9377b..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RegisterNodeEvent.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class RegisterNodeEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final String nodeId;
- private final NodeControllerState state;
-
- public RegisterNodeEvent(ClusterControllerService ccs, String nodeId, NodeControllerState state) {
- this.ccs = ccs;
- this.nodeId = nodeId;
- this.state = state;
- }
-
- @Override
- protected void doRun() throws Exception {
- Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
- if (nodeMap.containsKey(nodeId)) {
- throw new Exception("Node with this name already registered.");
- }
- nodeMap.put(nodeId, state);
- Map<String, Set<String>> ipAddressNodeNameMap = ccs.getIPAddressNodeNameMap();
- String ipAddress = state.getNCConfig().dataIPAddress;
- Set<String> nodes = ipAddressNodeNameMap.get(ipAddress);
- if (nodes == null) {
- nodes = new HashSet<String>();
- ipAddressNodeNameMap.put(ipAddress, nodes);
- }
- nodes.add(nodeId);
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RegisterPartitionAvailibilityEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RegisterPartitionAvailibilityEvent.java
deleted file mode 100644
index 3f49b97..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RegisterPartitionAvailibilityEvent.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.List;
-
-import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.AbstractEvent;
-import edu.uci.ics.hyracks.control.cc.partitions.PartitionMatchMaker;
-import edu.uci.ics.hyracks.control.cc.partitions.PartitionUtils;
-import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
-import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
-
-public class RegisterPartitionAvailibilityEvent extends AbstractEvent {
- private final ClusterControllerService ccs;
- private final PartitionDescriptor partitionDescriptor;
-
- public RegisterPartitionAvailibilityEvent(ClusterControllerService ccs, PartitionDescriptor partitionDescriptor) {
- this.ccs = ccs;
- this.partitionDescriptor = partitionDescriptor;
- }
-
- @Override
- public void run() {
- final PartitionId pid = partitionDescriptor.getPartitionId();
- JobRun run = ccs.getRunMap().get(pid.getJobId());
- if (run == null) {
- return;
- }
- PartitionMatchMaker pmm = run.getPartitionMatchMaker();
- List<Pair<PartitionDescriptor, PartitionRequest>> matches = pmm
- .registerPartitionDescriptor(partitionDescriptor);
- for (Pair<PartitionDescriptor, PartitionRequest> match : matches) {
- PartitionUtils.reportPartitionMatch(ccs, pid, match);
- }
- }
-
- @Override
- public String toString() {
- return "PartitionAvailable@" + partitionDescriptor;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RegisterPartitionRequestEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RegisterPartitionRequestEvent.java
deleted file mode 100644
index bac8dd1..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RegisterPartitionRequestEvent.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.AbstractEvent;
-import edu.uci.ics.hyracks.control.cc.partitions.PartitionMatchMaker;
-import edu.uci.ics.hyracks.control.cc.partitions.PartitionUtils;
-import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
-import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
-
-public class RegisterPartitionRequestEvent extends AbstractEvent {
- private final ClusterControllerService ccs;
- private final PartitionRequest partitionRequest;
-
- public RegisterPartitionRequestEvent(ClusterControllerService ccs, PartitionRequest partitionRequest) {
- this.ccs = ccs;
- this.partitionRequest = partitionRequest;
- }
-
- @Override
- public void run() {
- PartitionId pid = partitionRequest.getPartitionId();
- JobRun run = ccs.getRunMap().get(pid.getJobId());
- if (run == null) {
- return;
- }
- PartitionMatchMaker pmm = run.getPartitionMatchMaker();
- Pair<PartitionDescriptor, PartitionRequest> match = pmm.matchPartitionRequest(partitionRequest);
- if (match != null) {
- PartitionUtils.reportPartitionMatch(ccs, pid, match);
- }
- }
-
- @Override
- public String toString() {
- return "PartitionRequest@" + partitionRequest;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RemoveDeadNodesEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RemoveDeadNodesEvent.java
deleted file mode 100644
index b812594..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/RemoveDeadNodesEvent.java
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.AbstractEvent;
-
-public class RemoveDeadNodesEvent extends AbstractEvent {
- private static Logger LOGGER = Logger.getLogger(RemoveDeadNodesEvent.class.getName());
-
- private final ClusterControllerService ccs;
-
- public RemoveDeadNodesEvent(ClusterControllerService ccs) {
- this.ccs = ccs;
- }
-
- @Override
- public void run() {
- Set<String> deadNodes = new HashSet<String>();
- Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
- for (Map.Entry<String, NodeControllerState> e : nodeMap.entrySet()) {
- NodeControllerState state = e.getValue();
- if (state.incrementLastHeartbeatDuration() >= ccs.getConfig().maxHeartbeatLapsePeriods) {
- deadNodes.add(e.getKey());
- LOGGER.info(e.getKey() + " considered dead");
- }
- }
- Set<JobId> affectedJobIds = new HashSet<JobId>();
- Map<String, Set<String>> ipAddressNodeNameMap = ccs.getIPAddressNodeNameMap();
- for (String deadNode : deadNodes) {
- NodeControllerState state = nodeMap.remove(deadNode);
-
- // Deal with dead tasks.
- affectedJobIds.addAll(state.getActiveJobIds());
-
- String ipAddress = state.getNCConfig().dataIPAddress;
- Set<String> ipNodes = ipAddressNodeNameMap.get(ipAddress);
- if (ipNodes != null) {
- if (ipNodes.remove(deadNode) && ipNodes.isEmpty()) {
- ipAddressNodeNameMap.remove(ipAddress);
- }
- }
- }
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Number of affected jobs: " + affectedJobIds.size());
- }
- for (JobId jobId : affectedJobIds) {
- JobRun run = ccs.getRunMap().get(jobId);
- if (run != null) {
- run.getScheduler().notifyNodeFailures(deadNodes);
- }
- }
- }
-
- @Override
- public Level logLevel() {
- return Level.FINE;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/ReportProfilesEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/ReportProfilesEvent.java
deleted file mode 100644
index bd82f75..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/ReportProfilesEvent.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.JobRun;
-import edu.uci.ics.hyracks.control.cc.jobqueue.AbstractEvent;
-import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
-
-public class ReportProfilesEvent extends AbstractEvent {
- private final ClusterControllerService ccs;
- private final List<JobProfile> profiles;
-
- public ReportProfilesEvent(ClusterControllerService ccs, List<JobProfile> profiles) {
- this.ccs = ccs;
- this.profiles = profiles;
- }
-
- @Override
- public void run() {
- Map<JobId, JobRun> runMap = ccs.getRunMap();
- for (JobProfile profile : profiles) {
- JobRun run = runMap.get(profile.getJobId());
- }
- }
-
- @Override
- public Level logLevel() {
- return Level.FINEST;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/TaskCompleteEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/TaskCompleteEvent.java
deleted file mode 100644
index 19fd95b..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/TaskCompleteEvent.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
-import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
-
-public class TaskCompleteEvent extends AbstractTaskLifecycleEvent {
- public TaskCompleteEvent(ClusterControllerService ccs, JobId jobId, TaskAttemptId taId, String nodeId) {
- super(ccs, jobId, taId, nodeId);
- }
-
- @Override
- protected void performEvent(TaskAttempt ta) {
- try {
- ActivityCluster ac = ta.getTaskState().getTaskCluster().getActivityCluster();
- ac.getJobRun().getScheduler().notifyTaskComplete(ta);
- } catch (HyracksException e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public String toString() {
- return "TaskCompleteEvent@[" + nodeId + "[" + jobId + ":" + taId + "]";
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/TaskFailureEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/TaskFailureEvent.java
deleted file mode 100644
index 8b7e821..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/TaskFailureEvent.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
-import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
-
-public class TaskFailureEvent extends AbstractTaskLifecycleEvent {
- private final Exception exception;
-
- public TaskFailureEvent(ClusterControllerService ccs, JobId jobId, TaskAttemptId taId, String nodeId,
- Exception exception) {
- super(ccs, jobId, taId, nodeId);
- this.exception = exception;
- }
-
- @Override
- protected void performEvent(TaskAttempt ta) {
- ActivityCluster ac = ta.getTaskState().getTaskCluster().getActivityCluster();
- ac.getJobRun().getScheduler().notifyTaskFailure(ta, ac, exception);
- }
-
- @Override
- public String toString() {
- return "TaskFailureEvent[" + jobId + ":" + taId + ":" + nodeId + "]";
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/UnregisterNodeEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/UnregisterNodeEvent.java
deleted file mode 100644
index b89de07..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/job/manager/events/UnregisterNodeEvent.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.job.manager.events;
-
-import java.util.Map;
-
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
-
-public class UnregisterNodeEvent extends SynchronizableEvent {
- private final ClusterControllerService ccs;
- private final String nodeId;
-
- public UnregisterNodeEvent(ClusterControllerService ccs, String nodeId) {
- this.ccs = ccs;
- this.nodeId = nodeId;
- }
-
- @Override
- protected void doRun() throws Exception {
- Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
- nodeMap.remove(nodeId);
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/AbstractEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/AbstractEvent.java
deleted file mode 100644
index 5fcd56a..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/AbstractEvent.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.jobqueue;
-
-import java.util.logging.Level;
-
-public abstract class AbstractEvent implements Runnable {
- public Level logLevel() {
- return Level.INFO;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/FutureValue.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/FutureValue.java
deleted file mode 100644
index 25378c6..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/FutureValue.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.jobqueue;
-
-public class FutureValue {
- private boolean done;
-
- private Object value;
-
- private Exception e;
-
- public FutureValue() {
- done = false;
- value = null;
- e = null;
- }
-
- public synchronized void setValue(Object value) {
- done = true;
- this.value = value;
- e = null;
- notifyAll();
- }
-
- public synchronized void setException(Exception e) {
- done = true;
- this.e = e;
- value = null;
- notifyAll();
- }
-
- public synchronized void reset() {
- done = false;
- value = null;
- e = null;
- notifyAll();
- }
-
- public synchronized Object get() throws Exception {
- while (!done) {
- wait();
- }
- if (e != null) {
- throw e;
- }
- return value;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/JobQueue.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/JobQueue.java
deleted file mode 100644
index 84a844c..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/JobQueue.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.jobqueue;
-
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.logging.Logger;
-
-public class JobQueue {
- private static final Logger LOGGER = Logger.getLogger(JobQueue.class.getName());
-
- private final LinkedBlockingQueue<AbstractEvent> queue;
- private final JobThread thread;
-
- public JobQueue() {
- queue = new LinkedBlockingQueue<AbstractEvent>();
- thread = new JobThread();
- thread.start();
- }
-
- public void schedule(AbstractEvent event) {
- if (LOGGER.isLoggable(event.logLevel())) {
- LOGGER.info("Scheduling: " + event);
- }
- queue.offer(event);
- }
-
- public void scheduleAndSync(SynchronizableEvent sRunnable) throws Exception {
- schedule(sRunnable);
- sRunnable.sync();
- }
-
- private class JobThread extends Thread {
- JobThread() {
- setDaemon(true);
- }
-
- @Override
- public void run() {
- Runnable r;
- while (true) {
- try {
- r = queue.take();
- } catch (InterruptedException e) {
- continue;
- }
- try {
- r.run();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/SynchronizableEvent.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/SynchronizableEvent.java
deleted file mode 100644
index 88c9097..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/jobqueue/SynchronizableEvent.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.jobqueue;
-
-public abstract class SynchronizableEvent extends AbstractEvent {
- private boolean done;
-
- private Exception e;
-
- protected abstract void doRun() throws Exception;
-
- public void init() {
- done = false;
- e = null;
- }
-
- @Override
- public final void run() {
- try {
- doRun();
- } catch (Exception e) {
- this.e = e;
- } finally {
- synchronized (this) {
- done = true;
- notifyAll();
- }
- }
- }
-
- public final synchronized void sync() throws Exception {
- while (!done) {
- try {
- wait();
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- if (e != null) {
- throw e;
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionMatchMaker.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionMatchMaker.java
index afa1946..2dcbb3e 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionMatchMaker.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionMatchMaker.java
@@ -22,9 +22,10 @@
import java.util.Set;
import java.util.logging.Logger;
+import org.apache.commons.lang3.tuple.Pair;
+
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
import edu.uci.ics.hyracks.control.common.job.PartitionState;
@@ -52,7 +53,7 @@
while (i.hasNext()) {
PartitionRequest req = i.next();
if (partitionDescriptor.getState().isAtLeast(req.getMinimumState())) {
- matches.add(new Pair<PartitionDescriptor, PartitionRequest>(partitionDescriptor, req));
+ matches.add(Pair.<PartitionDescriptor, PartitionRequest> of(partitionDescriptor, req));
i.remove();
matched = true;
if (!partitionDescriptor.isReusable()) {
@@ -88,7 +89,7 @@
while (i.hasNext()) {
PartitionDescriptor descriptor = i.next();
if (descriptor.getState().isAtLeast(partitionRequest.getMinimumState())) {
- match = new Pair<PartitionDescriptor, PartitionRequest>(descriptor, partitionRequest);
+ match = Pair.<PartitionDescriptor, PartitionRequest> of(descriptor, partitionRequest);
if (!descriptor.isReusable()) {
i.remove();
}
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionUtils.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionUtils.java
index 90c6ad1..c36cf62 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionUtils.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/partitions/PartitionUtils.java
@@ -1,8 +1,9 @@
package edu.uci.ics.hyracks.control.cc.partitions;
+import org.apache.commons.lang3.tuple.Pair;
+
import edu.uci.ics.hyracks.api.comm.NetworkAddress;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
import edu.uci.ics.hyracks.control.cc.NodeControllerState;
import edu.uci.ics.hyracks.control.common.base.INodeController;
@@ -11,23 +12,14 @@
public class PartitionUtils {
public static void reportPartitionMatch(ClusterControllerService ccs, final PartitionId pid,
- Pair<PartitionDescriptor, PartitionRequest> match) {
- PartitionDescriptor desc = match.first;
- PartitionRequest req = match.second;
+ Pair<PartitionDescriptor, PartitionRequest> match) throws Exception {
+ PartitionDescriptor desc = match.getLeft();
+ PartitionRequest req = match.getRight();
NodeControllerState producerNCS = ccs.getNodeMap().get(desc.getNodeId());
NodeControllerState requestorNCS = ccs.getNodeMap().get(req.getNodeId());
final NetworkAddress dataport = producerNCS.getDataPort();
final INodeController requestorNC = requestorNCS.getNodeController();
- ccs.getExecutor().execute(new Runnable() {
- @Override
- public void run() {
- try {
- requestorNC.reportPartitionAvailability(pid, dataport);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
+ requestorNC.reportPartitionAvailability(pid, dataport);
}
}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
deleted file mode 100644
index c728b0b..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/Accumulator.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote;
-
-public interface Accumulator<T, R> {
- public void accumulate(T o);
-
- public R getResult();
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteOp.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteOp.java
deleted file mode 100644
index ec15186..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteOp.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote;
-
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public interface RemoteOp<T> {
- public String getNodeId();
-
- public T execute(INodeController node) throws Exception;
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteRunner.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteRunner.java
deleted file mode 100644
index a37c786..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/RemoteRunner.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote;
-
-import java.util.List;
-import java.util.Vector;
-import java.util.concurrent.Semaphore;
-
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.NodeControllerState;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public class RemoteRunner {
- public static <T, R> R runRemote(ClusterControllerService ccs, final RemoteOp<T>[] remoteOps,
- final Accumulator<T, R> accumulator) throws Exception {
- final Semaphore installComplete = new Semaphore(remoteOps.length);
- final List<Exception> errors = new Vector<Exception>();
- for (final RemoteOp<T> remoteOp : remoteOps) {
- NodeControllerState nodeState = ccs.getNodeMap().get(remoteOp.getNodeId());
- final INodeController node = nodeState.getNodeController();
-
- installComplete.acquire();
- Runnable remoteRunner = new Runnable() {
- @Override
- public void run() {
- try {
- T t = remoteOp.execute(node);
- if (accumulator != null) {
- synchronized (accumulator) {
- accumulator.accumulate(t);
- }
- }
- } catch (Exception e) {
- errors.add(e);
- } finally {
- installComplete.release();
- }
- }
- };
-
- ccs.getExecutor().execute(remoteRunner);
- }
- installComplete.acquire(remoteOps.length);
- if (!errors.isEmpty()) {
- throw errors.get(0);
- }
- return accumulator == null ? null : accumulator.getResult();
- }
-}
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationDestroyer.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationDestroyer.java
deleted file mode 100644
index 52e726c..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationDestroyer.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote.ops;
-
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public class ApplicationDestroyer implements RemoteOp<Void> {
- private String nodeId;
- private String appName;
-
- public ApplicationDestroyer(String nodeId, String appName) {
- this.nodeId = nodeId;
- this.appName = appName;
- }
-
- @Override
- public String getNodeId() {
- return nodeId;
- }
-
- @Override
- public Void execute(INodeController node) throws Exception {
- node.destroyApplication(appName);
- return null;
- }
-
- @Override
- public String toString() {
- return "Destroyed application: " + appName;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationStarter.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationStarter.java
deleted file mode 100644
index d6a1d26..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/ApplicationStarter.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote.ops;
-
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public class ApplicationStarter implements RemoteOp<Void> {
- private String nodeId;
- private String appName;
- private boolean deployHar;
- private byte[] distributedState;
-
- public ApplicationStarter(String nodeId, String appName, boolean deployHar, byte[] distributedState) {
- this.nodeId = nodeId;
- this.appName = appName;
- this.deployHar = deployHar;
- this.distributedState = distributedState;
- }
-
- @Override
- public String getNodeId() {
- return nodeId;
- }
-
- @Override
- public Void execute(INodeController node) throws Exception {
- node.createApplication(appName, deployHar, distributedState);
- return null;
- }
-
- @Override
- public String toString() {
- return "Started application: " + appName;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/JobCompleteNotifier.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/JobCompleteNotifier.java
deleted file mode 100644
index 414cf12..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/remote/ops/JobCompleteNotifier.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.remote.ops;
-
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.cc.remote.RemoteOp;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
-
-public class JobCompleteNotifier implements RemoteOp<Void> {
- private String nodeId;
- private JobId jobId;
-
- public JobCompleteNotifier(String nodeId, JobId jobId) {
- this.nodeId = nodeId;
- this.jobId = jobId;
- }
-
- @Override
- public Void execute(INodeController node) throws Exception {
- node.cleanUpJob(jobId);
- return null;
- }
-
- @Override
- public String toString() {
- return jobId + " Cleaning Up";
- }
-
- @Override
- public String getNodeId() {
- return nodeId;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java
index 0e59c8df..f407ae3 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterGraphBuilder.java
@@ -22,13 +22,14 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.commons.lang3.tuple.Pair;
+
import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.IActivity;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
import edu.uci.ics.hyracks.api.job.JobActivityGraph;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.api.util.Pair;
import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
import edu.uci.ics.hyracks.control.cc.job.ActivityClusterId;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
@@ -57,7 +58,7 @@
int producerOutputIndex = spec.getProducerOutputIndex(conn);
ActivityId inTask = jag.getOperatorOutputMap().get(producerId).get(producerOutputIndex);
if (!eqSet.getActivities().contains(inTask)) {
- return new Pair<ActivityId, ActivityId>(t, inTask);
+ return Pair.<ActivityId, ActivityId> of(t, inTask);
}
}
}
@@ -70,7 +71,7 @@
int consumerInputIndex = spec.getConsumerInputIndex(conn);
ActivityId outTask = jag.getOperatorInputMap().get(consumerId).get(consumerInputIndex);
if (!eqSet.getActivities().contains(outTask)) {
- return new Pair<ActivityId, ActivityId>(t, outTask);
+ return Pair.<ActivityId, ActivityId> of(t, outTask);
}
}
}
@@ -102,7 +103,7 @@
changed = false;
Pair<ActivityId, ActivityId> pair = findMergePair(jag, spec, stages);
if (pair != null) {
- merge(stageMap, stages, pair.first, pair.second);
+ merge(stageMap, stages, pair.getLeft(), pair.getRight());
changed = true;
}
}
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java
index e5d9f64..e6e6c2f 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/ActivityClusterPlanner.java
@@ -25,6 +25,8 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import org.apache.commons.lang3.tuple.Pair;
+
import edu.uci.ics.hyracks.api.constraints.expressions.LValueConstraintExpression;
import edu.uci.ics.hyracks.api.constraints.expressions.PartitionCountExpression;
import edu.uci.ics.hyracks.api.dataflow.ActivityId;
@@ -38,7 +40,6 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.job.JobActivityGraph;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.api.util.Pair;
import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
import edu.uci.ics.hyracks.control.cc.job.ActivityClusterPlan;
import edu.uci.ics.hyracks.control.cc.job.ActivityPlan;
@@ -48,6 +49,8 @@
import edu.uci.ics.hyracks.control.cc.job.TaskClusterId;
public class ActivityClusterPlanner {
+ private static final boolean USE_CONNECTOR_POLICY_IN_TASK_CLUSTER_CONSTRUCTION = true;
+
private static final Logger LOGGER = Logger.getLogger(ActivityClusterPlanner.class.getName());
private final JobScheduler scheduler;
@@ -63,13 +66,28 @@
JobRun jobRun = scheduler.getJobRun();
Map<ActivityId, ActivityPartitionDetails> pcMap = computePartitionCounts(ac);
+ Map<ActivityId, ActivityPlan> activityPlanMap = buildActivityPlanMap(ac, jobRun, pcMap);
+
+ assignConnectorPolicy(ac, activityPlanMap);
+
+ TaskCluster[] taskClusters = computeTaskClusters(ac, jobRun, activityPlanMap);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Plan for " + ac);
+ LOGGER.info("Built " + taskClusters.length + " Task Clusters");
+ for (TaskCluster tc : taskClusters) {
+ LOGGER.info("Tasks: " + Arrays.toString(tc.getTasks()));
+ }
+ }
+
+ ac.setPlan(new ActivityClusterPlan(taskClusters, activityPlanMap));
+ }
+
+ private Map<ActivityId, ActivityPlan> buildActivityPlanMap(ActivityCluster ac, JobRun jobRun,
+ Map<ActivityId, ActivityPartitionDetails> pcMap) {
Map<ActivityId, ActivityPlan> activityPlanMap = new HashMap<ActivityId, ActivityPlan>();
- Set<ActivityId> activities = ac.getActivities();
-
- Map<TaskId, Set<TaskId>> taskClusterMap = new HashMap<TaskId, Set<TaskId>>();
-
Set<ActivityId> depAnIds = new HashSet<ActivityId>();
- for (ActivityId anId : activities) {
+ for (ActivityId anId : ac.getActivities()) {
depAnIds.clear();
getDependencyActivityIds(depAnIds, anId);
ActivityPartitionDetails apd = pcMap.get(anId);
@@ -93,17 +111,73 @@
tasks[i].getDependencies().add(dTaskId);
dTask.getDependents().add(tid);
}
- Set<TaskId> cluster = new HashSet<TaskId>();
- cluster.add(tid);
- taskClusterMap.put(tid, cluster);
}
activityPlan.setTasks(tasks);
activityPlanMap.put(anId, activityPlan);
}
+ return activityPlanMap;
+ }
- Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = assignConnectorPolicy(ac, activityPlanMap);
- scheduler.getJobRun().getConnectorPolicyMap().putAll(connectorPolicies);
+ private TaskCluster[] computeTaskClusters(ActivityCluster ac, JobRun jobRun,
+ Map<ActivityId, ActivityPlan> activityPlanMap) {
+ Set<ActivityId> activities = ac.getActivities();
+ Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity = computeTaskConnectivity(jobRun,
+ activityPlanMap, activities);
+ TaskCluster[] taskClusters = USE_CONNECTOR_POLICY_IN_TASK_CLUSTER_CONSTRUCTION ? buildConnectorPolicyAwareTaskClusters(
+ ac, activityPlanMap, taskConnectivity) : buildConnectorPolicyUnawareTaskClusters(ac, activityPlanMap);
+
+ for (TaskCluster tc : taskClusters) {
+ Set<TaskCluster> tcDependencyTaskClusters = tc.getDependencyTaskClusters();
+ for (Task ts : tc.getTasks()) {
+ TaskId tid = ts.getTaskId();
+ List<Pair<TaskId, ConnectorDescriptorId>> cInfoList = taskConnectivity.get(tid);
+ if (cInfoList != null) {
+ for (Pair<TaskId, ConnectorDescriptorId> p : cInfoList) {
+ Task targetTS = activityPlanMap.get(p.getLeft().getActivityId()).getTasks()[p.getLeft()
+ .getPartition()];
+ TaskCluster targetTC = targetTS.getTaskCluster();
+ if (targetTC != tc) {
+ ConnectorDescriptorId cdId = p.getRight();
+ PartitionId pid = new PartitionId(jobRun.getJobId(), cdId, tid.getPartition(), p.getLeft()
+ .getPartition());
+ tc.getProducedPartitions().add(pid);
+ targetTC.getRequiredPartitions().add(pid);
+ partitionProducingTaskClusterMap.put(pid, tc);
+ }
+ }
+ }
+
+ for (TaskId dTid : ts.getDependencies()) {
+ TaskCluster dTC = getTaskCluster(dTid);
+ dTC.getDependentTaskClusters().add(tc);
+ tcDependencyTaskClusters.add(dTC);
+ }
+ }
+ }
+ return taskClusters;
+ }
+
+ private TaskCluster[] buildConnectorPolicyUnawareTaskClusters(ActivityCluster ac,
+ Map<ActivityId, ActivityPlan> activityPlanMap) {
+ List<Task> taskStates = new ArrayList<Task>();
+ for (ActivityId anId : ac.getActivities()) {
+ ActivityPlan ap = activityPlanMap.get(anId);
+ Task[] tasks = ap.getTasks();
+ for (Task t : tasks) {
+ taskStates.add(t);
+ }
+ }
+ TaskCluster tc = new TaskCluster(new TaskClusterId(ac.getActivityClusterId(), 0), ac,
+ taskStates.toArray(new Task[taskStates.size()]));
+ for (Task t : tc.getTasks()) {
+ t.setTaskCluster(tc);
+ }
+ return new TaskCluster[] { tc };
+ }
+
+ private Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> computeTaskConnectivity(JobRun jobRun,
+ Map<ActivityId, ActivityPlan> activityPlanMap, Set<ActivityId> activities) {
Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity = new HashMap<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>>();
JobActivityGraph jag = jobRun.getJobActivityGraph();
BitSet targetBitmap = new BitSet();
@@ -125,52 +199,102 @@
cInfoList = new ArrayList<Pair<TaskId, ConnectorDescriptorId>>();
taskConnectivity.put(ac1TaskStates[i].getTaskId(), cInfoList);
}
- Set<TaskId> cluster = taskClusterMap.get(ac1TaskStates[i].getTaskId());
for (int j = targetBitmap.nextSetBit(0); j >= 0; j = targetBitmap.nextSetBit(j + 1)) {
- cInfoList.add(new Pair<TaskId, ConnectorDescriptorId>(ac2TaskStates[j].getTaskId(), cdId));
- IConnectorPolicy cPolicy = connectorPolicies.get(cdId);
- if (cPolicy.requiresProducerConsumerCoscheduling()) {
- cluster.add(ac2TaskStates[j].getTaskId());
- }
+ TaskId targetTID = ac2TaskStates[j].getTaskId();
+ cInfoList.add(Pair.<TaskId, ConnectorDescriptorId> of(targetTID, cdId));
}
}
}
}
}
+ return taskConnectivity;
+ }
- boolean done = false;
- while (!done) {
- done = true;
- Set<TaskId> set = new HashSet<TaskId>();
- Set<TaskId> oldSet = null;
- for (Map.Entry<TaskId, Set<TaskId>> e : taskClusterMap.entrySet()) {
- set.clear();
- oldSet = e.getValue();
- set.addAll(e.getValue());
- for (TaskId tid : e.getValue()) {
- set.addAll(taskClusterMap.get(tid));
- }
- for (TaskId tid : set) {
- Set<TaskId> targetSet = taskClusterMap.get(tid);
- if (!targetSet.equals(set)) {
- done = false;
- break;
- }
- }
- if (!done) {
- break;
- }
- }
- for (TaskId tid : oldSet) {
- taskClusterMap.put(tid, set);
+ private TaskCluster[] buildConnectorPolicyAwareTaskClusters(ActivityCluster ac,
+ Map<ActivityId, ActivityPlan> activityPlanMap,
+ Map<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> taskConnectivity) {
+ Map<TaskId, Set<TaskId>> taskClusterMap = new HashMap<TaskId, Set<TaskId>>();
+ for (ActivityId anId : ac.getActivities()) {
+ ActivityPlan ap = activityPlanMap.get(anId);
+ Task[] tasks = ap.getTasks();
+ for (Task t : tasks) {
+ Set<TaskId> cluster = new HashSet<TaskId>();
+ TaskId tid = t.getTaskId();
+ cluster.add(tid);
+ taskClusterMap.put(tid, cluster);
}
}
- Set<Set<TaskId>> clusters = new HashSet<Set<TaskId>>(taskClusterMap.values());
- Set<TaskCluster> tcSet = new HashSet<TaskCluster>();
+ JobRun jobRun = ac.getJobRun();
+ Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = jobRun.getConnectorPolicyMap();
+ for (Map.Entry<TaskId, List<Pair<TaskId, ConnectorDescriptorId>>> e : taskConnectivity.entrySet()) {
+ Set<TaskId> cluster = taskClusterMap.get(e.getKey());
+ for (Pair<TaskId, ConnectorDescriptorId> p : e.getValue()) {
+ IConnectorPolicy cPolicy = connectorPolicies.get(p.getRight());
+ if (cPolicy.requiresProducerConsumerCoscheduling()) {
+ cluster.add(p.getLeft());
+ }
+ }
+ }
+
+ /*
+ * taskClusterMap contains for every TID x, x -> { coscheduled consumer TIDs U x }
+ * We compute the transitive closure of this relation to find the largest set of
+ * tasks that need to be co-scheduled
+ */
int counter = 0;
- for (Set<TaskId> cluster : clusters) {
- Set<Task> taskStates = new HashSet<Task>();
+ TaskId[] ordinalList = new TaskId[taskClusterMap.size()];
+ Map<TaskId, Integer> ordinalMap = new HashMap<TaskId, Integer>();
+ for (TaskId tid : taskClusterMap.keySet()) {
+ ordinalList[counter] = tid;
+ ordinalMap.put(tid, counter);
+ ++counter;
+ }
+
+ int n = ordinalList.length;
+ BitSet[] paths = new BitSet[n];
+ for (Map.Entry<TaskId, Set<TaskId>> e : taskClusterMap.entrySet()) {
+ int i = ordinalMap.get(e.getKey());
+ BitSet bsi = paths[i];
+ if (bsi == null) {
+ bsi = new BitSet(n);
+ paths[i] = bsi;
+ }
+ for (TaskId ttid : e.getValue()) {
+ int j = ordinalMap.get(ttid);
+ paths[i].set(j);
+ BitSet bsj = paths[j];
+ if (bsj == null) {
+ bsj = new BitSet(n);
+ paths[j] = bsj;
+ }
+ bsj.set(i);
+ }
+ }
+ for (int k = 0; k < n; ++k) {
+ for (int i = paths[k].nextSetBit(0); i >= 0; i = paths[k].nextSetBit(i + 1)) {
+ for (int j = paths[i].nextClearBit(0); j < n && j >= 0; j = paths[i].nextClearBit(j + 1)) {
+ paths[i].set(j, paths[k].get(j));
+ paths[j].set(i, paths[i].get(j));
+ }
+ }
+ }
+ BitSet pending = new BitSet(n);
+ pending.set(0, n);
+ List<List<TaskId>> clusters = new ArrayList<List<TaskId>>();
+ for (int i = pending.nextSetBit(0); i >= 0; i = pending.nextSetBit(i)) {
+ List<TaskId> cluster = new ArrayList<TaskId>();
+ for (int j = paths[i].nextSetBit(0); j >= 0; j = paths[i].nextSetBit(j + 1)) {
+ cluster.add(ordinalList[j]);
+ pending.clear(j);
+ }
+ clusters.add(cluster);
+ }
+
+ List<TaskCluster> tcSet = new ArrayList<TaskCluster>();
+ counter = 0;
+ for (List<TaskId> cluster : clusters) {
+ List<Task> taskStates = new ArrayList<Task>();
for (TaskId tid : cluster) {
taskStates.add(activityPlanMap.get(tid.getActivityId()).getTasks()[tid.getPartition()]);
}
@@ -182,44 +306,7 @@
}
}
TaskCluster[] taskClusters = tcSet.toArray(new TaskCluster[tcSet.size()]);
-
- for (TaskCluster tc : taskClusters) {
- Set<TaskCluster> tcDependencyTaskClusters = tc.getDependencyTaskClusters();
- for (Task ts : tc.getTasks()) {
- TaskId tid = ts.getTaskId();
- List<Pair<TaskId, ConnectorDescriptorId>> cInfoList = taskConnectivity.get(tid);
- if (cInfoList != null) {
- for (Pair<TaskId, ConnectorDescriptorId> p : cInfoList) {
- Task targetTS = activityPlanMap.get(p.first.getActivityId()).getTasks()[p.first.getPartition()];
- TaskCluster targetTC = targetTS.getTaskCluster();
- if (targetTC != tc) {
- ConnectorDescriptorId cdId = p.second;
- PartitionId pid = new PartitionId(jobRun.getJobId(), cdId, tid.getPartition(),
- p.first.getPartition());
- tc.getProducedPartitions().add(pid);
- targetTC.getRequiredPartitions().add(pid);
- partitionProducingTaskClusterMap.put(pid, tc);
- }
- }
- }
-
- for (TaskId dTid : ts.getDependencies()) {
- TaskCluster dTC = getTaskCluster(dTid);
- dTC.getDependentTaskClusters().add(tc);
- tcDependencyTaskClusters.add(dTC);
- }
- }
- }
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Plan for " + ac);
- LOGGER.info("Built " + tcSet.size() + " Task Clusters");
- for (TaskCluster tc : tcSet) {
- LOGGER.info("Tasks: " + Arrays.toString(tc.getTasks()));
- }
- }
-
- ac.setPlan(new ActivityClusterPlan(taskClusters, activityPlanMap));
+ return taskClusters;
}
private TaskCluster getTaskCluster(TaskId tid) {
@@ -239,8 +326,7 @@
}
}
- private Map<ConnectorDescriptorId, IConnectorPolicy> assignConnectorPolicy(ActivityCluster ac,
- Map<ActivityId, ActivityPlan> taskMap) {
+ private void assignConnectorPolicy(ActivityCluster ac, Map<ActivityId, ActivityPlan> taskMap) {
JobActivityGraph jag = scheduler.getJobRun().getJobActivityGraph();
Map<ConnectorDescriptorId, IConnectorPolicy> cPolicyMap = new HashMap<ConnectorDescriptorId, IConnectorPolicy>();
Set<ActivityId> activities = ac.getActivities();
@@ -266,7 +352,7 @@
}
}
}
- return cPolicyMap;
+ ac.getJobRun().getConnectorPolicyMap().putAll(cPolicyMap);
}
private IConnectorPolicy assignConnectorPolicy(IConnectorDescriptor c, int nProducers, int nConsumers, int[] fanouts) {
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java
index 68d794c..e0eee00 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/scheduler/JobScheduler.java
@@ -14,7 +14,6 @@
*/
package edu.uci.ics.hyracks.control.cc.scheduler;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -23,11 +22,11 @@
import java.util.PriorityQueue;
import java.util.Random;
import java.util.Set;
-import java.util.concurrent.Executor;
import java.util.logging.Level;
import java.util.logging.Logger;
import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
import edu.uci.ics.hyracks.api.constraints.Constraint;
import edu.uci.ics.hyracks.api.constraints.IConstraintAcceptor;
import edu.uci.ics.hyracks.api.constraints.expressions.LValueConstraintExpression;
@@ -49,6 +48,7 @@
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
import edu.uci.ics.hyracks.control.cc.NodeControllerState;
import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
+import edu.uci.ics.hyracks.control.cc.job.ActivityPlan;
import edu.uci.ics.hyracks.control.cc.job.IConnectorDescriptorVisitor;
import edu.uci.ics.hyracks.control.cc.job.IOperatorDescriptorVisitor;
import edu.uci.ics.hyracks.control.cc.job.JobRun;
@@ -57,8 +57,8 @@
import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
import edu.uci.ics.hyracks.control.cc.job.TaskCluster;
import edu.uci.ics.hyracks.control.cc.job.TaskClusterAttempt;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.JobCleanupEvent;
import edu.uci.ics.hyracks.control.cc.partitions.PartitionMatchMaker;
+import edu.uci.ics.hyracks.control.cc.work.JobCleanupWork;
import edu.uci.ics.hyracks.control.common.job.PartitionState;
import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
@@ -184,12 +184,12 @@
private void startRunnableActivityClusters() throws HyracksException {
Set<TaskCluster> taskClusterRoots = new HashSet<TaskCluster>();
findRunnableTaskClusterRoots(taskClusterRoots, rootActivityClusters);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Runnable TC roots: " + taskClusterRoots + ", inProgressTaskClusters: "
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Runnable TC roots: " + taskClusterRoots + ", inProgressTaskClusters: "
+ inProgressTaskClusters);
}
if (taskClusterRoots.isEmpty() && inProgressTaskClusters.isEmpty()) {
- ccs.getJobQueue().schedule(new JobCleanupEvent(ccs, jobRun.getJobId(), JobStatus.TERMINATED, null));
+ ccs.getWorkQueue().schedule(new JobCleanupWork(ccs, jobRun.getJobId(), JobStatus.TERMINATED, null));
return;
}
startRunnableTaskClusters(taskClusterRoots);
@@ -213,19 +213,19 @@
queue.add(new RankedRunnableTaskCluster(priority, tc));
}
}
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Ranked TCs: " + queue);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Ranked TCs: " + queue);
}
Map<String, List<TaskAttemptDescriptor>> taskAttemptMap = new HashMap<String, List<TaskAttemptDescriptor>>();
for (RankedRunnableTaskCluster rrtc : queue) {
TaskCluster tc = rrtc.getTaskCluster();
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Found runnable TC: " + tc);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Found runnable TC: " + tc);
List<TaskClusterAttempt> attempts = tc.getAttempts();
- LOGGER.info("Attempts so far:" + attempts.size());
+ LOGGER.fine("Attempts so far:" + attempts.size());
for (TaskClusterAttempt tcAttempt : attempts) {
- LOGGER.info("Status: " + tcAttempt.getStatus());
+ LOGGER.fine("Status: " + tcAttempt.getStatus());
}
}
assignTaskLocations(tc, taskAttemptMap);
@@ -245,16 +245,16 @@
* Runnability(Non-schedulable TaskCluster) = {NOT_RUNNABLE, _}
*/
private Runnability assignRunnabilityRank(TaskCluster goal, Map<TaskCluster, Runnability> runnabilityMap) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Computing runnability: " + goal);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Computing runnability: " + goal);
}
if (runnabilityMap.containsKey(goal)) {
return runnabilityMap.get(goal);
}
TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(goal);
if (lastAttempt != null) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Last Attempt Status: " + lastAttempt.getStatus());
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Last Attempt Status: " + lastAttempt.getStatus());
}
if (lastAttempt.getStatus() == TaskClusterAttempt.TaskClusterStatus.COMPLETED) {
Runnability runnability = new Runnability(Runnability.Tag.COMPLETED, Integer.MIN_VALUE);
@@ -271,15 +271,15 @@
PartitionMatchMaker pmm = jobRun.getPartitionMatchMaker();
Runnability aggregateRunnability = new Runnability(Runnability.Tag.RUNNABLE, 0);
for (PartitionId pid : goal.getRequiredPartitions()) {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Inspecting required partition: " + pid);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Inspecting required partition: " + pid);
}
Runnability runnability;
ConnectorDescriptorId cdId = pid.getConnectorDescriptorId();
IConnectorPolicy cPolicy = connectorPolicyMap.get(cdId);
PartitionState maxState = pmm.getMaximumAvailableState(pid);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Policy: " + cPolicy + " maxState: " + maxState);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Policy: " + cPolicy + " maxState: " + maxState);
}
if (PartitionState.COMMITTED.equals(maxState)) {
runnability = new Runnability(Runnability.Tag.RUNNABLE, 0);
@@ -309,8 +309,12 @@
}
}
aggregateRunnability = Runnability.getWorstCase(aggregateRunnability, runnability);
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("aggregateRunnability: " + aggregateRunnability);
+ if (aggregateRunnability.getTag() == Runnability.Tag.NOT_RUNNABLE) {
+ // already not runnable -- cannot get better. bail.
+ break;
+ }
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("aggregateRunnability: " + aggregateRunnability);
}
}
runnabilityMap.put(goal, aggregateRunnability);
@@ -324,7 +328,7 @@
List<TaskClusterAttempt> tcAttempts = tc.getAttempts();
int attempts = tcAttempts.size();
TaskClusterAttempt tcAttempt = new TaskClusterAttempt(tc, attempts);
- TaskAttempt[] taskAttempts = new TaskAttempt[tasks.length];
+ Map<TaskId, TaskAttempt> taskAttempts = new HashMap<TaskId, TaskAttempt>();
Map<TaskId, LValueConstraintExpression> locationMap = new HashMap<TaskId, LValueConstraintExpression>();
for (int i = 0; i < tasks.length; ++i) {
Task ts = tasks[i];
@@ -334,90 +338,118 @@
taskAttempt.setStatus(TaskAttempt.TaskStatus.INITIALIZED, null);
locationMap.put(tid,
new PartitionLocationExpression(tid.getActivityId().getOperatorDescriptorId(), tid.getPartition()));
- taskAttempts[i] = taskAttempt;
+ taskAttempts.put(tid, taskAttempt);
}
tcAttempt.setTaskAttempts(taskAttempts);
solver.solve(locationMap.values());
for (int i = 0; i < tasks.length; ++i) {
Task ts = tasks[i];
TaskId tid = ts.getTaskId();
- TaskAttempt taskAttempt = taskAttempts[i];
- ActivityId aid = tid.getActivityId();
- Set<ActivityId> blockers = jag.getBlocked2BlockerMap().get(aid);
- String nodeId = null;
- if (blockers != null) {
- for (ActivityId blocker : blockers) {
- nodeId = findLocationOfBlocker(jobRun, jag, new TaskId(blocker, tid.getPartition()));
- if (nodeId != null) {
- break;
- }
- }
- }
- Set<String> liveNodes = ccs.getNodeMap().keySet();
- if (nodeId == null) {
- LValueConstraintExpression pLocationExpr = locationMap.get(tid);
- Object location = solver.getValue(pLocationExpr);
- if (location == null) {
- // pick any
- nodeId = liveNodes.toArray(new String[liveNodes.size()])[Math.abs(new Random().nextInt())
- % liveNodes.size()];
- } else if (location instanceof String) {
- nodeId = (String) location;
- } else if (location instanceof String[]) {
- for (String choice : (String[]) location) {
- if (liveNodes.contains(choice)) {
- nodeId = choice;
- break;
- }
- }
- if (nodeId == null) {
- throw new HyracksException("No satisfiable location found for "
- + taskAttempt.getTaskAttemptId());
- }
- } else {
- throw new HyracksException("Unknown type of value for " + pLocationExpr + ": " + location + "("
- + location.getClass() + ")");
- }
- }
- if (nodeId == null) {
- throw new HyracksException("No satisfiable location found for " + taskAttempt.getTaskAttemptId());
- }
- if (!liveNodes.contains(nodeId)) {
- throw new HyracksException("Node " + nodeId + " not live");
- }
+ TaskAttempt taskAttempt = taskAttempts.get(tid);
+ String nodeId = assignLocation(jag, locationMap, tid, taskAttempt);
taskAttempt.setNodeId(nodeId);
taskAttempt.setStatus(TaskAttempt.TaskStatus.RUNNING, null);
+ taskAttempt.setStartTime(System.currentTimeMillis());
List<TaskAttemptDescriptor> tads = taskAttemptMap.get(nodeId);
if (tads == null) {
tads = new ArrayList<TaskAttemptDescriptor>();
taskAttemptMap.put(nodeId, tads);
}
ActivityPartitionDetails apd = ts.getActivityPlan().getActivityPartitionDetails();
- tads.add(new TaskAttemptDescriptor(taskAttempt.getTaskAttemptId(), apd.getPartitionCount(), apd
- .getInputPartitionCounts(), apd.getOutputPartitionCounts()));
+ TaskAttemptDescriptor tad = new TaskAttemptDescriptor(taskAttempt.getTaskAttemptId(),
+ apd.getPartitionCount(), apd.getInputPartitionCounts(), apd.getOutputPartitionCounts());
+ tads.add(tad);
}
tcAttempt.initializePendingTaskCounter();
tcAttempts.add(tcAttempt);
+
+ /* TODO - Further improvement for reducing messages -- not yet complete.
+ for (Map.Entry<String, List<TaskAttemptDescriptor>> e : taskAttemptMap.entrySet()) {
+ List<TaskAttemptDescriptor> tads = e.getValue();
+ for (TaskAttemptDescriptor tad : tads) {
+ TaskId tid = tad.getTaskAttemptId().getTaskId();
+ ActivityId aid = tid.getActivityId();
+ List<IConnectorDescriptor> inConnectors = jag.getActivityInputConnectorDescriptors(aid);
+ int[] inPartitionCounts = tad.getInputPartitionCounts();
+ NetworkAddress[][] partitionLocations = new NetworkAddress[inPartitionCounts.length][];
+ for (int i = 0; i < inPartitionCounts.length; ++i) {
+ ConnectorDescriptorId cdId = inConnectors.get(i).getConnectorId();
+ ActivityId producerAid = jag.getProducerActivity(cdId);
+ partitionLocations[i] = new NetworkAddress[inPartitionCounts[i]];
+ for (int j = 0; j < inPartitionCounts[i]; ++j) {
+ TaskId producerTaskId = new TaskId(producerAid, j);
+ String nodeId = findTaskLocation(producerTaskId);
+ partitionLocations[i][j] = ccs.getNodeMap().get(nodeId).getDataPort();
+ }
+ }
+ tad.setInputPartitionLocations(partitionLocations);
+ }
+ }
+ */
+
tcAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.RUNNING);
+ tcAttempt.setStartTime(System.currentTimeMillis());
inProgressTaskClusters.add(tc);
}
- private static String findLocationOfBlocker(JobRun jobRun, JobActivityGraph jag, TaskId tid) {
- ActivityId blockerAID = tid.getActivityId();
- ActivityCluster blockerAC = jobRun.getActivityClusterMap().get(blockerAID);
- Task[] blockerTasks = blockerAC.getPlan().getActivityPlanMap().get(blockerAID).getTasks();
- List<TaskClusterAttempt> tcAttempts = blockerTasks[tid.getPartition()].getTaskCluster().getAttempts();
+ private String assignLocation(JobActivityGraph jag, Map<TaskId, LValueConstraintExpression> locationMap,
+ TaskId tid, TaskAttempt taskAttempt) throws HyracksException {
+ ActivityId aid = tid.getActivityId();
+ Set<ActivityId> blockers = jag.getBlocked2BlockerMap().get(aid);
+ String nodeId = null;
+ if (blockers != null) {
+ for (ActivityId blocker : blockers) {
+ nodeId = findTaskLocation(new TaskId(blocker, tid.getPartition()));
+ if (nodeId != null) {
+ break;
+ }
+ }
+ }
+ Set<String> liveNodes = ccs.getNodeMap().keySet();
+ if (nodeId == null) {
+ LValueConstraintExpression pLocationExpr = locationMap.get(tid);
+ Object location = solver.getValue(pLocationExpr);
+ if (location == null) {
+ // pick any
+ nodeId = liveNodes.toArray(new String[liveNodes.size()])[Math.abs(new Random().nextInt())
+ % liveNodes.size()];
+ } else if (location instanceof String) {
+ nodeId = (String) location;
+ } else if (location instanceof String[]) {
+ for (String choice : (String[]) location) {
+ if (liveNodes.contains(choice)) {
+ nodeId = choice;
+ break;
+ }
+ }
+ if (nodeId == null) {
+ throw new HyracksException("No satisfiable location found for " + taskAttempt.getTaskAttemptId());
+ }
+ } else {
+ throw new HyracksException("Unknown type of value for " + pLocationExpr + ": " + location + "("
+ + location.getClass() + ")");
+ }
+ }
+ if (nodeId == null) {
+ throw new HyracksException("No satisfiable location found for " + taskAttempt.getTaskAttemptId());
+ }
+ if (!liveNodes.contains(nodeId)) {
+ throw new HyracksException("Node " + nodeId + " not live");
+ }
+ return nodeId;
+ }
+
+ private String findTaskLocation(TaskId tid) {
+ ActivityId aid = tid.getActivityId();
+ ActivityCluster ac = jobRun.getActivityClusterMap().get(aid);
+ Task[] tasks = ac.getPlan().getActivityPlanMap().get(aid).getTasks();
+ List<TaskClusterAttempt> tcAttempts = tasks[tid.getPartition()].getTaskCluster().getAttempts();
if (tcAttempts == null || tcAttempts.isEmpty()) {
return null;
}
TaskClusterAttempt lastTCA = tcAttempts.get(tcAttempts.size() - 1);
- for (TaskAttempt ta : lastTCA.getTaskAttempts()) {
- TaskId blockerTID = ta.getTaskAttemptId().getTaskId();
- if (tid.equals(blockerTID)) {
- return ta.getNodeId();
- }
- }
- return null;
+ TaskAttempt ta = lastTCA.getTaskAttempts().get(tid);
+ return ta == null ? null : ta.getNodeId();
}
private static TaskClusterAttempt findLastTaskClusterAttempt(TaskCluster tc) {
@@ -429,53 +461,51 @@
}
private void startTasks(Map<String, List<TaskAttemptDescriptor>> taskAttemptMap) throws HyracksException {
- Executor executor = ccs.getExecutor();
final JobId jobId = jobRun.getJobId();
final JobActivityGraph jag = jobRun.getJobActivityGraph();
final String appName = jag.getApplicationName();
final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = jobRun.getConnectorPolicyMap();
- for (Map.Entry<String, List<TaskAttemptDescriptor>> e : taskAttemptMap.entrySet()) {
- String nodeId = e.getKey();
- final List<TaskAttemptDescriptor> taskDescriptors = e.getValue();
+ for (Map.Entry<String, List<TaskAttemptDescriptor>> entry : taskAttemptMap.entrySet()) {
+ String nodeId = entry.getKey();
+ final List<TaskAttemptDescriptor> taskDescriptors = entry.getValue();
final NodeControllerState node = ccs.getNodeMap().get(nodeId);
if (node != null) {
node.getActiveJobIds().add(jobRun.getJobId());
- jobRun.getParticipatingNodeIds().add(nodeId);
- executor.execute(new Runnable() {
- @Override
- public void run() {
- try {
- node.getNodeController().startTasks(appName, jobId, JavaSerializationUtils.serialize(jag),
- taskDescriptors, connectorPolicies, null);
- } catch (IOException e) {
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
+ boolean changed = jobRun.getParticipatingNodeIds().add(nodeId);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Starting: " + taskDescriptors + " at " + entry.getKey());
+ }
+ try {
+ byte[] jagBytes = changed ? JavaSerializationUtils.serialize(jag) : null;
+ node.getNodeController().startTasks(appName, jobId, jagBytes, taskDescriptors, connectorPolicies);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
}
}
private void abortJob(Exception exception) {
- for (TaskCluster tc : inProgressTaskClusters) {
+ Set<TaskCluster> inProgressTaskClustersCopy = new HashSet<TaskCluster>(inProgressTaskClusters);
+ for (TaskCluster tc : inProgressTaskClustersCopy) {
abortTaskCluster(findLastTaskClusterAttempt(tc));
}
- jobRun.setStatus(JobStatus.FAILURE, exception);
+ assert inProgressTaskClusters.isEmpty();
+ ccs.getWorkQueue().schedule(new JobCleanupWork(ccs, jobRun.getJobId(), JobStatus.FAILURE, exception));
}
private void abortTaskCluster(TaskClusterAttempt tcAttempt) {
- LOGGER.info("Aborting task cluster: " + tcAttempt.getAttempt());
+ LOGGER.fine("Aborting task cluster: " + tcAttempt.getAttempt());
Set<TaskAttemptId> abortTaskIds = new HashSet<TaskAttemptId>();
Map<String, List<TaskAttemptId>> abortTaskAttemptMap = new HashMap<String, List<TaskAttemptId>>();
- for (TaskAttempt ta : tcAttempt.getTaskAttempts()) {
+ for (TaskAttempt ta : tcAttempt.getTaskAttempts().values()) {
TaskAttemptId taId = ta.getTaskAttemptId();
TaskAttempt.TaskStatus status = ta.getStatus();
abortTaskIds.add(taId);
- LOGGER.info("Checking " + taId + ": " + ta.getStatus());
+ LOGGER.fine("Checking " + taId + ": " + ta.getStatus());
if (status == TaskAttempt.TaskStatus.RUNNING || status == TaskAttempt.TaskStatus.COMPLETED) {
ta.setStatus(TaskAttempt.TaskStatus.ABORTED, null);
+ ta.setEndTime(System.currentTimeMillis());
List<TaskAttemptId> abortTaskAttempts = abortTaskAttemptMap.get(ta.getNodeId());
if (abortTaskAttempts == null) {
abortTaskAttempts = new ArrayList<TaskAttemptId>();
@@ -485,22 +515,19 @@
}
}
final JobId jobId = jobRun.getJobId();
- LOGGER.info("Abort map for job: " + jobId + ": " + abortTaskAttemptMap);
- for (Map.Entry<String, List<TaskAttemptId>> e : abortTaskAttemptMap.entrySet()) {
- final NodeControllerState node = ccs.getNodeMap().get(e.getKey());
- final List<TaskAttemptId> abortTaskAttempts = e.getValue();
+ LOGGER.fine("Abort map for job: " + jobId + ": " + abortTaskAttemptMap);
+ for (Map.Entry<String, List<TaskAttemptId>> entry : abortTaskAttemptMap.entrySet()) {
+ final NodeControllerState node = ccs.getNodeMap().get(entry.getKey());
+ final List<TaskAttemptId> abortTaskAttempts = entry.getValue();
if (node != null) {
- LOGGER.info("Aborting: " + abortTaskAttempts + " at " + e.getKey());
- ccs.getExecutor().execute(new Runnable() {
- @Override
- public void run() {
- try {
- node.getNodeController().abortTasks(jobId, abortTaskAttempts);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Aborting: " + abortTaskAttempts + " at " + entry.getKey());
+ }
+ try {
+ node.getNodeController().abortTasks(jobId, abortTaskAttempts);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
}
inProgressTaskClusters.remove(tcAttempt.getTaskCluster());
@@ -523,6 +550,7 @@
TaskClusterAttempt tca = findLastTaskClusterAttempt(tc);
if (tca != null) {
abortTaskCluster(tca);
+ tca.setEndTime(System.currentTimeMillis());
tca.setStatus(TaskClusterAttempt.TaskClusterStatus.ABORTED);
}
}
@@ -570,14 +598,16 @@
public void notifyTaskComplete(TaskAttempt ta) throws HyracksException {
TaskAttemptId taId = ta.getTaskAttemptId();
- TaskCluster tc = ta.getTaskState().getTaskCluster();
+ TaskCluster tc = ta.getTask().getTaskCluster();
TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(tc);
if (lastAttempt != null && taId.getAttempt() == lastAttempt.getAttempt()) {
TaskAttempt.TaskStatus taStatus = ta.getStatus();
if (taStatus == TaskAttempt.TaskStatus.RUNNING) {
ta.setStatus(TaskAttempt.TaskStatus.COMPLETED, null);
+ ta.setEndTime(System.currentTimeMillis());
if (lastAttempt.decrementPendingTasksCounter() == 0) {
lastAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.COMPLETED);
+ lastAttempt.setEndTime(System.currentTimeMillis());
inProgressTaskClusters.remove(tc);
startRunnableActivityClusters();
}
@@ -596,23 +626,24 @@
* - Failed Task Attempt
* @param ac
* - Activity Cluster that owns this Task
- * @param exception
+ * @param details
* - Cause of the failure
*/
- public void notifyTaskFailure(TaskAttempt ta, ActivityCluster ac, Exception exception) {
+ public void notifyTaskFailure(TaskAttempt ta, ActivityCluster ac, String details) {
try {
- LOGGER.info("Received failure notification for TaskAttempt " + ta.getTaskAttemptId());
+ LOGGER.fine("Received failure notification for TaskAttempt " + ta.getTaskAttemptId());
TaskAttemptId taId = ta.getTaskAttemptId();
- TaskCluster tc = ta.getTaskState().getTaskCluster();
+ TaskCluster tc = ta.getTask().getTaskCluster();
TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(tc);
if (lastAttempt != null && taId.getAttempt() == lastAttempt.getAttempt()) {
- LOGGER.info("Marking TaskAttempt " + ta.getTaskAttemptId() + " as failed");
- ta.setStatus(TaskAttempt.TaskStatus.FAILED, exception);
+ LOGGER.fine("Marking TaskAttempt " + ta.getTaskAttemptId() + " as failed");
+ ta.setStatus(TaskAttempt.TaskStatus.FAILED, details);
abortTaskCluster(lastAttempt);
lastAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.FAILED);
+ lastAttempt.setEndTime(System.currentTimeMillis());
abortDoomedTaskClusters();
- if (lastAttempt.getAttempt() >= ac.getMaxTaskClusterAttempts()) {
- abortJob(null);
+ if (lastAttempt.getAttempt() >= ac.getMaxTaskClusterReattempts()) {
+ abortJob(new HyracksException(details));
return;
}
startRunnableActivityClusters();
@@ -643,11 +674,11 @@
&& (lastTaskClusterAttempt.getStatus() == TaskClusterAttempt.TaskClusterStatus.COMPLETED || lastTaskClusterAttempt
.getStatus() == TaskClusterAttempt.TaskClusterStatus.RUNNING)) {
boolean abort = false;
- for (TaskAttempt ta : lastTaskClusterAttempt.getTaskAttempts()) {
+ for (TaskAttempt ta : lastTaskClusterAttempt.getTaskAttempts().values()) {
assert (ta.getStatus() == TaskAttempt.TaskStatus.COMPLETED || ta.getStatus() == TaskAttempt.TaskStatus.RUNNING);
if (deadNodes.contains(ta.getNodeId())) {
- ta.setStatus(TaskAttempt.TaskStatus.FAILED,
- new HyracksException("Node " + ta.getNodeId() + " failed"));
+ ta.setStatus(TaskAttempt.TaskStatus.FAILED, "Node " + ta.getNodeId() + " failed");
+ ta.setEndTime(System.currentTimeMillis());
abort = true;
}
}
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/AdminConsoleHandler.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/AdminConsoleHandler.java
deleted file mode 100644
index 8021d1a..0000000
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/AdminConsoleHandler.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.cc.web;
-
-import java.io.IOException;
-import java.net.URL;
-
-import javax.servlet.ServletException;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-
-import org.eclipse.jetty.server.Request;
-import org.eclipse.jetty.server.handler.AbstractHandler;
-
-import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-
-public class AdminConsoleHandler extends AbstractHandler {
- public AdminConsoleHandler(ClusterControllerService ccs) {
- }
-
- @Override
- public void handle(String target, Request baseRequest, HttpServletRequest request, HttpServletResponse response)
- throws IOException, ServletException {
- String basedir = System.getProperty("basedir");
- if (basedir == null) {
- response.getWriter().println("No Administration Console found.");
- }
-
- URL url = new URL(request.getRequestURL().toString());
- String ccLoc = url.getProtocol() + "://" + url.getHost() + ":" + url.getPort();
- response.sendRedirect("/console?cclocation=" + ccLoc);
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/ApplicationInstallationHandler.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/ApplicationInstallationHandler.java
index e047212..73f3a1a 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/ApplicationInstallationHandler.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/ApplicationInstallationHandler.java
@@ -28,8 +28,8 @@
import org.eclipse.jetty.server.handler.AbstractHandler;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.jobqueue.SynchronizableEvent;
import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
public class ApplicationInstallationHandler extends AbstractHandler {
private ClusterControllerService ccs;
@@ -54,7 +54,7 @@
}
final String appName = parts[0];
if (HttpMethods.PUT.equals(request.getMethod())) {
- class OutputStreamGetter extends SynchronizableEvent {
+ class OutputStreamGetter extends SynchronizableWork {
private OutputStream os;
@Override
@@ -68,7 +68,7 @@
}
OutputStreamGetter r = new OutputStreamGetter();
try {
- ccs.getJobQueue().scheduleAndSync(r);
+ ccs.getWorkQueue().scheduleAndSync(r);
} catch (Exception e) {
throw new IOException(e);
}
@@ -78,7 +78,7 @@
r.os.close();
}
} else if (HttpMethods.GET.equals(request.getMethod())) {
- class InputStreamGetter extends SynchronizableEvent {
+ class InputStreamGetter extends SynchronizableWork {
private InputStream is;
@Override
@@ -92,7 +92,7 @@
}
InputStreamGetter r = new InputStreamGetter();
try {
- ccs.getJobQueue().scheduleAndSync(r);
+ ccs.getWorkQueue().scheduleAndSync(r);
} catch (Exception e) {
throw new IOException(e);
}
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/JobsRESTAPIFunction.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/JobsRESTAPIFunction.java
index bd46276..4135b18 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/JobsRESTAPIFunction.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/JobsRESTAPIFunction.java
@@ -18,11 +18,11 @@
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.GetJobActivityGraphJSONEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.GetJobRunJSONEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.GetJobSpecificationJSONEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.GetJobSummariesJSONEvent;
import edu.uci.ics.hyracks.control.cc.web.util.IJSONOutputFunction;
+import edu.uci.ics.hyracks.control.cc.work.GetJobActivityGraphJSONWork;
+import edu.uci.ics.hyracks.control.cc.work.GetJobRunJSONWork;
+import edu.uci.ics.hyracks.control.cc.work.GetJobSpecificationJSONWork;
+import edu.uci.ics.hyracks.control.cc.work.GetJobSummariesJSONWork;
public class JobsRESTAPIFunction implements IJSONOutputFunction {
private ClusterControllerService ccs;
@@ -40,8 +40,8 @@
break;
}
case 0: {
- GetJobSummariesJSONEvent gjse = new GetJobSummariesJSONEvent(ccs);
- ccs.getJobQueue().scheduleAndSync(gjse);
+ GetJobSummariesJSONWork gjse = new GetJobSummariesJSONWork(ccs);
+ ccs.getWorkQueue().scheduleAndSync(gjse);
result.put("result", gjse.getSummaries());
break;
}
@@ -50,16 +50,16 @@
JobId jobId = JobId.parse(arguments[0]);
if ("job-specification".equalsIgnoreCase(arguments[1])) {
- GetJobSpecificationJSONEvent gjse = new GetJobSpecificationJSONEvent(ccs, jobId);
- ccs.getJobQueue().scheduleAndSync(gjse);
+ GetJobSpecificationJSONWork gjse = new GetJobSpecificationJSONWork(ccs, jobId);
+ ccs.getWorkQueue().scheduleAndSync(gjse);
result.put("result", gjse.getJSON());
} else if ("job-activity-graph".equalsIgnoreCase(arguments[1])) {
- GetJobActivityGraphJSONEvent gjage = new GetJobActivityGraphJSONEvent(ccs, jobId);
- ccs.getJobQueue().scheduleAndSync(gjage);
+ GetJobActivityGraphJSONWork gjage = new GetJobActivityGraphJSONWork(ccs, jobId);
+ ccs.getWorkQueue().scheduleAndSync(gjage);
result.put("result", gjage.getJSON());
} else if ("job-run".equalsIgnoreCase(arguments[1])) {
- GetJobRunJSONEvent gjre = new GetJobRunJSONEvent(ccs, jobId);
- ccs.getJobQueue().scheduleAndSync(gjre);
+ GetJobRunJSONWork gjre = new GetJobRunJSONWork(ccs, jobId);
+ ccs.getWorkQueue().scheduleAndSync(gjre);
result.put("result", gjre.getJSON());
}
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/NodesRESTAPIFunction.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/NodesRESTAPIFunction.java
index 9059de7..b8742c8 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/NodesRESTAPIFunction.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/NodesRESTAPIFunction.java
@@ -17,9 +17,9 @@
import org.json.JSONObject;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.GetNodeDetailsJSONEvent;
-import edu.uci.ics.hyracks.control.cc.job.manager.events.GetNodeSummariesJSONEvent;
import edu.uci.ics.hyracks.control.cc.web.util.IJSONOutputFunction;
+import edu.uci.ics.hyracks.control.cc.work.GetNodeDetailsJSONWork;
+import edu.uci.ics.hyracks.control.cc.work.GetNodeSummariesJSONWork;
public class NodesRESTAPIFunction implements IJSONOutputFunction {
private ClusterControllerService ccs;
@@ -34,13 +34,13 @@
switch (arguments.length) {
case 1: {
if ("".equals(arguments[0])) {
- GetNodeSummariesJSONEvent gnse = new GetNodeSummariesJSONEvent(ccs);
- ccs.getJobQueue().scheduleAndSync(gnse);
+ GetNodeSummariesJSONWork gnse = new GetNodeSummariesJSONWork(ccs);
+ ccs.getWorkQueue().scheduleAndSync(gnse);
result.put("result", gnse.getSummaries());
} else {
String nodeId = arguments[0];
- GetNodeDetailsJSONEvent gnde = new GetNodeDetailsJSONEvent(ccs, nodeId);
- ccs.getJobQueue().scheduleAndSync(gnde);
+ GetNodeDetailsJSONWork gnde = new GetNodeDetailsJSONWork(ccs, nodeId);
+ ccs.getWorkQueue().scheduleAndSync(gnde);
result.put("result", gnde.getDetail());
}
}
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/WebServer.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/WebServer.java
index fb5ad21..c32cb97 100644
--- a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/WebServer.java
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/WebServer.java
@@ -14,9 +14,15 @@
*/
package edu.uci.ics.hyracks.control.cc.web;
-import java.io.File;
+import java.util.EnumSet;
import java.util.logging.Logger;
+import javax.servlet.DispatcherType;
+
+import org.apache.wicket.Application;
+import org.apache.wicket.RuntimeConfigurationType;
+import org.apache.wicket.protocol.http.ContextParamWebApplicationFactory;
+import org.apache.wicket.protocol.http.WicketFilter;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.Handler;
import org.eclipse.jetty.server.Server;
@@ -24,9 +30,12 @@
import org.eclipse.jetty.server.handler.ContextHandlerCollection;
import org.eclipse.jetty.server.handler.HandlerCollection;
import org.eclipse.jetty.server.nio.SelectChannelConnector;
-import org.eclipse.jetty.webapp.WebAppContext;
+import org.eclipse.jetty.servlet.DefaultServlet;
+import org.eclipse.jetty.servlet.FilterHolder;
+import org.eclipse.jetty.servlet.ServletContextHandler;
import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.adminconsole.HyracksAdminConsoleApplication;
import edu.uci.ics.hyracks.control.cc.web.util.JSONOutputRequestHandler;
import edu.uci.ics.hyracks.control.cc.web.util.RoutingHandler;
@@ -59,25 +68,36 @@
handler.setHandler(rh);
addHandler(handler);
- handler = new ContextHandler("/adminconsole");
- handler.setHandler(new AdminConsoleHandler(ccs));
- addHandler(handler);
+ addHandler(createAdminConsoleHandler());
+ addHandler(createStaticResourcesHandler());
handler = new ContextHandler("/applications");
handler.setHandler(new ApplicationInstallationHandler(ccs));
addHandler(handler);
+ }
- String basedir = System.getProperty("basedir");
- if (basedir != null) {
- File warFile = new File(basedir, "console/hyracks-admin-console.war");
- LOGGER.info("Looking for admin console binary in: " + warFile.getAbsolutePath());
- if (warFile.exists()) {
- WebAppContext waCtx = new WebAppContext();
- waCtx.setContextPath("/console");
- waCtx.setWar(warFile.getAbsolutePath());
- addHandler(waCtx);
- }
- }
+ private Handler createAdminConsoleHandler() {
+ FilterHolder filter = new FilterHolder(WicketFilter.class);
+ filter.setInitParameter(ContextParamWebApplicationFactory.APP_CLASS_PARAM,
+ HyracksAdminConsoleApplication.class.getName());
+ filter.setInitParameter(WicketFilter.FILTER_MAPPING_PARAM, "/*");
+ filter.setInitParameter(Application.CONFIGURATION, RuntimeConfigurationType.DEPLOYMENT.toString());
+
+ ServletContextHandler handler = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ handler.setContextPath("/adminconsole");
+ handler.setAttribute(ClusterControllerService.class.getName(), ccs);
+ handler.addFilter(filter, "/*", EnumSet.of(DispatcherType.REQUEST, DispatcherType.ERROR));
+ handler.addServlet(DefaultServlet.class, "/");
+ return handler;
+ }
+
+ private Handler createStaticResourcesHandler() {
+ String staticDir = WebServer.class.getClassLoader().getResource("static").toExternalForm();
+ ServletContextHandler handler = new ServletContextHandler(ServletContextHandler.SESSIONS);
+ handler.setContextPath("/static");
+ handler.setResourceBase(staticDir);
+ handler.addServlet(DefaultServlet.class, "/");
+ return handler;
}
public void setPort(int port) {
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/util/JSONUtils.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/util/JSONUtils.java
new file mode 100644
index 0000000..137b398
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/web/util/JSONUtils.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.web.util;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+public class JSONUtils {
+ public static List<JSONObject> toList(JSONArray array) throws JSONException {
+ List<JSONObject> list = new ArrayList<JSONObject>();
+ for (int i = 0; i < array.length(); ++i) {
+ list.add((JSONObject) array.get(i));
+ }
+ return list;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/AbstractTaskLifecycleWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/AbstractTaskLifecycleWork.java
new file mode 100644
index 0000000..01f14f3
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/AbstractTaskLifecycleWork.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
+import edu.uci.ics.hyracks.control.cc.job.ActivityPlan;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.cc.job.Task;
+import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
+import edu.uci.ics.hyracks.control.cc.job.TaskCluster;
+import edu.uci.ics.hyracks.control.cc.job.TaskClusterAttempt;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+
+public abstract class AbstractTaskLifecycleWork extends AbstractWork {
+ protected final ClusterControllerService ccs;
+ protected final JobId jobId;
+ protected final TaskAttemptId taId;
+ protected final String nodeId;
+
+ public AbstractTaskLifecycleWork(ClusterControllerService ccs, JobId jobId, TaskAttemptId taId, String nodeId) {
+ this.ccs = ccs;
+ this.jobId = jobId;
+ this.taId = taId;
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ public final void run() {
+ JobRun run = ccs.getActiveRunMap().get(jobId);
+ if (run != null) {
+ TaskId tid = taId.getTaskId();
+ Map<ActivityId, ActivityCluster> activityClusterMap = run.getActivityClusterMap();
+ ActivityCluster ac = activityClusterMap.get(tid.getActivityId());
+ if (ac != null) {
+ Map<ActivityId, ActivityPlan> taskStateMap = ac.getPlan().getActivityPlanMap();
+ Task[] taskStates = taskStateMap.get(tid.getActivityId()).getTasks();
+ if (taskStates != null && taskStates.length > tid.getPartition()) {
+ Task ts = taskStates[tid.getPartition()];
+ TaskCluster tc = ts.getTaskCluster();
+ List<TaskClusterAttempt> taskClusterAttempts = tc.getAttempts();
+ if (taskClusterAttempts != null && taskClusterAttempts.size() > taId.getAttempt()) {
+ TaskClusterAttempt tca = taskClusterAttempts.get(taId.getAttempt());
+ TaskAttempt ta = tca.getTaskAttempts().get(tid);
+ if (ta != null) {
+ performEvent(ta);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ protected abstract void performEvent(TaskAttempt ta);
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationCreateWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationCreateWork.java
new file mode 100644
index 0000000..15d6d1f
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationCreateWork.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.io.IOException;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+
+public class ApplicationCreateWork extends AbstractWork {
+ private final ClusterControllerService ccs;
+ private final String appName;
+ private IResultCallback<Object> callback;
+
+ public ApplicationCreateWork(ClusterControllerService ccs, String appName, IResultCallback<Object> callback) {
+ this.ccs = ccs;
+ this.appName = appName;
+ this.callback = callback;
+ }
+
+ @Override
+ public void run() {
+ try {
+ Map<String, CCApplicationContext> applications = ccs.getApplicationMap();
+ if (applications.containsKey(appName)) {
+ callback.setException(new HyracksException("Duplicate application with name: " + appName
+ + " being created."));
+ return;
+ }
+ CCApplicationContext appCtx;
+ try {
+ appCtx = new CCApplicationContext(ccs.getServerContext(), ccs.getCCContext(), appName);
+ } catch (IOException e) {
+ callback.setException(e);
+ return;
+ }
+ appCtx.setStatus(ApplicationStatus.CREATED);
+ applications.put(appName, appCtx);
+ callback.setValue(null);
+ } catch (Exception e) {
+ callback.setException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationDestroyWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationDestroyWork.java
new file mode 100644
index 0000000..fc86c4c
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationDestroyWork.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.base.INodeController;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+
+public class ApplicationDestroyWork extends AbstractWork {
+ private final ClusterControllerService ccs;
+ private final String appName;
+ private IResultCallback<Object> callback;
+
+ public ApplicationDestroyWork(ClusterControllerService ccs, String appName, IResultCallback<Object> callback) {
+ this.ccs = ccs;
+ this.appName = appName;
+ this.callback = callback;
+ }
+
+ @Override
+ public void run() {
+ try {
+ final CCApplicationContext appCtx = ccs.getApplicationMap().get(appName);
+ if (appCtx == null) {
+ callback.setException(new HyracksException("No application with name: " + appName));
+ return;
+ }
+ if (appCtx.getStatus() == ApplicationStatus.IN_DEINITIALIZATION
+ || appCtx.getStatus() == ApplicationStatus.DEINITIALIZED) {
+ return;
+ }
+ Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
+ appCtx.getDeinitializationPendingNodeIds().addAll(nodeMap.keySet());
+ appCtx.setStatus(ApplicationStatus.IN_DEINITIALIZATION);
+ appCtx.setDeinitializationCallback(callback);
+ for (String nodeId : ccs.getNodeMap().keySet()) {
+ NodeControllerState nodeState = nodeMap.get(nodeId);
+ final INodeController node = nodeState.getNodeController();
+ node.destroyApplication(appName);
+ }
+ } catch (Exception e) {
+ callback.setException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStartWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStartWork.java
new file mode 100644
index 0000000..e4ad56c
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStartWork.java
@@ -0,0 +1,80 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.util.JavaSerializationUtils;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.base.INodeController;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+
+public class ApplicationStartWork extends AbstractWork {
+ private final ClusterControllerService ccs;
+ private final String appName;
+ private final IResultCallback<Object> callback;
+
+ public ApplicationStartWork(ClusterControllerService ccs, String appName, IResultCallback<Object> callback) {
+ this.ccs = ccs;
+ this.appName = appName;
+ this.callback = callback;
+ }
+
+ @Override
+ public void run() {
+ try {
+ final CCApplicationContext appCtx = ccs.getApplicationMap().get(appName);
+ if (appCtx == null) {
+ callback.setException(new HyracksException("No application with name: " + appName));
+ return;
+ }
+ if (appCtx.getStatus() != ApplicationStatus.CREATED) {
+ callback.setException(new HyracksException("Application in incorrect state for starting: "
+ + appCtx.getStatus()));
+ }
+ final Map<String, NodeControllerState> nodeMapCopy = new HashMap<String, NodeControllerState>(
+ ccs.getNodeMap());
+ appCtx.getInitializationPendingNodeIds().addAll(nodeMapCopy.keySet());
+ appCtx.setStatus(ApplicationStatus.IN_INITIALIZATION);
+ appCtx.setInitializationCallback(callback);
+ ccs.getExecutor().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ appCtx.initializeClassPath();
+ appCtx.initialize();
+ final byte[] distributedState = JavaSerializationUtils.serialize(appCtx.getDistributedState());
+ final boolean deployHar = appCtx.containsHar();
+ for (final String nodeId : nodeMapCopy.keySet()) {
+ NodeControllerState nodeState = nodeMapCopy.get(nodeId);
+ final INodeController node = nodeState.getNodeController();
+ node.createApplication(appName, deployHar, distributedState);
+ }
+ } catch (Exception e) {
+ callback.setException(e);
+ }
+ }
+ });
+ } catch (Exception e) {
+ callback.setException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStateChangeWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStateChangeWork.java
new file mode 100644
index 0000000..f6271fe
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ApplicationStateChangeWork.java
@@ -0,0 +1,85 @@
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Set;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.ipc.CCNCFunctions;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+
+public class ApplicationStateChangeWork extends AbstractWork {
+ private static final Logger LOGGER = Logger.getLogger(ApplicationStateChangeWork.class.getName());
+
+ private final ClusterControllerService ccs;
+ private final CCNCFunctions.ApplicationStateChangeResponseFunction ascrf;
+
+ public ApplicationStateChangeWork(ClusterControllerService ccs,
+ CCNCFunctions.ApplicationStateChangeResponseFunction ascrf) {
+ this.ccs = ccs;
+ this.ascrf = ascrf;
+ }
+
+ @Override
+ public void run() {
+ final CCApplicationContext appCtx = ccs.getApplicationMap().get(ascrf.getApplicationName());
+ if (appCtx == null) {
+ LOGGER.warning("Got ApplicationStateChangeResponse for application " + ascrf.getApplicationName()
+ + " that does not exist");
+ return;
+ }
+ switch (ascrf.getStatus()) {
+ case INITIALIZED: {
+ Set<String> pendingNodeIds = appCtx.getInitializationPendingNodeIds();
+ boolean changed = pendingNodeIds.remove(ascrf.getNodeId());
+ if (!changed) {
+ LOGGER.warning("Got ApplicationStateChangeResponse for application " + ascrf.getApplicationName()
+ + " from unexpected node " + ascrf.getNodeId() + " to state " + ascrf.getStatus());
+ return;
+ }
+ if (pendingNodeIds.isEmpty()) {
+ appCtx.setStatus(ApplicationStatus.INITIALIZED);
+ IResultCallback<Object> callback = appCtx.getInitializationCallback();
+ appCtx.setInitializationCallback(null);
+ callback.setValue(null);
+ }
+ return;
+ }
+
+ case DEINITIALIZED: {
+ Set<String> pendingNodeIds = appCtx.getDeinitializationPendingNodeIds();
+ boolean changed = pendingNodeIds.remove(ascrf.getNodeId());
+ if (!changed) {
+ LOGGER.warning("Got ApplicationStateChangeResponse for application " + ascrf.getApplicationName()
+ + " from unexpected node " + ascrf.getNodeId() + " to state " + ascrf.getStatus());
+ return;
+ }
+ if (pendingNodeIds.isEmpty()) {
+ appCtx.setStatus(ApplicationStatus.DEINITIALIZED);
+ ccs.getExecutor().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ appCtx.deinitialize();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ ccs.getWorkQueue().schedule(new AbstractWork() {
+ @Override
+ public void run() {
+ ccs.getApplicationMap().remove(ascrf.getApplicationName());
+ IResultCallback<Object> callback = appCtx.getDeinitializationCallback();
+ appCtx.setDeinitializationCallback(null);
+ callback.setValue(null);
+ }
+ });
+ }
+ });
+ }
+ return;
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetIpAddressNodeNameMapWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetIpAddressNodeNameMapWork.java
new file mode 100644
index 0000000..1f29903
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetIpAddressNodeNameMapWork.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Map;
+import java.util.Set;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class GetIpAddressNodeNameMapWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private Map<String, Set<String>> map;
+
+ public GetIpAddressNodeNameMapWork(ClusterControllerService ccs, Map<String, Set<String>> map) {
+ this.ccs = ccs;
+ this.map = map;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ map.putAll(ccs.getIpAddressNodeNameMap());
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobActivityGraphJSONWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobActivityGraphJSONWork.java
new file mode 100644
index 0000000..677bece
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobActivityGraphJSONWork.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class GetJobActivityGraphJSONWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final JobId jobId;
+ private JSONObject json;
+
+ public GetJobActivityGraphJSONWork(ClusterControllerService ccs, JobId jobId) {
+ this.ccs = ccs;
+ this.jobId = jobId;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ JobRun run = ccs.getActiveRunMap().get(jobId);
+ if (run == null) {
+ run = ccs.getRunMapArchive().get(jobId);
+ if (run == null) {
+ json = new JSONObject();
+ return;
+ }
+ }
+ json = run.getJobActivityGraph().toJSON();
+ }
+
+ public JSONObject getJSON() {
+ return json;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobRunJSONWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobRunJSONWork.java
new file mode 100644
index 0000000..07c8f87
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobRunJSONWork.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class GetJobRunJSONWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final JobId jobId;
+ private JSONObject json;
+
+ public GetJobRunJSONWork(ClusterControllerService ccs, JobId jobId) {
+ this.ccs = ccs;
+ this.jobId = jobId;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ JobRun run = ccs.getActiveRunMap().get(jobId);
+ if (run == null) {
+ run = ccs.getRunMapArchive().get(jobId);
+ if (run == null) {
+ json = new JSONObject();
+ return;
+ }
+ }
+ json = run.toJSON();
+ }
+
+ public JSONObject getJSON() {
+ return json;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSpecificationJSONWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSpecificationJSONWork.java
new file mode 100644
index 0000000..05b7cae
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSpecificationJSONWork.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class GetJobSpecificationJSONWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final JobId jobId;
+ private JSONObject json;
+
+ public GetJobSpecificationJSONWork(ClusterControllerService ccs, JobId jobId) {
+ this.ccs = ccs;
+ this.jobId = jobId;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ JobRun run = ccs.getActiveRunMap().get(jobId);
+ if (run == null) {
+ run = ccs.getRunMapArchive().get(jobId);
+ if (run == null) {
+ json = new JSONObject();
+ return;
+ }
+ }
+ json = run.getJobActivityGraph().getJobSpecification().toJSON();
+ }
+
+ public JSONObject getJSON() {
+ return json;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusWork.java
new file mode 100644
index 0000000..c70e24f
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobStatusWork.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class GetJobStatusWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final JobId jobId;
+ private final IResultCallback<JobStatus> callback;
+
+ public GetJobStatusWork(ClusterControllerService ccs, JobId jobId, IResultCallback<JobStatus> callback) {
+ this.ccs = ccs;
+ this.jobId = jobId;
+ this.callback = callback;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ try {
+ JobRun run = ccs.getActiveRunMap().get(jobId);
+ if (run == null) {
+ run = ccs.getRunMapArchive().get(jobId);
+ }
+ JobStatus status = run == null ? null : run.getStatus();
+ callback.setValue(status);
+ } catch (Exception e) {
+ callback.setException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSummariesJSONWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSummariesJSONWork.java
new file mode 100644
index 0000000..5cc31f6
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetJobSummariesJSONWork.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Collection;
+
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class GetJobSummariesJSONWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private JSONArray summaries;
+
+ public GetJobSummariesJSONWork(ClusterControllerService ccs) {
+ this.ccs = ccs;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ summaries = new JSONArray();
+ populateJSON(ccs.getActiveRunMap().values());
+ populateJSON(ccs.getRunMapArchive().values());
+ }
+
+ private void populateJSON(Collection<JobRun> jobRuns) throws JSONException {
+ for (JobRun run : jobRuns) {
+ JSONObject jo = new JSONObject();
+ jo.put("type", "job-summary");
+ jo.put("job-id", run.getJobId().toString());
+ jo.put("application-name", run.getJobActivityGraph().getApplicationName());
+ jo.put("create-time", run.getCreateTime());
+ jo.put("start-time", run.getCreateTime());
+ jo.put("end-time", run.getCreateTime());
+ jo.put("status", run.getStatus().toString());
+ summaries.put(jo);
+ }
+ }
+
+ public JSONArray getSummaries() {
+ return summaries;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeControllersInfoWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeControllersInfoWork.java
new file mode 100644
index 0000000..2f23a2c
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeControllersInfoWork.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.client.NodeControllerInfo;
+import edu.uci.ics.hyracks.api.client.NodeStatus;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+
+public class GetNodeControllersInfoWork extends AbstractWork {
+ private final ClusterControllerService ccs;
+ private IResultCallback<Map<String, NodeControllerInfo>> callback;
+
+ public GetNodeControllersInfoWork(ClusterControllerService ccs,
+ IResultCallback<Map<String, NodeControllerInfo>> callback) {
+ this.ccs = ccs;
+ this.callback = callback;
+ }
+
+ @Override
+ public void run() {
+ Map<String, NodeControllerInfo> result = new LinkedHashMap<String, NodeControllerInfo>();
+ Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
+ for (Map.Entry<String, NodeControllerState> e : nodeMap.entrySet()) {
+ result.put(e.getKey(), new NodeControllerInfo(e.getKey(), NodeStatus.ALIVE, e.getValue().getDataPort()));
+ }
+ callback.setValue(result);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeDetailsJSONWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeDetailsJSONWork.java
new file mode 100644
index 0000000..4a1f108
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeDetailsJSONWork.java
@@ -0,0 +1,46 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class GetNodeDetailsJSONWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final String nodeId;
+ private JSONObject detail;
+
+ public GetNodeDetailsJSONWork(ClusterControllerService ccs, String nodeId) {
+ this.ccs = ccs;
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ NodeControllerState ncs = ccs.getNodeMap().get(nodeId);
+ if (ncs == null) {
+ detail = new JSONObject();
+ return;
+ }
+ detail = ncs.toDetailedJSON();
+ }
+
+ public JSONObject getDetail() {
+ return detail;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeSummariesJSONWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeSummariesJSONWork.java
new file mode 100644
index 0000000..6ac4ac2
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/GetNodeSummariesJSONWork.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import org.json.JSONArray;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class GetNodeSummariesJSONWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private JSONArray summaries;
+
+ public GetNodeSummariesJSONWork(ClusterControllerService ccs) {
+ this.ccs = ccs;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ summaries = new JSONArray();
+ for (NodeControllerState ncs : ccs.getNodeMap().values()) {
+ summaries.put(ncs.toSummaryJSON());
+ }
+ }
+
+ public JSONArray getSummaries() {
+ return summaries;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java
new file mode 100644
index 0000000..07088bb
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCleanupWork.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Set;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+
+public class JobCleanupWork extends AbstractWork {
+ private static final Logger LOGGER = Logger.getLogger(JobCleanupWork.class.getName());
+
+ private ClusterControllerService ccs;
+ private JobId jobId;
+ private JobStatus status;
+ private Exception exception;
+
+ public JobCleanupWork(ClusterControllerService ccs, JobId jobId, JobStatus status, Exception exception) {
+ this.ccs = ccs;
+ this.jobId = jobId;
+ this.status = status;
+ this.exception = exception;
+ }
+
+ @Override
+ public void run() {
+ final JobRun run = ccs.getActiveRunMap().get(jobId);
+ if (run == null) {
+ LOGGER.warning("Unable to find JobRun with id: " + jobId);
+ return;
+ }
+ if (run.getPendingStatus() != null) {
+ LOGGER.warning("Ignoring duplicate cleanup for JobRun with id: " + jobId);
+ return;
+ }
+ Set<String> targetNodes = run.getParticipatingNodeIds();
+ run.getCleanupPendingNodeIds().addAll(targetNodes);
+ run.setPendingStatus(status, exception);
+ for (String n : targetNodes) {
+ NodeControllerState ncs = ccs.getNodeMap().get(n);
+ try {
+ ncs.getNodeController().cleanUpJoblet(jobId, status);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java
new file mode 100644
index 0000000..b7cf629
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobCreateWork.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.EnumSet;
+
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.cc.job.IOperatorDescriptorVisitor;
+import edu.uci.ics.hyracks.control.cc.job.JobActivityGraphBuilder;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.cc.job.PlanUtils;
+import edu.uci.ics.hyracks.control.cc.scheduler.JobScheduler;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class JobCreateWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final byte[] jobSpec;
+ private final EnumSet<JobFlag> jobFlags;
+ private final JobId jobId;
+ private final String appName;
+ private final IResultCallback<JobId> callback;
+
+ public JobCreateWork(ClusterControllerService ccs, JobId jobId, String appName, byte[] jobSpec,
+ EnumSet<JobFlag> jobFlags, IResultCallback<JobId> callback) {
+ this.jobId = jobId;
+ this.ccs = ccs;
+ this.jobSpec = jobSpec;
+ this.jobFlags = jobFlags;
+ this.appName = appName;
+ this.callback = callback;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ try {
+ CCApplicationContext appCtx = ccs.getApplicationMap().get(appName);
+ if (appCtx == null) {
+ throw new HyracksException("No application with id " + appName + " found");
+ }
+ JobSpecification spec = appCtx.createJobSpecification(jobSpec);
+
+ final JobActivityGraphBuilder builder = new JobActivityGraphBuilder();
+ builder.init(appName, spec, jobFlags);
+ PlanUtils.visit(spec, new IOperatorDescriptorVisitor() {
+ @Override
+ public void visit(IOperatorDescriptor op) {
+ op.contributeActivities(builder);
+ }
+ });
+ final JobActivityGraph jag = builder.getActivityGraph();
+
+ JobRun run = new JobRun(jobId, jag);
+
+ run.setStatus(JobStatus.INITIALIZED, null);
+
+ ccs.getActiveRunMap().put(jobId, run);
+ JobScheduler jrs = new JobScheduler(ccs, run);
+ run.setScheduler(jrs);
+ appCtx.notifyJobCreation(jobId, spec);
+ callback.setValue(jobId);
+ } catch (Exception e) {
+ callback.setException(e);
+ return;
+ }
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java
new file mode 100644
index 0000000..dd834ed
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobStartWork.java
@@ -0,0 +1,56 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class JobStartWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final JobId jobId;
+ private final IResultCallback<Object> callback;
+
+ public JobStartWork(ClusterControllerService ccs, JobId jobId, IResultCallback<Object> callback) {
+ this.ccs = ccs;
+ this.jobId = jobId;
+ this.callback = callback;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ try {
+ JobRun run = ccs.getActiveRunMap().get(jobId);
+ if (run == null) {
+ throw new Exception("Unable to find job with id = " + jobId);
+ }
+ if (run.getStatus() != JobStatus.INITIALIZED) {
+ throw new Exception("Job already started");
+ }
+ run.setStatus(JobStatus.RUNNING, null);
+ try {
+ run.getScheduler().startJob();
+ } catch (Exception e) {
+ ccs.getWorkQueue().schedule(new JobCleanupWork(ccs, run.getJobId(), JobStatus.FAILURE, e));
+ }
+ callback.setValue(null);
+ } catch (Exception e) {
+ callback.setException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobletCleanupNotificationWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobletCleanupNotificationWork.java
new file mode 100644
index 0000000..80ab5ec
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/JobletCleanupNotificationWork.java
@@ -0,0 +1,94 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.json.JSONException;
+import org.json.JSONObject;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.cc.application.CCApplicationContext;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+
+public class JobletCleanupNotificationWork extends AbstractWork {
+ private static final Logger LOGGER = Logger.getLogger(JobletCleanupNotificationWork.class.getName());
+
+ private ClusterControllerService ccs;
+ private JobId jobId;
+ private String nodeId;
+
+ public JobletCleanupNotificationWork(ClusterControllerService ccs, JobId jobId, String nodeId) {
+ this.ccs = ccs;
+ this.jobId = jobId;
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ public void run() {
+ final JobRun run = ccs.getActiveRunMap().get(jobId);
+ Set<String> cleanupPendingNodes = run.getCleanupPendingNodeIds();
+ if (!cleanupPendingNodes.remove(nodeId)) {
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning(nodeId + " not in pending cleanup nodes set: " + cleanupPendingNodes + " for Job: "
+ + jobId);
+ }
+ return;
+ }
+ NodeControllerState ncs = ccs.getNodeMap().get(nodeId);
+ if (ncs != null) {
+ ncs.getActiveJobIds().remove(jobId);
+ }
+ if (cleanupPendingNodes.isEmpty()) {
+ CCApplicationContext appCtx = ccs.getApplicationMap().get(run.getJobActivityGraph().getApplicationName());
+ if (appCtx != null) {
+ try {
+ appCtx.notifyJobFinish(jobId);
+ } catch (HyracksException e) {
+ e.printStackTrace();
+ }
+ }
+ run.setStatus(run.getPendingStatus(), run.getPendingException());
+ ccs.getActiveRunMap().remove(jobId);
+ ccs.getRunMapArchive().put(jobId, run);
+ try {
+ ccs.getJobLogFile().log(createJobLogObject(run));
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+ }
+
+ private JSONObject createJobLogObject(final JobRun run) {
+ JSONObject jobLogObject = new JSONObject();
+ try {
+ JobActivityGraph jag = run.getJobActivityGraph();
+ jobLogObject.put("job-specification", jag.getJobSpecification().toJSON());
+ jobLogObject.put("job-activity-graph", jag.toJSON());
+ jobLogObject.put("job-run", run.toJSON());
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ return jobLogObject;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/NodeHeartbeatWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/NodeHeartbeatWork.java
new file mode 100644
index 0000000..6b139a5
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/NodeHeartbeatWork.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Map;
+import java.util.logging.Level;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class NodeHeartbeatWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final String nodeId;
+ private final HeartbeatData hbData;
+
+ public NodeHeartbeatWork(ClusterControllerService ccs, String nodeId, HeartbeatData hbData) {
+ this.ccs = ccs;
+ this.nodeId = nodeId;
+ this.hbData = hbData;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
+ NodeControllerState state = nodeMap.get(nodeId);
+ if (state != null) {
+ state.notifyHeartbeat(hbData);
+ }
+ }
+
+ @Override
+ public Level logLevel() {
+ return Level.FINEST;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterNodeWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterNodeWork.java
new file mode 100644
index 0000000..e4e0aae
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterNodeWork.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.common.base.INodeController;
+import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
+import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
+import edu.uci.ics.hyracks.control.common.ipc.CCNCFunctions;
+import edu.uci.ics.hyracks.control.common.ipc.NodeControllerRemoteProxy;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+
+public class RegisterNodeWork extends SynchronizableWork {
+ private static final Logger LOGGER = Logger.getLogger(RegisterNodeWork.class.getName());
+
+ private final ClusterControllerService ccs;
+ private final NodeRegistration reg;
+
+ public RegisterNodeWork(ClusterControllerService ccs, NodeRegistration reg) {
+ this.ccs = ccs;
+ this.reg = reg;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ String id = reg.getNodeId();
+
+ IIPCHandle ncIPCHandle = ccs.getClusterIPC().getHandle(reg.getNodeControllerAddress());
+ CCNCFunctions.NodeRegistrationResult result = null;
+ try {
+ INodeController nodeController = new NodeControllerRemoteProxy(ncIPCHandle);
+
+ NodeControllerState state = new NodeControllerState(nodeController, reg);
+ Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
+ if (nodeMap.containsKey(id)) {
+ throw new Exception("Node with this name already registered.");
+ }
+ nodeMap.put(id, state);
+ Map<String, Set<String>> ipAddressNodeNameMap = ccs.getIpAddressNodeNameMap();
+ String ipAddress = state.getNCConfig().dataIPAddress;
+ Set<String> nodes = ipAddressNodeNameMap.get(ipAddress);
+ if (nodes == null) {
+ nodes = new HashSet<String>();
+ ipAddressNodeNameMap.put(ipAddress, nodes);
+ }
+ nodes.add(id);
+ LOGGER.log(Level.INFO, "Registered INodeController: id = " + id);
+ NodeParameters params = new NodeParameters();
+ params.setClusterControllerInfo(ccs.getClusterControllerInfo());
+ params.setHeartbeatPeriod(ccs.getCCConfig().heartbeatPeriod);
+ params.setProfileDumpPeriod(ccs.getCCConfig().profileDumpPeriod);
+ result = new CCNCFunctions.NodeRegistrationResult(params, null);
+ } catch (Exception e) {
+ result = new CCNCFunctions.NodeRegistrationResult(null, e);
+ }
+ ncIPCHandle.send(-1, result, null);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java
new file mode 100644
index 0000000..2456c25
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionAvailibilityWork.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.List;
+import java.util.logging.Level;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.cc.partitions.PartitionMatchMaker;
+import edu.uci.ics.hyracks.control.cc.partitions.PartitionUtils;
+import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
+import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+
+public class RegisterPartitionAvailibilityWork extends AbstractWork {
+ private final ClusterControllerService ccs;
+ private final PartitionDescriptor partitionDescriptor;
+
+ public RegisterPartitionAvailibilityWork(ClusterControllerService ccs, PartitionDescriptor partitionDescriptor) {
+ this.ccs = ccs;
+ this.partitionDescriptor = partitionDescriptor;
+ }
+
+ @Override
+ public void run() {
+ final PartitionId pid = partitionDescriptor.getPartitionId();
+ JobRun run = ccs.getActiveRunMap().get(pid.getJobId());
+ if (run == null) {
+ return;
+ }
+ PartitionMatchMaker pmm = run.getPartitionMatchMaker();
+ List<Pair<PartitionDescriptor, PartitionRequest>> matches = pmm
+ .registerPartitionDescriptor(partitionDescriptor);
+ for (Pair<PartitionDescriptor, PartitionRequest> match : matches) {
+ try {
+ PartitionUtils.reportPartitionMatch(ccs, pid, match);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ public Level logLevel() {
+ return Level.FINEST;
+ }
+
+ @Override
+ public String toString() {
+ return "PartitionAvailable@" + partitionDescriptor;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionRequestWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionRequestWork.java
new file mode 100644
index 0000000..a71875f
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RegisterPartitionRequestWork.java
@@ -0,0 +1,59 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import org.apache.commons.lang3.tuple.Pair;
+
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.cc.partitions.PartitionMatchMaker;
+import edu.uci.ics.hyracks.control.cc.partitions.PartitionUtils;
+import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
+import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+
+public class RegisterPartitionRequestWork extends AbstractWork {
+ private final ClusterControllerService ccs;
+ private final PartitionRequest partitionRequest;
+
+ public RegisterPartitionRequestWork(ClusterControllerService ccs, PartitionRequest partitionRequest) {
+ this.ccs = ccs;
+ this.partitionRequest = partitionRequest;
+ }
+
+ @Override
+ public void run() {
+ PartitionId pid = partitionRequest.getPartitionId();
+ JobRun run = ccs.getActiveRunMap().get(pid.getJobId());
+ if (run == null) {
+ return;
+ }
+ PartitionMatchMaker pmm = run.getPartitionMatchMaker();
+ Pair<PartitionDescriptor, PartitionRequest> match = pmm.matchPartitionRequest(partitionRequest);
+ if (match != null) {
+ try {
+ PartitionUtils.reportPartitionMatch(ccs, pid, match);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "PartitionRequest@" + partitionRequest;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RemoveDeadNodesWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RemoveDeadNodesWork.java
new file mode 100644
index 0000000..7255503
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/RemoveDeadNodesWork.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+
+public class RemoveDeadNodesWork extends AbstractWork {
+ private static Logger LOGGER = Logger.getLogger(RemoveDeadNodesWork.class.getName());
+
+ private final ClusterControllerService ccs;
+
+ public RemoveDeadNodesWork(ClusterControllerService ccs) {
+ this.ccs = ccs;
+ }
+
+ @Override
+ public void run() {
+ Set<String> deadNodes = new HashSet<String>();
+ Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
+ for (Map.Entry<String, NodeControllerState> e : nodeMap.entrySet()) {
+ NodeControllerState state = e.getValue();
+ if (state.incrementLastHeartbeatDuration() >= ccs.getConfig().maxHeartbeatLapsePeriods) {
+ deadNodes.add(e.getKey());
+ LOGGER.info(e.getKey() + " considered dead");
+ }
+ }
+ Set<JobId> affectedJobIds = new HashSet<JobId>();
+ for (String deadNode : deadNodes) {
+ NodeControllerState state = nodeMap.remove(deadNode);
+
+ // Deal with dead tasks.
+ affectedJobIds.addAll(state.getActiveJobIds());
+ }
+ int size = affectedJobIds.size();
+ if (size > 0) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Number of affected jobs: " + size);
+ }
+ for (JobId jobId : affectedJobIds) {
+ JobRun run = ccs.getActiveRunMap().get(jobId);
+ if (run != null) {
+ run.getScheduler().notifyNodeFailures(deadNodes);
+ }
+ }
+ }
+ }
+
+ @Override
+ public Level logLevel() {
+ return Level.FINE;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ReportProfilesWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ReportProfilesWork.java
new file mode 100644
index 0000000..b8eba7e
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/ReportProfilesWork.java
@@ -0,0 +1,52 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+
+public class ReportProfilesWork extends AbstractWork {
+ private final ClusterControllerService ccs;
+ private final List<JobProfile> profiles;
+
+ public ReportProfilesWork(ClusterControllerService ccs, List<JobProfile> profiles) {
+ this.ccs = ccs;
+ this.profiles = profiles;
+ }
+
+ @Override
+ public void run() {
+ Map<JobId, JobRun> runMap = ccs.getActiveRunMap();
+ for (JobProfile profile : profiles) {
+ JobRun run = runMap.get(profile.getJobId());
+ if (run != null) {
+ JobProfile jp = run.getJobProfile();
+ jp.merge(profile);
+ }
+ }
+ }
+
+ @Override
+ public Level logLevel() {
+ return Level.FINEST;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java
new file mode 100644
index 0000000..2694af4
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskCompleteWork.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
+import edu.uci.ics.hyracks.control.cc.job.JobRun;
+import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.JobletProfile;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
+
+public class TaskCompleteWork extends AbstractTaskLifecycleWork {
+ private final TaskProfile statistics;
+
+ public TaskCompleteWork(ClusterControllerService ccs, JobId jobId, TaskAttemptId taId, String nodeId,
+ TaskProfile statistics) {
+ super(ccs, jobId, taId, nodeId);
+ this.statistics = statistics;
+ }
+
+ @Override
+ protected void performEvent(TaskAttempt ta) {
+ try {
+ ActivityCluster ac = ta.getTask().getTaskCluster().getActivityCluster();
+ JobRun run = ac.getJobRun();
+ if (statistics != null) {
+ JobProfile jobProfile = run.getJobProfile();
+ Map<String, JobletProfile> jobletProfiles = jobProfile.getJobletProfiles();
+ JobletProfile jobletProfile = jobletProfiles.get(nodeId);
+ if (jobletProfile == null) {
+ jobletProfile = new JobletProfile(nodeId);
+ jobletProfiles.put(nodeId, jobletProfile);
+ }
+ jobletProfile.getTaskProfiles().put(taId, statistics);
+ }
+ ac.getJobRun().getScheduler().notifyTaskComplete(ta);
+ } catch (HyracksException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "TaskCompleteEvent@[" + nodeId + "[" + jobId + ":" + taId + "]";
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java
new file mode 100644
index 0000000..3201bec
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/TaskFailureWork.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.ActivityCluster;
+import edu.uci.ics.hyracks.control.cc.job.TaskAttempt;
+
+public class TaskFailureWork extends AbstractTaskLifecycleWork {
+ private final String details;
+
+ public TaskFailureWork(ClusterControllerService ccs, JobId jobId, TaskAttemptId taId, String nodeId,
+ String details) {
+ super(ccs, jobId, taId, nodeId);
+ this.details = details;
+ }
+
+ @Override
+ protected void performEvent(TaskAttempt ta) {
+ ActivityCluster ac = ta.getTask().getTaskCluster().getActivityCluster();
+ ac.getJobRun().getScheduler().notifyTaskFailure(ta, ac, details);
+ }
+
+ @Override
+ public String toString() {
+ return "TaskFailureEvent[" + jobId + ":" + taId + ":" + nodeId + "]";
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/UnregisterNodeWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/UnregisterNodeWork.java
new file mode 100644
index 0000000..32ddba9
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/UnregisterNodeWork.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import java.util.Map;
+
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.NodeControllerState;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class UnregisterNodeWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final String nodeId;
+
+ public UnregisterNodeWork(ClusterControllerService ccs, String nodeId) {
+ this.ccs = ccs;
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ Map<String, NodeControllerState> nodeMap = ccs.getNodeMap();
+ nodeMap.remove(nodeId);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/WaitForJobCompletionWork.java b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/WaitForJobCompletionWork.java
new file mode 100644
index 0000000..6cfe025
--- /dev/null
+++ b/hyracks-control-cc/src/main/java/edu/uci/ics/hyracks/control/cc/work/WaitForJobCompletionWork.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.cc.work;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.cc.job.IJobStatusConditionVariable;
+import edu.uci.ics.hyracks.control.common.work.IResultCallback;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+
+public class WaitForJobCompletionWork extends SynchronizableWork {
+ private final ClusterControllerService ccs;
+ private final JobId jobId;
+ private final IResultCallback<Object> callback;
+
+ public WaitForJobCompletionWork(ClusterControllerService ccs, JobId jobId, IResultCallback<Object> callback) {
+ this.ccs = ccs;
+ this.jobId = jobId;
+ this.callback = callback;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ final IJobStatusConditionVariable cRunningVar = ccs.getActiveRunMap().get(jobId);
+ if (cRunningVar != null) {
+ ccs.getExecutor().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ cRunningVar.waitForCompletion();
+ callback.setValue(null);
+ } catch (Exception e) {
+ callback.setException(e);
+ }
+ }
+ });
+ } else {
+ final IJobStatusConditionVariable cArchivedVar = ccs.getRunMapArchive().get(jobId);
+ ccs.getExecutor().execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ cArchivedVar.waitForCompletion();
+ callback.setValue(null);
+ } catch (Exception e) {
+ callback.setException(e);
+ }
+ }
+ });
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/AbstractPage.html b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/AbstractPage.html
new file mode 100644
index 0000000..93cf631
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/AbstractPage.html
@@ -0,0 +1,18 @@
+<html>
+<head>
+ <script type="text/javascript" src="/static/javascript/jquery/jquery.min.js"></script>
+ <script type="text/javascript" src="/static/javascript/jquery/plugins/jquery.timer.js"></script>
+ <script type="text/javascript" src="/static/javascript/jquery/plugins/jquery.getParams.js"></script>
+ <script type="text/javascript" src="/static/javascript/jquery/plugins/jquery-ui.min.js"></script>
+ <script type="text/javascript" src="/static/javascript/flot/jquery.flot.min.js"></script>
+ <script type="text/javascript" src="/static/javascript/flot/jquery.flot.resize.min.js"></script>
+ <script type="text/javascript" src="/static/javascript/jsplumb/jquery.jsPlumb-1.3.5-all-min.js"></script>
+ <link rel="stylesheet" type="text/css" href="/static/stylesheet/jquery-ui/themes/base/jquery-ui.css"></link>
+ <link rel="stylesheet" type="text/css" href="/static/stylesheet/adminconsole.css"></link>
+</head>
+<body>
+ <div id="header">
+ </div>
+ <wicket:child xmlns:wicket/>
+</body>
+</html>
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/IndexPage.html b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/IndexPage.html
new file mode 100644
index 0000000..3f8719e
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/IndexPage.html
@@ -0,0 +1,46 @@
+<wicket:extend xmlns:wicket>
+ <div>
+ Registered Node Count: <span wicket:id="node-count"></span>
+ </div>
+ <table>
+ <tr wicket:id="node-list">
+ <td>
+ <span wicket:id="node-id"></span>
+ </td>
+ <td>
+ <span wicket:id="heap-used"></span>
+ </td>
+ <td>
+ <span wicket:id="system-load-average"></span>
+ </td>
+ <td>
+ <a wicket:id="node-details">Details</a>
+ </td>
+ </tr>
+ </table>
+ <table>
+ <tr wicket:id="jobs-list">
+ <td>
+ <span wicket:id="job-id"></span>
+ </td>
+ <td>
+ <span wicket:id="application-name"></span>
+ </td>
+ <td>
+ <span wicket:id="status"></span>
+ </td>
+ <td>
+ <span wicket:id="create-time"></span>
+ </td>
+ <td>
+ <span wicket:id="start-time"></span>
+ </td>
+ <td>
+ <span wicket:id="end-time"></span>
+ </td>
+ <td>
+ <a wicket:id="job-details">Details</a>
+ </td>
+ </tr>
+ </table>
+</wicket:extend>
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.html b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.html
new file mode 100644
index 0000000..df3350d
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/JobDetailsPage.html
@@ -0,0 +1,13 @@
+<wicket:extend xmlns:wicket>
+ <div id="job-specification" wicket:id="job-specification" style="display: none;">
+ </div>
+ <div id="job-activity-graph" wicket:id="job-activity-graph" style="display: none;">
+ </div>
+ <div id="job-run" wicket:id="job-run" style="display: none;">
+ </div>
+ <div wicket:id="job-timeline" style="overflow: auto;"></div>
+ <div id="job-graph" style="position:relative"></div>
+
+ <script src="/static/javascript/adminconsole/Graphs.js" type="text/javascript"></script>
+ <script src="/static/javascript/adminconsole/JobDetailsPage.js" type="text/javascript"></script>
+</wicket:extend>
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.html b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.html
new file mode 100644
index 0000000..1d8f735
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/edu/uci/ics/hyracks/control/cc/adminconsole/pages/NodeDetailsPage.html
@@ -0,0 +1,68 @@
+<wicket:extend xmlns:wicket>
+ <div id="tabs">
+ <ul>
+ <li><a href="#tabs-1">Node Information</a></li>
+ <li><a href="#tabs-2">System Load</a></li>
+ <li><a href="#tabs-3">Garbage Collection</a></li>
+ <li><a href="#tabs-4">Memory Usage</a></li>
+ <li><a href="#tabs-5">Thread Statistics</a></li>
+ <li><a href="#tabs-6">Network Statistics</a></li>
+ <li><a href="#tabs-7">IPC Statistics</a></li>
+ </ul>
+
+ <div id="tabs-1">
+ <table border="0">
+ <tr><td>Node Id</td><td><span id="node-id"></span></td></tr>
+ <tr><td>Architecture</td><td><span id="arch"></span></td></tr>
+ <tr><td>OS Name</td><td><span id="os-name"></span></td></tr>
+ <tr><td>OS Version</td><td><span id="os-version"></span></td></tr>
+ <tr><td># of processors</td><td><span id="num-processors"></span></td></tr>
+ <tr><td>Java VM Name</td><td><span id="vm-name"></span></td></tr>
+ <tr><td>Java VM Version</td><td><span id="vm-version"></span></td></tr>
+ <tr><td>Java VM Vendor</td><td><span id="vm-vendor"></span></td></tr>
+ <tr><td>Java Classpath</td><td><span id="classpath"></span></td></tr>
+ <tr><td>Java Library Path</td><td><span id="library-path"></span></td></tr>
+ <tr><td>Java Boot Classpath</td><td><span id="boot-classpath"></span></td></tr>
+ <tr><td>Java VM Arguments</td><td><div id="input-arguments"></div></td></tr>
+ </table>
+ </div>
+
+ <div id="tabs-2">
+ <div id='system-load-chart' class="time-chart">
+ </div>
+ </div>
+
+ <div id="tabs-3">
+ <div id='gc-charts'>
+ </div>
+ </div>
+
+ <div id="tabs-4">
+ <div id='heap-usage-chart' class="time-chart">
+ </div>
+ <div id='nonheap-usage-chart' class="time-chart">
+ </div>
+ </div>
+
+ <div id="tabs-5">
+ <div id='thread-chart' class="time-chart">
+ </div>
+ </div>
+
+ <div id="tabs-6">
+ <div id='net-payload-bandwidth' class="time-chart">
+ </div>
+ <div id='net-signaling-bandwidth' class="time-chart">
+ </div>
+ </div>
+
+ <div id="tabs-7">
+ <div id='ipc-messages' class="time-chart">
+ </div>
+ <div id='ipc-message-bytes' class="time-chart">
+ </div>
+ </div>
+ </div>
+
+ <script src="/static/javascript/adminconsole/NodeDetailsPage.js" type="text/javascript"></script>
+</wicket:extend>
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/adminconsole/Graphs.js b/hyracks-control-cc/src/main/resources/static/javascript/adminconsole/Graphs.js
new file mode 100644
index 0000000..dcba9c0
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/adminconsole/Graphs.js
@@ -0,0 +1,160 @@
+Graphs = {};
+
+Graphs.DAG = function() {
+ this.nodeMap = {};
+ this.edgeMap = {};
+}
+
+Graphs.DAG.prototype.addNode = function(key, attachment) {
+ var node = new Graphs.Node(this, key, attachment);
+ this.nodeMap[key] = node;
+ return node;
+}
+
+Graphs.DAG.prototype.addEdge = function(key, attachment, sNode, sIndex, tNode, tIndex) {
+ var edge = new Graphs.DirectedEdge(this, key, attachment, sNode, sIndex, tNode, tIndex);
+ this.edgeMap[key] = edge;
+ sNode.outEdges[sIndex] = edge;
+ tNode.inEdges[tIndex] = edge;
+ return edge;
+}
+
+Graphs.DAG.prototype.lookupNode = function(key) {
+ return this.nodeMap[key];
+}
+
+Graphs.DAG.prototype.lookupEdge = function(key) {
+ return this.edgeMap[key];
+}
+
+Graphs.DAG.prototype.walkNodes = function(callback) {
+ for ( var nKey in this.nodeMap) {
+ callback(this.nodeMap[nKey]);
+ }
+}
+
+Graphs.DAG.prototype.walkEdges = function(callback) {
+ for ( var eKey in this.edgeMap) {
+ callback(this.edgeMap[eKey]);
+ }
+}
+
+Graphs.DAG.prototype.findRoots = function() {
+ var roots = [];
+ var callback = function(node) {
+ if (node.inEdges.length == 0) {
+ roots.push(node);
+ }
+ }
+ this.walkNodes(callback);
+ return roots;
+}
+
+Graphs.DAG.prototype.findLeaves = function() {
+ var leaves = [];
+ var callback = function(node) {
+ if (node.outEdges.length == 0) {
+ leaves.push(node);
+ }
+ }
+ this.walkNodes(callback);
+ return leaves;
+}
+
+Graphs.DAG.prototype.tsort = function() {
+ var sortedNodes = [];
+ var nodeState = {};
+
+ function visit(node) {
+ if (!nodeState[node.key]) {
+ nodeState[node.key] = true;
+ for ( var i = 0; i < node.inEdges.length; ++i) {
+ visit(node.inEdges[i].sNode);
+ }
+ sortedNodes.push(node);
+ }
+ }
+
+ var roots = this.findLeaves();
+ for ( var i = 0; i < roots.length; ++i) {
+ visit(roots[i]);
+ }
+ return sortedNodes;
+}
+
+Graphs.DAG.prototype.stratify = function() {
+ var sortedNodes = this.tsort();
+ var stratumMap = {};
+ var strata = [];
+ for ( var i = 0; i < sortedNodes.length; ++i) {
+ var node = sortedNodes[i];
+ var maxParentStratum = -1;
+ for ( var j = 0; j < node.inEdges.length; ++j) {
+ var edge = node.inEdges[j];
+ maxParentStratum = Math.max(maxParentStratum, stratumMap[edge.sNode.key]);
+ }
+ var stratum = maxParentStratum + 1;
+ stratumMap[node.key] = stratum;
+ var stratumList = strata[stratum];
+ if (!stratumList) {
+ stratumList = [];
+ strata[stratum] = stratumList;
+ }
+ stratumList.push(node);
+ }
+ return strata;
+}
+
+Graphs.Node = function(dag, key, attachment) {
+ this.dag = dag;
+ this.key = key;
+ this.attachment = attachment;
+ this.inEdges = [];
+ this.outEdges = [];
+}
+
+Graphs.DirectedEdge = function(dag, key, attachment, sNode, sIndex, tNode, tIndex) {
+ this.dag = dag;
+ this.key = key;
+ this.attachment = attachment;
+ this.sNode = sNode;
+ this.sIndex = sIndex;
+ this.tNode = tNode;
+ this.tIndex = tIndex;
+}
+
+Graphs.JsPlumbRenderer = function(dag, element, options) {
+ this.dag = dag;
+ this.element = element;
+ this.options = options || {
+ levelStep : 100
+ };
+}
+
+Graphs.JsPlumbRenderer.prototype.configureDiv = function(div, node, stratum, inStratumIndex, stratumWidth) {
+ div.id = node.key;
+ div.dagNode = node;
+ /*
+ * div.style.position = 'absolute'; div.style.left = (stratum *
+ * this.options.levelStep) + 'px'; div.style.top = (inStratumIndex * 100) +
+ * 'px'; div.style.width = '50px'; div.style.height = '50px';
+ * div.style.border = 'thick solid #000000';
+ */
+}
+
+Graphs.JsPlumbRenderer.prototype.refresh = function() {
+ var strata = this.dag.stratify();
+
+ while (this.element.hasChildNodes()) {
+ this.element.removeChild(this.element.lastChild);
+ }
+ for ( var i = 0; i < strata.length; ++i) {
+ var stratumList = strata[i];
+ for ( var j = 0; j < stratumList.length; ++j) {
+ var node = stratumList[j];
+ var div = document.createElement('div');
+ this.configureDiv(div, node, i, j, stratumList.length);
+ this.element.appendChild(div);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/adminconsole/JobDetailsPage.js b/hyracks-control-cc/src/main/resources/static/javascript/adminconsole/JobDetailsPage.js
new file mode 100644
index 0000000..828e310
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/adminconsole/JobDetailsPage.js
@@ -0,0 +1,85 @@
+$(function() {
+ var jobSpecDAG = new Graphs.DAG();
+ var jobSpecRenderer;
+
+ function drawJobGraph() {
+ var jobGraphDiv = $('#job-graph')[0];
+ jobSpecRenderer = new Graphs.JsPlumbRenderer(jobSpecDAG, jobGraphDiv, null);
+ jobSpecRenderer.refresh();
+ }
+
+ function onJobRunDataReceived(data) {
+ var run = data.result;
+
+ if (run.status != 'TERMINATED' && run.status != 'FAILURE') {
+ setTimeout(fetchJobRun, 10000);
+ }
+ }
+
+ function fetchJobRun() {
+ $.ajax({
+ url : '/rest/jobs/' + $.getURLParam('job-id') + '/job-run',
+ method : 'GET',
+ dataType : 'json',
+ success : onJobRunDataReceived
+ });
+ }
+
+ function onJobActivityGraphDataReceived(data) {
+ var jag = data.result;
+ activityMap = new Object;
+ var activities = jag.activities;
+ for ( var i = 0; i < activities.length; ++i) {
+ var activity = activities[i];
+ }
+
+ drawJobGraph();
+
+ fetchJobRun();
+ }
+
+ function fetchJobActivityGraph() {
+ $.ajax({
+ url : '/rest/jobs/' + $.getURLParam('job-id') + '/job-activity-graph',
+ method : 'GET',
+ dataType : 'json',
+ success : onJobActivityGraphDataReceived
+ });
+ }
+
+ function onJobSpecificationDataReceived(data) {
+ var jobSpec = data.result;
+ var operators = jobSpec.operators;
+ for ( var i = 0; i < operators.length; ++i) {
+ var op = operators[i];
+ jobSpecDAG.addNode(op.id, op);
+ }
+ var connectors = jobSpec.connectors;
+ for ( var i = 0; i < connectors.length; ++i) {
+ var conn = connectors[i];
+ var sNode = jobSpecDAG.lookupNode(conn['in-operator-id']);
+ var sIndex = conn['in-operator-port'];
+ var tNode = jobSpecDAG.lookupNode(conn['out-operator-id']);
+ var tIndex = conn['out-operator-port'];
+ jobSpecDAG.addEdge(conn.id, conn, sNode, sIndex, tNode, tIndex);
+ }
+ fetchJobActivityGraph();
+ }
+
+ function fetchJobSpecification() {
+ $.ajax({
+ url : '/rest/jobs/' + $.getURLParam('job-id') + '/job-specification',
+ method : 'GET',
+ dataType : 'json',
+ success : onJobSpecificationDataReceived
+ });
+ }
+
+ function init() {
+ fetchJobSpecification();
+ }
+
+ jsPlumb.bind("ready", function() {
+ init();
+ });
+});
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/adminconsole/NodeDetailsPage.js b/hyracks-control-cc/src/main/resources/static/javascript/adminconsole/NodeDetailsPage.js
new file mode 100644
index 0000000..ff9d8a0
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/adminconsole/NodeDetailsPage.js
@@ -0,0 +1,232 @@
+$(function() {
+ $('#tabs').tabs();
+
+ var options = {
+ lines : {
+ show : true,
+ fill : true
+ },
+ points : {
+ show : false
+ },
+ xaxis : {
+ tickDecimals : 0,
+ tickSize : 0
+ }
+ };
+
+ function computeRate(array, rrdPtr) {
+ return (array[(rrdPtr + 1) % array.length] - array[rrdPtr]) / 10;
+ }
+
+ function onDataReceived(data) {
+ var result = data.result;
+ $('#node-id')[0].innerHTML = result['node-id'];
+ $('#arch')[0].innerHTML = result['arch'];
+ $('#os-name')[0].innerHTML = result['os-name'];
+ $('#os-version')[0].innerHTML = result['os-version'];
+ $('#num-processors')[0].innerHTML = result['num-processors'];
+ $('#vm-name')[0].innerHTML = result['vm-name'];
+ $('#vm-version')[0].innerHTML = result['vm-version'];
+ $('#vm-vendor')[0].innerHTML = result['vm-vendor'];
+ $('#classpath')[0].innerHTML = result['classpath'];
+ $('#library-path')[0].innerHTML = result['library-path'];
+ $('#boot-classpath')[0].innerHTML = result['boot-classpath'];
+ var argsHTML = "";
+ var args = result['input-arguments'];
+ for ( var i = 0; i < args.length; ++i) {
+ if (argsHTML != "") {
+ argsHTML += "<br/>";
+ }
+ argsHTML += "<span>" + args[i] + "</span>";
+ }
+ $('#input-arguments')[0].innerHTML = argsHTML;
+ var sysLoad = result['system-load-averages'];
+ var heapUsageInitSizes = result['heap-init-sizes'];
+ var heapUsageUsedSizes = result['heap-used-sizes'];
+ var heapUsageCommittedSizes = result['heap-committed-sizes'];
+ var heapUsageMaxSizes = result['heap-max-sizes'];
+ var nonheapUsageInitSizes = result['nonheap-init-sizes'];
+ var nonheapUsageUsedSizes = result['nonheap-used-sizes'];
+ var nonheapUsageCommittedSizes = result['nonheap-committed-sizes'];
+ var nonheapUsageMaxSizes = result['nonheap-max-sizes'];
+ var threadCounts = result['thread-counts'];
+ var peakThreadCounts = result['peak-thread-counts'];
+ var gcCollectionCounts = result['gc-collection-counts'];
+ var gcCollectionTimes = result['gc-collection-times'];
+ var netPayloadBytesRead = result['net-payload-bytes-read'];
+ var netPayloadBytesWritten = result['net-payload-bytes-written'];
+ var netSignalingBytesRead = result['net-signaling-bytes-read'];
+ var netSignalingBytesWritten = result['net-signaling-bytes-written'];
+ var ipcMessagesSent = result['ipc-messages-sent'];
+ var ipcMessageBytesSent = result['ipc-message-bytes-sent'];
+ var ipcMessagesReceived = result['ipc-messages-received'];
+ var ipcMessageBytesReceived = result['ipc-message-bytes-received'];
+
+ var sysLoadArray = [];
+ var heapUsageInitSizesArray = [];
+ var heapUsageUsedSizesArray = [];
+ var heapUsageCommittedSizesArray = [];
+ var heapUsageMaxSizesArray = [];
+ var nonheapUsageInitSizesArray = [];
+ var nonheapUsageUsedSizesArray = [];
+ var nonheapUsageCommittedSizesArray = [];
+ var nonheapUsageMaxSizesArray = [];
+ var threadCountsArray = [];
+ var peakThreadCountsArray = [];
+ var gcCollectionCountsArray = [];
+ var gcCollectionTimesArray = [];
+ var netPayloadReadBWArray = [];
+ var netPayloadWriteBWArray = [];
+ var netSignalingReadBWArray = [];
+ var netSignalingWriteBWArray = [];
+ var ipcMessageSendRateArray = [];
+ var ipcMessageBytesSendRateArray = [];
+ var ipcMessageReceiveRateArray = [];
+ var ipcMessageBytesReceiveRateArray = [];
+ var gcChartsDiv = document.getElementById('gc-charts');
+ for ( var i = 0; i < gcCollectionCounts.length; ++i) {
+ gcCollectionCountsArray.push([]);
+ gcCollectionTimesArray.push([]);
+ var gcCollectionCountDiv = document.createElement('div');
+ gcCollectionCountDiv.id = 'gc-collection-count-' + i;
+ gcCollectionCountDiv.style = "jqplot-target time-chart";
+ gcChartsDiv.appendChild(gcCollectionCountDiv);
+ var gcCollectionTimeDiv = document.createElement('div');
+ gcCollectionTimeDiv.id = 'gc-collection-time-' + i;
+ gcChartsDiv.appendChild(gcCollectionTimeDiv);
+ }
+ var rrdPtr = result['rrd-ptr'];
+ for ( var i = 0; i < sysLoad.length; ++i) {
+ sysLoadArray.push([ i, sysLoad[rrdPtr] ]);
+ heapUsageInitSizesArray.push([ i, heapUsageInitSizes[rrdPtr] / (1024 * 1024) ]);
+ heapUsageUsedSizesArray.push([ i, heapUsageUsedSizes[rrdPtr] / (1024 * 1024) ]);
+ heapUsageCommittedSizesArray.push([ i, heapUsageCommittedSizes[rrdPtr] / (1024 * 1024) ]);
+ heapUsageMaxSizesArray.push([ i, heapUsageMaxSizes[rrdPtr] / (1024 * 1024) ]);
+ nonheapUsageInitSizesArray.push([ i, nonheapUsageInitSizes[rrdPtr] / (1024 * 1024) ]);
+ nonheapUsageUsedSizesArray.push([ i, nonheapUsageUsedSizes[rrdPtr] / (1024 * 1024) ]);
+ nonheapUsageCommittedSizesArray.push([ i, nonheapUsageCommittedSizes[rrdPtr] / (1024 * 1024) ]);
+ nonheapUsageMaxSizesArray.push([ i, nonheapUsageMaxSizes[rrdPtr] / (1024 * 1024) ]);
+ threadCountsArray.push([ i, threadCounts[rrdPtr] ]);
+ peakThreadCountsArray.push([ i, peakThreadCounts[rrdPtr] ]);
+ for ( var j = 0; j < gcCollectionCounts.length; ++j) {
+ gcCollectionCountsArray[j].push([ i, gcCollectionCounts[j][rrdPtr] ]);
+ }
+ for ( var j = 0; j < gcCollectionTimes.length; ++j) {
+ gcCollectionTimesArray[j].push([ i, gcCollectionTimes[j][rrdPtr] ]);
+ }
+ if (i < sysLoad.length - 1) {
+ netPayloadReadBWArray.push([ i, computeRate(netPayloadBytesRead, rrdPtr) ]);
+ netPayloadWriteBWArray.push([ i, computeRate(netPayloadBytesWritten, rrdPtr) ]);
+ netSignalingReadBWArray.push([ i, computeRate(netSignalingBytesRead, rrdPtr) ]);
+ netSignalingWriteBWArray.push([ i, computeRate(netSignalingBytesWritten, rrdPtr) ]);
+ ipcMessageSendRateArray.push([ i, computeRate(ipcMessagesSent, rrdPtr) ]);
+ ipcMessageBytesSendRateArray.push([ i, computeRate(ipcMessageBytesSent, rrdPtr) ]);
+ ipcMessageReceiveRateArray.push([ i, computeRate(ipcMessagesReceived, rrdPtr) ]);
+ ipcMessageBytesReceiveRateArray.push([ i, computeRate(ipcMessageBytesReceived, rrdPtr) ]);
+ }
+ rrdPtr = (rrdPtr + 1) % sysLoad.length;
+ }
+
+ $.plot($('#system-load-chart'), [ {
+ label : 'System Load',
+ data : sysLoadArray
+ } ], options);
+
+ var gcNames = result['gc-names'];
+ for ( var i = 0; i < gcCollectionCounts.length; ++i) {
+ $('#gc-collection-count-' + i).addClass("time-chart");
+ $.plot($('#gc-collection-count-' + i), [ {
+ label : gcNames[i] + ' Collection Count',
+ data : gcCollectionCountsArray[i]
+ } ], options);
+
+ $('#gc-collection-time-' + i).addClass("time-chart");
+ $.plot($('#gc-collection-time-' + i), [ {
+ label : gcNames[i] + ' Collection Time (ms)',
+ data : gcCollectionTimesArray[i]
+ } ], options);
+ }
+
+ $.plot($('#heap-usage-chart'), [ {
+ label : 'Heap Usage Initial Size (MB)',
+ data : heapUsageInitSizesArray
+ }, {
+ label : 'Heap Usage Used Size (MB)',
+ data : heapUsageUsedSizesArray
+ }, {
+ label : 'Heap Usage Committed Size (MB)',
+ data : heapUsageCommittedSizesArray
+ }, {
+ label : 'Heap Usage Max Size (MB)',
+ data : heapUsageMaxSizesArray
+ } ], options);
+
+ $.plot($('#nonheap-usage-chart'), [ {
+ label : 'Non-Heap Usage Initial Size (MB)',
+ data : nonheapUsageInitSizesArray
+ }, {
+ label : 'Non-Heap Usage Used Size (MB)',
+ data : nonheapUsageUsedSizesArray
+ }, {
+ label : 'Non-Heap Usage Committed Size (MB)',
+ data : nonheapUsageCommittedSizesArray
+ }, {
+ label : 'Non-Heap Usage Max Size (MB)',
+ data : nonheapUsageMaxSizesArray
+ } ], options);
+
+ $.plot($('#thread-chart'), [ {
+ label : 'Thread Count',
+ data : threadCountsArray
+ }, {
+ label : 'Peak Thread Count',
+ data : peakThreadCountsArray
+ } ], options);
+
+ $.plot($('#net-payload-bandwidth'), [ {
+ label : 'Payload Read Bandwidth (bytes/sec)',
+ data : netPayloadReadBWArray
+ }, {
+ label : 'Payload Write Bandwidth (bytes/sec)',
+ data : netPayloadWriteBWArray
+ } ], options);
+
+ $.plot($('#net-signaling-bandwidth'), [ {
+ label : 'Signaling Read Bandwidth (bytes/sec)',
+ data : netSignalingReadBWArray
+ }, {
+ label : 'Signaling Write Bandwidth (bytes/sec)',
+ data : netSignalingWriteBWArray
+ } ], options);
+
+ $.plot($('#ipc-messages'), [ {
+ label : 'IPC Messages Send Rate (messages/sec)',
+ data : ipcMessageSendRateArray
+ }, {
+ label : 'IPC Messages Receive Rate (messages/sec)',
+ data : ipcMessageReceiveRateArray
+ } ], options);
+
+ $.plot($('#ipc-message-bytes'), [ {
+ label : 'IPC Message Send Bandwidth (bytes/sec)',
+ data : ipcMessageBytesSendRateArray
+ }, {
+ label : 'IPC Message Receive Bandwidth (bytes/sec)',
+ data : ipcMessageBytesReceiveRateArray
+ } ], options);
+ }
+
+ function fetchData() {
+ $.ajax({
+ url : '/rest/nodes/' + $.getURLParam('node-id'),
+ method : 'GET',
+ dataType : 'json',
+ success : onDataReceived
+ });
+
+ setTimeout(fetchData, 10000);
+ }
+
+ fetchData();
+});
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/excanvas.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/excanvas.min.js
new file mode 100644
index 0000000..12c74f7
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/excanvas.min.js
@@ -0,0 +1 @@
+if(!document.createElement("canvas").getContext){(function(){var z=Math;var K=z.round;var J=z.sin;var U=z.cos;var b=z.abs;var k=z.sqrt;var D=10;var F=D/2;function T(){return this.context_||(this.context_=new W(this))}var O=Array.prototype.slice;function G(i,j,m){var Z=O.call(arguments,2);return function(){return i.apply(j,Z.concat(O.call(arguments)))}}function AD(Z){return String(Z).replace(/&/g,"&").replace(/"/g,""")}function r(i){if(!i.namespaces.g_vml_){i.namespaces.add("g_vml_","urn:schemas-microsoft-com:vml","#default#VML")}if(!i.namespaces.g_o_){i.namespaces.add("g_o_","urn:schemas-microsoft-com:office:office","#default#VML")}if(!i.styleSheets.ex_canvas_){var Z=i.createStyleSheet();Z.owningElement.id="ex_canvas_";Z.cssText="canvas{display:inline-block;overflow:hidden;text-align:left;width:300px;height:150px}"}}r(document);var E={init:function(Z){if(/MSIE/.test(navigator.userAgent)&&!window.opera){var i=Z||document;i.createElement("canvas");i.attachEvent("onreadystatechange",G(this.init_,this,i))}},init_:function(m){var j=m.getElementsByTagName("canvas");for(var Z=0;Z<j.length;Z++){this.initElement(j[Z])}},initElement:function(i){if(!i.getContext){i.getContext=T;r(i.ownerDocument);i.innerHTML="";i.attachEvent("onpropertychange",S);i.attachEvent("onresize",w);var Z=i.attributes;if(Z.width&&Z.width.specified){i.style.width=Z.width.nodeValue+"px"}else{i.width=i.clientWidth}if(Z.height&&Z.height.specified){i.style.height=Z.height.nodeValue+"px"}else{i.height=i.clientHeight}}return i}};function S(i){var Z=i.srcElement;switch(i.propertyName){case"width":Z.getContext().clearRect();Z.style.width=Z.attributes.width.nodeValue+"px";Z.firstChild.style.width=Z.clientWidth+"px";break;case"height":Z.getContext().clearRect();Z.style.height=Z.attributes.height.nodeValue+"px";Z.firstChild.style.height=Z.clientHeight+"px";break}}function w(i){var Z=i.srcElement;if(Z.firstChild){Z.firstChild.style.width=Z.clientWidth+"px";Z.firstChild.style.height=Z.clientHeight+"px"}}E.init();var I=[];for(var AC=0;AC<16;AC++){for(var AB=0;AB<16;AB++){I[AC*16+AB]=AC.toString(16)+AB.toString(16)}}function V(){return[[1,0,0],[0,1,0],[0,0,1]]}function d(m,j){var i=V();for(var Z=0;Z<3;Z++){for(var AF=0;AF<3;AF++){var p=0;for(var AE=0;AE<3;AE++){p+=m[Z][AE]*j[AE][AF]}i[Z][AF]=p}}return i}function Q(i,Z){Z.fillStyle=i.fillStyle;Z.lineCap=i.lineCap;Z.lineJoin=i.lineJoin;Z.lineWidth=i.lineWidth;Z.miterLimit=i.miterLimit;Z.shadowBlur=i.shadowBlur;Z.shadowColor=i.shadowColor;Z.shadowOffsetX=i.shadowOffsetX;Z.shadowOffsetY=i.shadowOffsetY;Z.strokeStyle=i.strokeStyle;Z.globalAlpha=i.globalAlpha;Z.font=i.font;Z.textAlign=i.textAlign;Z.textBaseline=i.textBaseline;Z.arcScaleX_=i.arcScaleX_;Z.arcScaleY_=i.arcScaleY_;Z.lineScale_=i.lineScale_}var B={aliceblue:"#F0F8FF",antiquewhite:"#FAEBD7",aquamarine:"#7FFFD4",azure:"#F0FFFF",beige:"#F5F5DC",bisque:"#FFE4C4",black:"#000000",blanchedalmond:"#FFEBCD",blueviolet:"#8A2BE2",brown:"#A52A2A",burlywood:"#DEB887",cadetblue:"#5F9EA0",chartreuse:"#7FFF00",chocolate:"#D2691E",coral:"#FF7F50",cornflowerblue:"#6495ED",cornsilk:"#FFF8DC",crimson:"#DC143C",cyan:"#00FFFF",darkblue:"#00008B",darkcyan:"#008B8B",darkgoldenrod:"#B8860B",darkgray:"#A9A9A9",darkgreen:"#006400",darkgrey:"#A9A9A9",darkkhaki:"#BDB76B",darkmagenta:"#8B008B",darkolivegreen:"#556B2F",darkorange:"#FF8C00",darkorchid:"#9932CC",darkred:"#8B0000",darksalmon:"#E9967A",darkseagreen:"#8FBC8F",darkslateblue:"#483D8B",darkslategray:"#2F4F4F",darkslategrey:"#2F4F4F",darkturquoise:"#00CED1",darkviolet:"#9400D3",deeppink:"#FF1493",deepskyblue:"#00BFFF",dimgray:"#696969",dimgrey:"#696969",dodgerblue:"#1E90FF",firebrick:"#B22222",floralwhite:"#FFFAF0",forestgreen:"#228B22",gainsboro:"#DCDCDC",ghostwhite:"#F8F8FF",gold:"#FFD700",goldenrod:"#DAA520",grey:"#808080",greenyellow:"#ADFF2F",honeydew:"#F0FFF0",hotpink:"#FF69B4",indianred:"#CD5C5C",indigo:"#4B0082",ivory:"#FFFFF0",khaki:"#F0E68C",lavender:"#E6E6FA",lavenderblush:"#FFF0F5",lawngreen:"#7CFC00",lemonchiffon:"#FFFACD",lightblue:"#ADD8E6",lightcoral:"#F08080",lightcyan:"#E0FFFF",lightgoldenrodyellow:"#FAFAD2",lightgreen:"#90EE90",lightgrey:"#D3D3D3",lightpink:"#FFB6C1",lightsalmon:"#FFA07A",lightseagreen:"#20B2AA",lightskyblue:"#87CEFA",lightslategray:"#778899",lightslategrey:"#778899",lightsteelblue:"#B0C4DE",lightyellow:"#FFFFE0",limegreen:"#32CD32",linen:"#FAF0E6",magenta:"#FF00FF",mediumaquamarine:"#66CDAA",mediumblue:"#0000CD",mediumorchid:"#BA55D3",mediumpurple:"#9370DB",mediumseagreen:"#3CB371",mediumslateblue:"#7B68EE",mediumspringgreen:"#00FA9A",mediumturquoise:"#48D1CC",mediumvioletred:"#C71585",midnightblue:"#191970",mintcream:"#F5FFFA",mistyrose:"#FFE4E1",moccasin:"#FFE4B5",navajowhite:"#FFDEAD",oldlace:"#FDF5E6",olivedrab:"#6B8E23",orange:"#FFA500",orangered:"#FF4500",orchid:"#DA70D6",palegoldenrod:"#EEE8AA",palegreen:"#98FB98",paleturquoise:"#AFEEEE",palevioletred:"#DB7093",papayawhip:"#FFEFD5",peachpuff:"#FFDAB9",peru:"#CD853F",pink:"#FFC0CB",plum:"#DDA0DD",powderblue:"#B0E0E6",rosybrown:"#BC8F8F",royalblue:"#4169E1",saddlebrown:"#8B4513",salmon:"#FA8072",sandybrown:"#F4A460",seagreen:"#2E8B57",seashell:"#FFF5EE",sienna:"#A0522D",skyblue:"#87CEEB",slateblue:"#6A5ACD",slategray:"#708090",slategrey:"#708090",snow:"#FFFAFA",springgreen:"#00FF7F",steelblue:"#4682B4",tan:"#D2B48C",thistle:"#D8BFD8",tomato:"#FF6347",turquoise:"#40E0D0",violet:"#EE82EE",wheat:"#F5DEB3",whitesmoke:"#F5F5F5",yellowgreen:"#9ACD32"};function g(i){var m=i.indexOf("(",3);var Z=i.indexOf(")",m+1);var j=i.substring(m+1,Z).split(",");if(j.length==4&&i.substr(3,1)=="a"){alpha=Number(j[3])}else{j[3]=1}return j}function C(Z){return parseFloat(Z)/100}function N(i,j,Z){return Math.min(Z,Math.max(j,i))}function c(AF){var j,i,Z;h=parseFloat(AF[0])/360%360;if(h<0){h++}s=N(C(AF[1]),0,1);l=N(C(AF[2]),0,1);if(s==0){j=i=Z=l}else{var m=l<0.5?l*(1+s):l+s-l*s;var AE=2*l-m;j=A(AE,m,h+1/3);i=A(AE,m,h);Z=A(AE,m,h-1/3)}return"#"+I[Math.floor(j*255)]+I[Math.floor(i*255)]+I[Math.floor(Z*255)]}function A(i,Z,j){if(j<0){j++}if(j>1){j--}if(6*j<1){return i+(Z-i)*6*j}else{if(2*j<1){return Z}else{if(3*j<2){return i+(Z-i)*(2/3-j)*6}else{return i}}}}function Y(Z){var AE,p=1;Z=String(Z);if(Z.charAt(0)=="#"){AE=Z}else{if(/^rgb/.test(Z)){var m=g(Z);var AE="#",AF;for(var j=0;j<3;j++){if(m[j].indexOf("%")!=-1){AF=Math.floor(C(m[j])*255)}else{AF=Number(m[j])}AE+=I[N(AF,0,255)]}p=m[3]}else{if(/^hsl/.test(Z)){var m=g(Z);AE=c(m);p=m[3]}else{AE=B[Z]||Z}}}return{color:AE,alpha:p}}var L={style:"normal",variant:"normal",weight:"normal",size:10,family:"sans-serif"};var f={};function X(Z){if(f[Z]){return f[Z]}var m=document.createElement("div");var j=m.style;try{j.font=Z}catch(i){}return f[Z]={style:j.fontStyle||L.style,variant:j.fontVariant||L.variant,weight:j.fontWeight||L.weight,size:j.fontSize||L.size,family:j.fontFamily||L.family}}function P(j,i){var Z={};for(var AF in j){Z[AF]=j[AF]}var AE=parseFloat(i.currentStyle.fontSize),m=parseFloat(j.size);if(typeof j.size=="number"){Z.size=j.size}else{if(j.size.indexOf("px")!=-1){Z.size=m}else{if(j.size.indexOf("em")!=-1){Z.size=AE*m}else{if(j.size.indexOf("%")!=-1){Z.size=(AE/100)*m}else{if(j.size.indexOf("pt")!=-1){Z.size=m/0.75}else{Z.size=AE}}}}}Z.size*=0.981;return Z}function AA(Z){return Z.style+" "+Z.variant+" "+Z.weight+" "+Z.size+"px "+Z.family}function t(Z){switch(Z){case"butt":return"flat";case"round":return"round";case"square":default:return"square"}}function W(i){this.m_=V();this.mStack_=[];this.aStack_=[];this.currentPath_=[];this.strokeStyle="#000";this.fillStyle="#000";this.lineWidth=1;this.lineJoin="miter";this.lineCap="butt";this.miterLimit=D*1;this.globalAlpha=1;this.font="10px sans-serif";this.textAlign="left";this.textBaseline="alphabetic";this.canvas=i;var Z=i.ownerDocument.createElement("div");Z.style.width=i.clientWidth+"px";Z.style.height=i.clientHeight+"px";Z.style.overflow="hidden";Z.style.position="absolute";i.appendChild(Z);this.element_=Z;this.arcScaleX_=1;this.arcScaleY_=1;this.lineScale_=1}var M=W.prototype;M.clearRect=function(){if(this.textMeasureEl_){this.textMeasureEl_.removeNode(true);this.textMeasureEl_=null}this.element_.innerHTML=""};M.beginPath=function(){this.currentPath_=[]};M.moveTo=function(i,Z){var j=this.getCoords_(i,Z);this.currentPath_.push({type:"moveTo",x:j.x,y:j.y});this.currentX_=j.x;this.currentY_=j.y};M.lineTo=function(i,Z){var j=this.getCoords_(i,Z);this.currentPath_.push({type:"lineTo",x:j.x,y:j.y});this.currentX_=j.x;this.currentY_=j.y};M.bezierCurveTo=function(j,i,AI,AH,AG,AE){var Z=this.getCoords_(AG,AE);var AF=this.getCoords_(j,i);var m=this.getCoords_(AI,AH);e(this,AF,m,Z)};function e(Z,m,j,i){Z.currentPath_.push({type:"bezierCurveTo",cp1x:m.x,cp1y:m.y,cp2x:j.x,cp2y:j.y,x:i.x,y:i.y});Z.currentX_=i.x;Z.currentY_=i.y}M.quadraticCurveTo=function(AG,j,i,Z){var AF=this.getCoords_(AG,j);var AE=this.getCoords_(i,Z);var AH={x:this.currentX_+2/3*(AF.x-this.currentX_),y:this.currentY_+2/3*(AF.y-this.currentY_)};var m={x:AH.x+(AE.x-this.currentX_)/3,y:AH.y+(AE.y-this.currentY_)/3};e(this,AH,m,AE)};M.arc=function(AJ,AH,AI,AE,i,j){AI*=D;var AN=j?"at":"wa";var AK=AJ+U(AE)*AI-F;var AM=AH+J(AE)*AI-F;var Z=AJ+U(i)*AI-F;var AL=AH+J(i)*AI-F;if(AK==Z&&!j){AK+=0.125}var m=this.getCoords_(AJ,AH);var AG=this.getCoords_(AK,AM);var AF=this.getCoords_(Z,AL);this.currentPath_.push({type:AN,x:m.x,y:m.y,radius:AI,xStart:AG.x,yStart:AG.y,xEnd:AF.x,yEnd:AF.y})};M.rect=function(j,i,Z,m){this.moveTo(j,i);this.lineTo(j+Z,i);this.lineTo(j+Z,i+m);this.lineTo(j,i+m);this.closePath()};M.strokeRect=function(j,i,Z,m){var p=this.currentPath_;this.beginPath();this.moveTo(j,i);this.lineTo(j+Z,i);this.lineTo(j+Z,i+m);this.lineTo(j,i+m);this.closePath();this.stroke();this.currentPath_=p};M.fillRect=function(j,i,Z,m){var p=this.currentPath_;this.beginPath();this.moveTo(j,i);this.lineTo(j+Z,i);this.lineTo(j+Z,i+m);this.lineTo(j,i+m);this.closePath();this.fill();this.currentPath_=p};M.createLinearGradient=function(i,m,Z,j){var p=new v("gradient");p.x0_=i;p.y0_=m;p.x1_=Z;p.y1_=j;return p};M.createRadialGradient=function(m,AE,j,i,p,Z){var AF=new v("gradientradial");AF.x0_=m;AF.y0_=AE;AF.r0_=j;AF.x1_=i;AF.y1_=p;AF.r1_=Z;return AF};M.drawImage=function(AO,j){var AH,AF,AJ,AV,AM,AK,AQ,AX;var AI=AO.runtimeStyle.width;var AN=AO.runtimeStyle.height;AO.runtimeStyle.width="auto";AO.runtimeStyle.height="auto";var AG=AO.width;var AT=AO.height;AO.runtimeStyle.width=AI;AO.runtimeStyle.height=AN;if(arguments.length==3){AH=arguments[1];AF=arguments[2];AM=AK=0;AQ=AJ=AG;AX=AV=AT}else{if(arguments.length==5){AH=arguments[1];AF=arguments[2];AJ=arguments[3];AV=arguments[4];AM=AK=0;AQ=AG;AX=AT}else{if(arguments.length==9){AM=arguments[1];AK=arguments[2];AQ=arguments[3];AX=arguments[4];AH=arguments[5];AF=arguments[6];AJ=arguments[7];AV=arguments[8]}else{throw Error("Invalid number of arguments")}}}var AW=this.getCoords_(AH,AF);var m=AQ/2;var i=AX/2;var AU=[];var Z=10;var AE=10;AU.push(" <g_vml_:group",' coordsize="',D*Z,",",D*AE,'"',' coordorigin="0,0"',' style="width:',Z,"px;height:",AE,"px;position:absolute;");if(this.m_[0][0]!=1||this.m_[0][1]||this.m_[1][1]!=1||this.m_[1][0]){var p=[];p.push("M11=",this.m_[0][0],",","M12=",this.m_[1][0],",","M21=",this.m_[0][1],",","M22=",this.m_[1][1],",","Dx=",K(AW.x/D),",","Dy=",K(AW.y/D),"");var AS=AW;var AR=this.getCoords_(AH+AJ,AF);var AP=this.getCoords_(AH,AF+AV);var AL=this.getCoords_(AH+AJ,AF+AV);AS.x=z.max(AS.x,AR.x,AP.x,AL.x);AS.y=z.max(AS.y,AR.y,AP.y,AL.y);AU.push("padding:0 ",K(AS.x/D),"px ",K(AS.y/D),"px 0;filter:progid:DXImageTransform.Microsoft.Matrix(",p.join(""),", sizingmethod='clip');")}else{AU.push("top:",K(AW.y/D),"px;left:",K(AW.x/D),"px;")}AU.push(' ">','<g_vml_:image src="',AO.src,'"',' style="width:',D*AJ,"px;"," height:",D*AV,'px"',' cropleft="',AM/AG,'"',' croptop="',AK/AT,'"',' cropright="',(AG-AM-AQ)/AG,'"',' cropbottom="',(AT-AK-AX)/AT,'"'," />","</g_vml_:group>");this.element_.insertAdjacentHTML("BeforeEnd",AU.join(""))};M.stroke=function(AM){var m=10;var AN=10;var AE=5000;var AG={x:null,y:null};var AL={x:null,y:null};for(var AH=0;AH<this.currentPath_.length;AH+=AE){var AK=[];var AF=false;AK.push("<g_vml_:shape",' filled="',!!AM,'"',' style="position:absolute;width:',m,"px;height:",AN,'px;"',' coordorigin="0,0"',' coordsize="',D*m,",",D*AN,'"',' stroked="',!AM,'"',' path="');var AO=false;for(var AI=AH;AI<Math.min(AH+AE,this.currentPath_.length);AI++){if(AI%AE==0&&AI>0){AK.push(" m ",K(this.currentPath_[AI-1].x),",",K(this.currentPath_[AI-1].y))}var Z=this.currentPath_[AI];var AJ;switch(Z.type){case"moveTo":AJ=Z;AK.push(" m ",K(Z.x),",",K(Z.y));break;case"lineTo":AK.push(" l ",K(Z.x),",",K(Z.y));break;case"close":AK.push(" x ");Z=null;break;case"bezierCurveTo":AK.push(" c ",K(Z.cp1x),",",K(Z.cp1y),",",K(Z.cp2x),",",K(Z.cp2y),",",K(Z.x),",",K(Z.y));break;case"at":case"wa":AK.push(" ",Z.type," ",K(Z.x-this.arcScaleX_*Z.radius),",",K(Z.y-this.arcScaleY_*Z.radius)," ",K(Z.x+this.arcScaleX_*Z.radius),",",K(Z.y+this.arcScaleY_*Z.radius)," ",K(Z.xStart),",",K(Z.yStart)," ",K(Z.xEnd),",",K(Z.yEnd));break}if(Z){if(AG.x==null||Z.x<AG.x){AG.x=Z.x}if(AL.x==null||Z.x>AL.x){AL.x=Z.x}if(AG.y==null||Z.y<AG.y){AG.y=Z.y}if(AL.y==null||Z.y>AL.y){AL.y=Z.y}}}AK.push(' ">');if(!AM){R(this,AK)}else{a(this,AK,AG,AL)}AK.push("</g_vml_:shape>");this.element_.insertAdjacentHTML("beforeEnd",AK.join(""))}};function R(j,AE){var i=Y(j.strokeStyle);var m=i.color;var p=i.alpha*j.globalAlpha;var Z=j.lineScale_*j.lineWidth;if(Z<1){p*=Z}AE.push("<g_vml_:stroke",' opacity="',p,'"',' joinstyle="',j.lineJoin,'"',' miterlimit="',j.miterLimit,'"',' endcap="',t(j.lineCap),'"',' weight="',Z,'px"',' color="',m,'" />')}function a(AO,AG,Ah,AP){var AH=AO.fillStyle;var AY=AO.arcScaleX_;var AX=AO.arcScaleY_;var Z=AP.x-Ah.x;var m=AP.y-Ah.y;if(AH instanceof v){var AL=0;var Ac={x:0,y:0};var AU=0;var AK=1;if(AH.type_=="gradient"){var AJ=AH.x0_/AY;var j=AH.y0_/AX;var AI=AH.x1_/AY;var Aj=AH.y1_/AX;var Ag=AO.getCoords_(AJ,j);var Af=AO.getCoords_(AI,Aj);var AE=Af.x-Ag.x;var p=Af.y-Ag.y;AL=Math.atan2(AE,p)*180/Math.PI;if(AL<0){AL+=360}if(AL<0.000001){AL=0}}else{var Ag=AO.getCoords_(AH.x0_,AH.y0_);Ac={x:(Ag.x-Ah.x)/Z,y:(Ag.y-Ah.y)/m};Z/=AY*D;m/=AX*D;var Aa=z.max(Z,m);AU=2*AH.r0_/Aa;AK=2*AH.r1_/Aa-AU}var AS=AH.colors_;AS.sort(function(Ak,i){return Ak.offset-i.offset});var AN=AS.length;var AR=AS[0].color;var AQ=AS[AN-1].color;var AW=AS[0].alpha*AO.globalAlpha;var AV=AS[AN-1].alpha*AO.globalAlpha;var Ab=[];for(var Ae=0;Ae<AN;Ae++){var AM=AS[Ae];Ab.push(AM.offset*AK+AU+" "+AM.color)}AG.push('<g_vml_:fill type="',AH.type_,'"',' method="none" focus="100%"',' color="',AR,'"',' color2="',AQ,'"',' colors="',Ab.join(","),'"',' opacity="',AV,'"',' g_o_:opacity2="',AW,'"',' angle="',AL,'"',' focusposition="',Ac.x,",",Ac.y,'" />')}else{if(AH instanceof u){if(Z&&m){var AF=-Ah.x;var AZ=-Ah.y;AG.push("<g_vml_:fill",' position="',AF/Z*AY*AY,",",AZ/m*AX*AX,'"',' type="tile"',' src="',AH.src_,'" />')}}else{var Ai=Y(AO.fillStyle);var AT=Ai.color;var Ad=Ai.alpha*AO.globalAlpha;AG.push('<g_vml_:fill color="',AT,'" opacity="',Ad,'" />')}}}M.fill=function(){this.stroke(true)};M.closePath=function(){this.currentPath_.push({type:"close"})};M.getCoords_=function(j,i){var Z=this.m_;return{x:D*(j*Z[0][0]+i*Z[1][0]+Z[2][0])-F,y:D*(j*Z[0][1]+i*Z[1][1]+Z[2][1])-F}};M.save=function(){var Z={};Q(this,Z);this.aStack_.push(Z);this.mStack_.push(this.m_);this.m_=d(V(),this.m_)};M.restore=function(){if(this.aStack_.length){Q(this.aStack_.pop(),this);this.m_=this.mStack_.pop()}};function H(Z){return isFinite(Z[0][0])&&isFinite(Z[0][1])&&isFinite(Z[1][0])&&isFinite(Z[1][1])&&isFinite(Z[2][0])&&isFinite(Z[2][1])}function y(i,Z,j){if(!H(Z)){return }i.m_=Z;if(j){var p=Z[0][0]*Z[1][1]-Z[0][1]*Z[1][0];i.lineScale_=k(b(p))}}M.translate=function(j,i){var Z=[[1,0,0],[0,1,0],[j,i,1]];y(this,d(Z,this.m_),false)};M.rotate=function(i){var m=U(i);var j=J(i);var Z=[[m,j,0],[-j,m,0],[0,0,1]];y(this,d(Z,this.m_),false)};M.scale=function(j,i){this.arcScaleX_*=j;this.arcScaleY_*=i;var Z=[[j,0,0],[0,i,0],[0,0,1]];y(this,d(Z,this.m_),true)};M.transform=function(p,m,AF,AE,i,Z){var j=[[p,m,0],[AF,AE,0],[i,Z,1]];y(this,d(j,this.m_),true)};M.setTransform=function(AE,p,AG,AF,j,i){var Z=[[AE,p,0],[AG,AF,0],[j,i,1]];y(this,Z,true)};M.drawText_=function(AK,AI,AH,AN,AG){var AM=this.m_,AQ=1000,i=0,AP=AQ,AF={x:0,y:0},AE=[];var Z=P(X(this.font),this.element_);var j=AA(Z);var AR=this.element_.currentStyle;var p=this.textAlign.toLowerCase();switch(p){case"left":case"center":case"right":break;case"end":p=AR.direction=="ltr"?"right":"left";break;case"start":p=AR.direction=="rtl"?"right":"left";break;default:p="left"}switch(this.textBaseline){case"hanging":case"top":AF.y=Z.size/1.75;break;case"middle":break;default:case null:case"alphabetic":case"ideographic":case"bottom":AF.y=-Z.size/2.25;break}switch(p){case"right":i=AQ;AP=0.05;break;case"center":i=AP=AQ/2;break}var AO=this.getCoords_(AI+AF.x,AH+AF.y);AE.push('<g_vml_:line from="',-i,' 0" to="',AP,' 0.05" ',' coordsize="100 100" coordorigin="0 0"',' filled="',!AG,'" stroked="',!!AG,'" style="position:absolute;width:1px;height:1px;">');if(AG){R(this,AE)}else{a(this,AE,{x:-i,y:0},{x:AP,y:Z.size})}var AL=AM[0][0].toFixed(3)+","+AM[1][0].toFixed(3)+","+AM[0][1].toFixed(3)+","+AM[1][1].toFixed(3)+",0,0";var AJ=K(AO.x/D)+","+K(AO.y/D);AE.push('<g_vml_:skew on="t" matrix="',AL,'" ',' offset="',AJ,'" origin="',i,' 0" />','<g_vml_:path textpathok="true" />','<g_vml_:textpath on="true" string="',AD(AK),'" style="v-text-align:',p,";font:",AD(j),'" /></g_vml_:line>');this.element_.insertAdjacentHTML("beforeEnd",AE.join(""))};M.fillText=function(j,Z,m,i){this.drawText_(j,Z,m,i,false)};M.strokeText=function(j,Z,m,i){this.drawText_(j,Z,m,i,true)};M.measureText=function(j){if(!this.textMeasureEl_){var Z='<span style="position:absolute;top:-20000px;left:0;padding:0;margin:0;border:none;white-space:pre;"></span>';this.element_.insertAdjacentHTML("beforeEnd",Z);this.textMeasureEl_=this.element_.lastChild}var i=this.element_.ownerDocument;this.textMeasureEl_.innerHTML="";this.textMeasureEl_.style.font=this.font;this.textMeasureEl_.appendChild(i.createTextNode(j));return{width:this.textMeasureEl_.offsetWidth}};M.clip=function(){};M.arcTo=function(){};M.createPattern=function(i,Z){return new u(i,Z)};function v(Z){this.type_=Z;this.x0_=0;this.y0_=0;this.r0_=0;this.x1_=0;this.y1_=0;this.r1_=0;this.colors_=[]}v.prototype.addColorStop=function(i,Z){Z=Y(Z);this.colors_.push({offset:i,color:Z.color,alpha:Z.alpha})};function u(i,Z){q(i);switch(Z){case"repeat":case null:case"":this.repetition_="repeat";break;case"repeat-x":case"repeat-y":case"no-repeat":this.repetition_=Z;break;default:n("SYNTAX_ERR")}this.src_=i.src;this.width_=i.width;this.height_=i.height}function n(Z){throw new o(Z)}function q(Z){if(!Z||Z.nodeType!=1||Z.tagName!="IMG"){n("TYPE_MISMATCH_ERR")}if(Z.readyState!="complete"){n("INVALID_STATE_ERR")}}function o(Z){this.code=this[Z];this.message=Z+": DOM Exception "+this.code}var x=o.prototype=new Error;x.INDEX_SIZE_ERR=1;x.DOMSTRING_SIZE_ERR=2;x.HIERARCHY_REQUEST_ERR=3;x.WRONG_DOCUMENT_ERR=4;x.INVALID_CHARACTER_ERR=5;x.NO_DATA_ALLOWED_ERR=6;x.NO_MODIFICATION_ALLOWED_ERR=7;x.NOT_FOUND_ERR=8;x.NOT_SUPPORTED_ERR=9;x.INUSE_ATTRIBUTE_ERR=10;x.INVALID_STATE_ERR=11;x.SYNTAX_ERR=12;x.INVALID_MODIFICATION_ERR=13;x.NAMESPACE_ERR=14;x.INVALID_ACCESS_ERR=15;x.VALIDATION_ERR=16;x.TYPE_MISMATCH_ERR=17;G_vmlCanvasManager=E;CanvasRenderingContext2D=W;CanvasGradient=v;CanvasPattern=u;DOMException=o})()};
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.colorhelpers.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.colorhelpers.min.js
new file mode 100644
index 0000000..7f44c57
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.colorhelpers.min.js
@@ -0,0 +1 @@
+(function(b){b.color={};b.color.make=function(f,e,c,d){var h={};h.r=f||0;h.g=e||0;h.b=c||0;h.a=d!=null?d:1;h.add=function(k,j){for(var g=0;g<k.length;++g){h[k.charAt(g)]+=j}return h.normalize()};h.scale=function(k,j){for(var g=0;g<k.length;++g){h[k.charAt(g)]*=j}return h.normalize()};h.toString=function(){if(h.a>=1){return"rgb("+[h.r,h.g,h.b].join(",")+")"}else{return"rgba("+[h.r,h.g,h.b,h.a].join(",")+")"}};h.normalize=function(){function g(j,k,i){return k<j?j:(k>i?i:k)}h.r=g(0,parseInt(h.r),255);h.g=g(0,parseInt(h.g),255);h.b=g(0,parseInt(h.b),255);h.a=g(0,h.a,1);return h};h.clone=function(){return b.color.make(h.r,h.b,h.g,h.a)};return h.normalize()};b.color.extract=function(e,d){var f;do{f=e.css(d).toLowerCase();if(f!=""&&f!="transparent"){break}e=e.parent()}while(!b.nodeName(e.get(0),"body"));if(f=="rgba(0, 0, 0, 0)"){f="transparent"}return b.color.parse(f)};b.color.parse=function(f){var e,c=b.color.make;if(e=/rgb\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*\)/.exec(f)){return c(parseInt(e[1],10),parseInt(e[2],10),parseInt(e[3],10))}if(e=/rgba\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(f)){return c(parseInt(e[1],10),parseInt(e[2],10),parseInt(e[3],10),parseFloat(e[4]))}if(e=/rgb\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*\)/.exec(f)){return c(parseFloat(e[1])*2.55,parseFloat(e[2])*2.55,parseFloat(e[3])*2.55)}if(e=/rgba\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(f)){return c(parseFloat(e[1])*2.55,parseFloat(e[2])*2.55,parseFloat(e[3])*2.55,parseFloat(e[4]))}if(e=/#([a-fA-F0-9]{2})([a-fA-F0-9]{2})([a-fA-F0-9]{2})/.exec(f)){return c(parseInt(e[1],16),parseInt(e[2],16),parseInt(e[3],16))}if(e=/#([a-fA-F0-9])([a-fA-F0-9])([a-fA-F0-9])/.exec(f)){return c(parseInt(e[1]+e[1],16),parseInt(e[2]+e[2],16),parseInt(e[3]+e[3],16))}var d=b.trim(f).toLowerCase();if(d=="transparent"){return c(255,255,255,0)}else{e=a[d]||[0,0,0];return c(e[0],e[1],e[2])}};var a={aqua:[0,255,255],azure:[240,255,255],beige:[245,245,220],black:[0,0,0],blue:[0,0,255],brown:[165,42,42],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgrey:[169,169,169],darkgreen:[0,100,0],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkviolet:[148,0,211],fuchsia:[255,0,255],gold:[255,215,0],green:[0,128,0],indigo:[75,0,130],khaki:[240,230,140],lightblue:[173,216,230],lightcyan:[224,255,255],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightyellow:[255,255,224],lime:[0,255,0],magenta:[255,0,255],maroon:[128,0,0],navy:[0,0,128],olive:[128,128,0],orange:[255,165,0],pink:[255,192,203],purple:[128,0,128],violet:[128,0,128],red:[255,0,0],silver:[192,192,192],white:[255,255,255],yellow:[255,255,0]}})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.crosshair.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.crosshair.min.js
new file mode 100644
index 0000000..ccaf240
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.crosshair.min.js
@@ -0,0 +1 @@
+(function(b){var a={crosshair:{mode:null,color:"rgba(170, 0, 0, 0.80)",lineWidth:1}};function c(h){var j={x:-1,y:-1,locked:false};h.setCrosshair=function e(l){if(!l){j.x=-1}else{var k=h.p2c(l);j.x=Math.max(0,Math.min(k.left,h.width()));j.y=Math.max(0,Math.min(k.top,h.height()))}h.triggerRedrawOverlay()};h.clearCrosshair=h.setCrosshair;h.lockCrosshair=function f(k){if(k){h.setCrosshair(k)}j.locked=true};h.unlockCrosshair=function g(){j.locked=false};function d(k){if(j.locked){return}if(j.x!=-1){j.x=-1;h.triggerRedrawOverlay()}}function i(k){if(j.locked){return}if(h.getSelection&&h.getSelection()){j.x=-1;return}var l=h.offset();j.x=Math.max(0,Math.min(k.pageX-l.left,h.width()));j.y=Math.max(0,Math.min(k.pageY-l.top,h.height()));h.triggerRedrawOverlay()}h.hooks.bindEvents.push(function(l,k){if(!l.getOptions().crosshair.mode){return}k.mouseout(d);k.mousemove(i)});h.hooks.drawOverlay.push(function(m,k){var n=m.getOptions().crosshair;if(!n.mode){return}var l=m.getPlotOffset();k.save();k.translate(l.left,l.top);if(j.x!=-1){k.strokeStyle=n.color;k.lineWidth=n.lineWidth;k.lineJoin="round";k.beginPath();if(n.mode.indexOf("x")!=-1){k.moveTo(j.x,0);k.lineTo(j.x,m.height())}if(n.mode.indexOf("y")!=-1){k.moveTo(0,j.y);k.lineTo(m.width(),j.y)}k.stroke()}k.restore()});h.hooks.shutdown.push(function(l,k){k.unbind("mouseout",d);k.unbind("mousemove",i)})}b.plot.plugins.push({init:c,options:a,name:"crosshair",version:"1.0"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.fillbetween.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.fillbetween.min.js
new file mode 100644
index 0000000..47f3dfb
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.fillbetween.min.js
@@ -0,0 +1 @@
+(function(b){var a={series:{fillBetween:null}};function c(f){function d(j,h){var g;for(g=0;g<h.length;++g){if(h[g].id==j.fillBetween){return h[g]}}if(typeof j.fillBetween=="number"){g=j.fillBetween;if(g<0||g>=h.length){return null}return h[g]}return null}function e(B,u,g){if(u.fillBetween==null){return}var p=d(u,B.getData());if(!p){return}var y=g.pointsize,E=g.points,h=p.datapoints.pointsize,x=p.datapoints.points,r=[],w,v,k,G,F,q,t=u.lines.show,o=y>2&&g.format[2].y,n=t&&u.lines.steps,D=true,C=0,A=0,z;while(true){if(C>=E.length){break}z=r.length;if(E[C]==null){for(m=0;m<y;++m){r.push(E[C+m])}C+=y}else{if(A>=x.length){if(!t){for(m=0;m<y;++m){r.push(E[C+m])}}C+=y}else{if(x[A]==null){for(m=0;m<y;++m){r.push(null)}D=true;A+=h}else{w=E[C];v=E[C+1];G=x[A];F=x[A+1];q=0;if(w==G){for(m=0;m<y;++m){r.push(E[C+m])}q=F;C+=y;A+=h}else{if(w>G){if(t&&C>0&&E[C-y]!=null){k=v+(E[C-y+1]-v)*(G-w)/(E[C-y]-w);r.push(G);r.push(k);for(m=2;m<y;++m){r.push(E[C+m])}q=F}A+=h}else{if(D&&t){C+=y;continue}for(m=0;m<y;++m){r.push(E[C+m])}if(t&&A>0&&x[A-h]!=null){q=F+(x[A-h+1]-F)*(w-G)/(x[A-h]-G)}C+=y}}D=false;if(z!=r.length&&o){r[z+2]=q}}}}if(n&&z!=r.length&&z>0&&r[z]!=null&&r[z]!=r[z-y]&&r[z+1]!=r[z-y+1]){for(m=0;m<y;++m){r[z+y+m]=r[z+m]}r[z+1]=r[z-y+1]}}g.points=r}f.hooks.processDatapoints.push(e)}b.plot.plugins.push({init:c,options:a,name:"fillbetween",version:"1.0"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.image.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.image.min.js
new file mode 100644
index 0000000..9480c1e
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.image.min.js
@@ -0,0 +1 @@
+(function(c){var a={series:{images:{show:false,alpha:1,anchor:"corner"}}};c.plot.image={};c.plot.image.loadDataImages=function(g,f,k){var j=[],h=[];var i=f.series.images.show;c.each(g,function(l,m){if(!(i||m.images.show)){return}if(m.data){m=m.data}c.each(m,function(n,o){if(typeof o[0]=="string"){j.push(o[0]);h.push(o)}})});c.plot.image.load(j,function(l){c.each(h,function(n,o){var m=o[0];if(l[m]){o[0]=l[m]}});k()})};c.plot.image.load=function(h,i){var g=h.length,f={};if(g==0){i({})}c.each(h,function(k,j){var l=function(){--g;f[j]=this;if(g==0){i(f)}};c("<img />").load(l).error(l).attr("src",j)})};function d(q,o,l){var m=q.getPlotOffset();if(!l.images||!l.images.show){return}var r=l.datapoints.points,n=l.datapoints.pointsize;for(var t=0;t<r.length;t+=n){var y=r[t],w=r[t+1],g=r[t+2],v=r[t+3],f=r[t+4],h=l.xaxis,u=l.yaxis,x;if(!y||y.width<=0||y.height<=0){continue}if(w>v){x=v;v=w;w=x}if(g>f){x=f;f=g;g=x}if(l.images.anchor=="center"){x=0.5*(v-w)/(y.width-1);w-=x;v+=x;x=0.5*(f-g)/(y.height-1);g-=x;f+=x}if(w==v||g==f||w>=h.max||v<=h.min||g>=u.max||f<=u.min){continue}var k=0,s=0,j=y.width,p=y.height;if(w<h.min){k+=(j-k)*(h.min-w)/(v-w);w=h.min}if(v>h.max){j+=(j-k)*(h.max-v)/(v-w);v=h.max}if(g<u.min){p+=(s-p)*(u.min-g)/(f-g);g=u.min}if(f>u.max){s+=(s-p)*(u.max-f)/(f-g);f=u.max}w=h.p2c(w);v=h.p2c(v);g=u.p2c(g);f=u.p2c(f);if(w>v){x=v;v=w;w=x}if(g>f){x=f;f=g;g=x}x=o.globalAlpha;o.globalAlpha*=l.images.alpha;o.drawImage(y,k,s,j-k,p-s,w+m.left,g+m.top,v-w,f-g);o.globalAlpha=x}}function b(i,f,g,h){if(!f.images.show){return}h.format=[{required:true},{x:true,number:true,required:true},{y:true,number:true,required:true},{x:true,number:true,required:true},{y:true,number:true,required:true}]}function e(f){f.hooks.processRawData.push(b);f.hooks.drawSeries.push(d)}c.plot.plugins.push({init:e,options:a,name:"image",version:"1.1"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.min.js
new file mode 100644
index 0000000..4467fc5
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.min.js
@@ -0,0 +1,6 @@
+/* Javascript plotting library for jQuery, v. 0.7.
+ *
+ * Released under the MIT license by IOLA, December 2007.
+ *
+ */
+(function(b){b.color={};b.color.make=function(d,e,g,f){var c={};c.r=d||0;c.g=e||0;c.b=g||0;c.a=f!=null?f:1;c.add=function(h,j){for(var k=0;k<h.length;++k){c[h.charAt(k)]+=j}return c.normalize()};c.scale=function(h,j){for(var k=0;k<h.length;++k){c[h.charAt(k)]*=j}return c.normalize()};c.toString=function(){if(c.a>=1){return"rgb("+[c.r,c.g,c.b].join(",")+")"}else{return"rgba("+[c.r,c.g,c.b,c.a].join(",")+")"}};c.normalize=function(){function h(k,j,l){return j<k?k:(j>l?l:j)}c.r=h(0,parseInt(c.r),255);c.g=h(0,parseInt(c.g),255);c.b=h(0,parseInt(c.b),255);c.a=h(0,c.a,1);return c};c.clone=function(){return b.color.make(c.r,c.b,c.g,c.a)};return c.normalize()};b.color.extract=function(d,e){var c;do{c=d.css(e).toLowerCase();if(c!=""&&c!="transparent"){break}d=d.parent()}while(!b.nodeName(d.get(0),"body"));if(c=="rgba(0, 0, 0, 0)"){c="transparent"}return b.color.parse(c)};b.color.parse=function(c){var d,f=b.color.make;if(d=/rgb\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*\)/.exec(c)){return f(parseInt(d[1],10),parseInt(d[2],10),parseInt(d[3],10))}if(d=/rgba\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(c)){return f(parseInt(d[1],10),parseInt(d[2],10),parseInt(d[3],10),parseFloat(d[4]))}if(d=/rgb\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*\)/.exec(c)){return f(parseFloat(d[1])*2.55,parseFloat(d[2])*2.55,parseFloat(d[3])*2.55)}if(d=/rgba\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\s*\)/.exec(c)){return f(parseFloat(d[1])*2.55,parseFloat(d[2])*2.55,parseFloat(d[3])*2.55,parseFloat(d[4]))}if(d=/#([a-fA-F0-9]{2})([a-fA-F0-9]{2})([a-fA-F0-9]{2})/.exec(c)){return f(parseInt(d[1],16),parseInt(d[2],16),parseInt(d[3],16))}if(d=/#([a-fA-F0-9])([a-fA-F0-9])([a-fA-F0-9])/.exec(c)){return f(parseInt(d[1]+d[1],16),parseInt(d[2]+d[2],16),parseInt(d[3]+d[3],16))}var e=b.trim(c).toLowerCase();if(e=="transparent"){return f(255,255,255,0)}else{d=a[e]||[0,0,0];return f(d[0],d[1],d[2])}};var a={aqua:[0,255,255],azure:[240,255,255],beige:[245,245,220],black:[0,0,0],blue:[0,0,255],brown:[165,42,42],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgrey:[169,169,169],darkgreen:[0,100,0],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkviolet:[148,0,211],fuchsia:[255,0,255],gold:[255,215,0],green:[0,128,0],indigo:[75,0,130],khaki:[240,230,140],lightblue:[173,216,230],lightcyan:[224,255,255],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightyellow:[255,255,224],lime:[0,255,0],magenta:[255,0,255],maroon:[128,0,0],navy:[0,0,128],olive:[128,128,0],orange:[255,165,0],pink:[255,192,203],purple:[128,0,128],violet:[128,0,128],red:[255,0,0],silver:[192,192,192],white:[255,255,255],yellow:[255,255,0]}})(jQuery);(function(c){function b(av,ai,J,af){var Q=[],O={colors:["#edc240","#afd8f8","#cb4b4b","#4da74d","#9440ed"],legend:{show:true,noColumns:1,labelFormatter:null,labelBoxBorderColor:"#ccc",container:null,position:"ne",margin:5,backgroundColor:null,backgroundOpacity:0.85},xaxis:{show:null,position:"bottom",mode:null,color:null,tickColor:null,transform:null,inverseTransform:null,min:null,max:null,autoscaleMargin:null,ticks:null,tickFormatter:null,labelWidth:null,labelHeight:null,reserveSpace:null,tickLength:null,alignTicksWithAxis:null,tickDecimals:null,tickSize:null,minTickSize:null,monthNames:null,timeformat:null,twelveHourClock:false},yaxis:{autoscaleMargin:0.02,position:"left"},xaxes:[],yaxes:[],series:{points:{show:false,radius:3,lineWidth:2,fill:true,fillColor:"#ffffff",symbol:"circle"},lines:{lineWidth:2,fill:false,fillColor:null,steps:false},bars:{show:false,lineWidth:2,barWidth:1,fill:true,fillColor:null,align:"left",horizontal:false},shadowSize:3},grid:{show:true,aboveData:false,color:"#545454",backgroundColor:null,borderColor:null,tickColor:null,labelMargin:5,axisMargin:8,borderWidth:2,minBorderMargin:null,markings:null,markingsColor:"#f4f4f4",markingsLineWidth:2,clickable:false,hoverable:false,autoHighlight:true,mouseActiveRadius:10},hooks:{}},az=null,ad=null,y=null,H=null,A=null,p=[],aw=[],q={left:0,right:0,top:0,bottom:0},G=0,I=0,h=0,w=0,ak={processOptions:[],processRawData:[],processDatapoints:[],drawSeries:[],draw:[],bindEvents:[],drawOverlay:[],shutdown:[]},aq=this;aq.setData=aj;aq.setupGrid=t;aq.draw=W;aq.getPlaceholder=function(){return av};aq.getCanvas=function(){return az};aq.getPlotOffset=function(){return q};aq.width=function(){return h};aq.height=function(){return w};aq.offset=function(){var aB=y.offset();aB.left+=q.left;aB.top+=q.top;return aB};aq.getData=function(){return Q};aq.getAxes=function(){var aC={},aB;c.each(p.concat(aw),function(aD,aE){if(aE){aC[aE.direction+(aE.n!=1?aE.n:"")+"axis"]=aE}});return aC};aq.getXAxes=function(){return p};aq.getYAxes=function(){return aw};aq.c2p=C;aq.p2c=ar;aq.getOptions=function(){return O};aq.highlight=x;aq.unhighlight=T;aq.triggerRedrawOverlay=f;aq.pointOffset=function(aB){return{left:parseInt(p[aA(aB,"x")-1].p2c(+aB.x)+q.left),top:parseInt(aw[aA(aB,"y")-1].p2c(+aB.y)+q.top)}};aq.shutdown=ag;aq.resize=function(){B();g(az);g(ad)};aq.hooks=ak;F(aq);Z(J);X();aj(ai);t();W();ah();function an(aD,aB){aB=[aq].concat(aB);for(var aC=0;aC<aD.length;++aC){aD[aC].apply(this,aB)}}function F(){for(var aB=0;aB<af.length;++aB){var aC=af[aB];aC.init(aq);if(aC.options){c.extend(true,O,aC.options)}}}function Z(aC){var aB;c.extend(true,O,aC);if(O.xaxis.color==null){O.xaxis.color=O.grid.color}if(O.yaxis.color==null){O.yaxis.color=O.grid.color}if(O.xaxis.tickColor==null){O.xaxis.tickColor=O.grid.tickColor}if(O.yaxis.tickColor==null){O.yaxis.tickColor=O.grid.tickColor}if(O.grid.borderColor==null){O.grid.borderColor=O.grid.color}if(O.grid.tickColor==null){O.grid.tickColor=c.color.parse(O.grid.color).scale("a",0.22).toString()}for(aB=0;aB<Math.max(1,O.xaxes.length);++aB){O.xaxes[aB]=c.extend(true,{},O.xaxis,O.xaxes[aB])}for(aB=0;aB<Math.max(1,O.yaxes.length);++aB){O.yaxes[aB]=c.extend(true,{},O.yaxis,O.yaxes[aB])}if(O.xaxis.noTicks&&O.xaxis.ticks==null){O.xaxis.ticks=O.xaxis.noTicks}if(O.yaxis.noTicks&&O.yaxis.ticks==null){O.yaxis.ticks=O.yaxis.noTicks}if(O.x2axis){O.xaxes[1]=c.extend(true,{},O.xaxis,O.x2axis);O.xaxes[1].position="top"}if(O.y2axis){O.yaxes[1]=c.extend(true,{},O.yaxis,O.y2axis);O.yaxes[1].position="right"}if(O.grid.coloredAreas){O.grid.markings=O.grid.coloredAreas}if(O.grid.coloredAreasColor){O.grid.markingsColor=O.grid.coloredAreasColor}if(O.lines){c.extend(true,O.series.lines,O.lines)}if(O.points){c.extend(true,O.series.points,O.points)}if(O.bars){c.extend(true,O.series.bars,O.bars)}if(O.shadowSize!=null){O.series.shadowSize=O.shadowSize}for(aB=0;aB<O.xaxes.length;++aB){V(p,aB+1).options=O.xaxes[aB]}for(aB=0;aB<O.yaxes.length;++aB){V(aw,aB+1).options=O.yaxes[aB]}for(var aD in ak){if(O.hooks[aD]&&O.hooks[aD].length){ak[aD]=ak[aD].concat(O.hooks[aD])}}an(ak.processOptions,[O])}function aj(aB){Q=Y(aB);ax();z()}function Y(aE){var aC=[];for(var aB=0;aB<aE.length;++aB){var aD=c.extend(true,{},O.series);if(aE[aB].data!=null){aD.data=aE[aB].data;delete aE[aB].data;c.extend(true,aD,aE[aB]);aE[aB].data=aD.data}else{aD.data=aE[aB]}aC.push(aD)}return aC}function aA(aC,aD){var aB=aC[aD+"axis"];if(typeof aB=="object"){aB=aB.n}if(typeof aB!="number"){aB=1}return aB}function m(){return c.grep(p.concat(aw),function(aB){return aB})}function C(aE){var aC={},aB,aD;for(aB=0;aB<p.length;++aB){aD=p[aB];if(aD&&aD.used){aC["x"+aD.n]=aD.c2p(aE.left)}}for(aB=0;aB<aw.length;++aB){aD=aw[aB];if(aD&&aD.used){aC["y"+aD.n]=aD.c2p(aE.top)}}if(aC.x1!==undefined){aC.x=aC.x1}if(aC.y1!==undefined){aC.y=aC.y1}return aC}function ar(aF){var aD={},aC,aE,aB;for(aC=0;aC<p.length;++aC){aE=p[aC];if(aE&&aE.used){aB="x"+aE.n;if(aF[aB]==null&&aE.n==1){aB="x"}if(aF[aB]!=null){aD.left=aE.p2c(aF[aB]);break}}}for(aC=0;aC<aw.length;++aC){aE=aw[aC];if(aE&&aE.used){aB="y"+aE.n;if(aF[aB]==null&&aE.n==1){aB="y"}if(aF[aB]!=null){aD.top=aE.p2c(aF[aB]);break}}}return aD}function V(aC,aB){if(!aC[aB-1]){aC[aB-1]={n:aB,direction:aC==p?"x":"y",options:c.extend(true,{},aC==p?O.xaxis:O.yaxis)}}return aC[aB-1]}function ax(){var aG;var aM=Q.length,aB=[],aE=[];for(aG=0;aG<Q.length;++aG){var aJ=Q[aG].color;if(aJ!=null){--aM;if(typeof aJ=="number"){aE.push(aJ)}else{aB.push(c.color.parse(Q[aG].color))}}}for(aG=0;aG<aE.length;++aG){aM=Math.max(aM,aE[aG]+1)}var aC=[],aF=0;aG=0;while(aC.length<aM){var aI;if(O.colors.length==aG){aI=c.color.make(100,100,100)}else{aI=c.color.parse(O.colors[aG])}var aD=aF%2==1?-1:1;aI.scale("rgb",1+aD*Math.ceil(aF/2)*0.2);aC.push(aI);++aG;if(aG>=O.colors.length){aG=0;++aF}}var aH=0,aN;for(aG=0;aG<Q.length;++aG){aN=Q[aG];if(aN.color==null){aN.color=aC[aH].toString();++aH}else{if(typeof aN.color=="number"){aN.color=aC[aN.color].toString()}}if(aN.lines.show==null){var aL,aK=true;for(aL in aN){if(aN[aL]&&aN[aL].show){aK=false;break}}if(aK){aN.lines.show=true}}aN.xaxis=V(p,aA(aN,"x"));aN.yaxis=V(aw,aA(aN,"y"))}}function z(){var aO=Number.POSITIVE_INFINITY,aI=Number.NEGATIVE_INFINITY,aB=Number.MAX_VALUE,aU,aS,aR,aN,aD,aJ,aT,aP,aH,aG,aC,a0,aX,aL;function aF(a3,a2,a1){if(a2<a3.datamin&&a2!=-aB){a3.datamin=a2}if(a1>a3.datamax&&a1!=aB){a3.datamax=a1}}c.each(m(),function(a1,a2){a2.datamin=aO;a2.datamax=aI;a2.used=false});for(aU=0;aU<Q.length;++aU){aJ=Q[aU];aJ.datapoints={points:[]};an(ak.processRawData,[aJ,aJ.data,aJ.datapoints])}for(aU=0;aU<Q.length;++aU){aJ=Q[aU];var aZ=aJ.data,aW=aJ.datapoints.format;if(!aW){aW=[];aW.push({x:true,number:true,required:true});aW.push({y:true,number:true,required:true});if(aJ.bars.show||(aJ.lines.show&&aJ.lines.fill)){aW.push({y:true,number:true,required:false,defaultValue:0});if(aJ.bars.horizontal){delete aW[aW.length-1].y;aW[aW.length-1].x=true}}aJ.datapoints.format=aW}if(aJ.datapoints.pointsize!=null){continue}aJ.datapoints.pointsize=aW.length;aP=aJ.datapoints.pointsize;aT=aJ.datapoints.points;insertSteps=aJ.lines.show&&aJ.lines.steps;aJ.xaxis.used=aJ.yaxis.used=true;for(aS=aR=0;aS<aZ.length;++aS,aR+=aP){aL=aZ[aS];var aE=aL==null;if(!aE){for(aN=0;aN<aP;++aN){a0=aL[aN];aX=aW[aN];if(aX){if(aX.number&&a0!=null){a0=+a0;if(isNaN(a0)){a0=null}else{if(a0==Infinity){a0=aB}else{if(a0==-Infinity){a0=-aB}}}}if(a0==null){if(aX.required){aE=true}if(aX.defaultValue!=null){a0=aX.defaultValue}}}aT[aR+aN]=a0}}if(aE){for(aN=0;aN<aP;++aN){a0=aT[aR+aN];if(a0!=null){aX=aW[aN];if(aX.x){aF(aJ.xaxis,a0,a0)}if(aX.y){aF(aJ.yaxis,a0,a0)}}aT[aR+aN]=null}}else{if(insertSteps&&aR>0&&aT[aR-aP]!=null&&aT[aR-aP]!=aT[aR]&&aT[aR-aP+1]!=aT[aR+1]){for(aN=0;aN<aP;++aN){aT[aR+aP+aN]=aT[aR+aN]}aT[aR+1]=aT[aR-aP+1];aR+=aP}}}}for(aU=0;aU<Q.length;++aU){aJ=Q[aU];an(ak.processDatapoints,[aJ,aJ.datapoints])}for(aU=0;aU<Q.length;++aU){aJ=Q[aU];aT=aJ.datapoints.points,aP=aJ.datapoints.pointsize;var aK=aO,aQ=aO,aM=aI,aV=aI;for(aS=0;aS<aT.length;aS+=aP){if(aT[aS]==null){continue}for(aN=0;aN<aP;++aN){a0=aT[aS+aN];aX=aW[aN];if(!aX||a0==aB||a0==-aB){continue}if(aX.x){if(a0<aK){aK=a0}if(a0>aM){aM=a0}}if(aX.y){if(a0<aQ){aQ=a0}if(a0>aV){aV=a0}}}}if(aJ.bars.show){var aY=aJ.bars.align=="left"?0:-aJ.bars.barWidth/2;if(aJ.bars.horizontal){aQ+=aY;aV+=aY+aJ.bars.barWidth}else{aK+=aY;aM+=aY+aJ.bars.barWidth}}aF(aJ.xaxis,aK,aM);aF(aJ.yaxis,aQ,aV)}c.each(m(),function(a1,a2){if(a2.datamin==aO){a2.datamin=null}if(a2.datamax==aI){a2.datamax=null}})}function j(aB,aC){var aD=document.createElement("canvas");aD.className=aC;aD.width=G;aD.height=I;if(!aB){c(aD).css({position:"absolute",left:0,top:0})}c(aD).appendTo(av);if(!aD.getContext){aD=window.G_vmlCanvasManager.initElement(aD)}aD.getContext("2d").save();return aD}function B(){G=av.width();I=av.height();if(G<=0||I<=0){throw"Invalid dimensions for plot, width = "+G+", height = "+I}}function g(aC){if(aC.width!=G){aC.width=G}if(aC.height!=I){aC.height=I}var aB=aC.getContext("2d");aB.restore();aB.save()}function X(){var aC,aB=av.children("canvas.base"),aD=av.children("canvas.overlay");if(aB.length==0||aD==0){av.html("");av.css({padding:0});if(av.css("position")=="static"){av.css("position","relative")}B();az=j(true,"base");ad=j(false,"overlay");aC=false}else{az=aB.get(0);ad=aD.get(0);aC=true}H=az.getContext("2d");A=ad.getContext("2d");y=c([ad,az]);if(aC){av.data("plot").shutdown();aq.resize();A.clearRect(0,0,G,I);y.unbind();av.children().not([az,ad]).remove()}av.data("plot",aq)}function ah(){if(O.grid.hoverable){y.mousemove(aa);y.mouseleave(l)}if(O.grid.clickable){y.click(R)}an(ak.bindEvents,[y])}function ag(){if(M){clearTimeout(M)}y.unbind("mousemove",aa);y.unbind("mouseleave",l);y.unbind("click",R);an(ak.shutdown,[y])}function r(aG){function aC(aH){return aH}var aF,aB,aD=aG.options.transform||aC,aE=aG.options.inverseTransform;if(aG.direction=="x"){aF=aG.scale=h/Math.abs(aD(aG.max)-aD(aG.min));aB=Math.min(aD(aG.max),aD(aG.min))}else{aF=aG.scale=w/Math.abs(aD(aG.max)-aD(aG.min));aF=-aF;aB=Math.max(aD(aG.max),aD(aG.min))}if(aD==aC){aG.p2c=function(aH){return(aH-aB)*aF}}else{aG.p2c=function(aH){return(aD(aH)-aB)*aF}}if(!aE){aG.c2p=function(aH){return aB+aH/aF}}else{aG.c2p=function(aH){return aE(aB+aH/aF)}}}function L(aD){var aB=aD.options,aF,aJ=aD.ticks||[],aI=[],aE,aK=aB.labelWidth,aG=aB.labelHeight,aC;function aH(aM,aL){return c('<div style="position:absolute;top:-10000px;'+aL+'font-size:smaller"><div class="'+aD.direction+"Axis "+aD.direction+aD.n+'Axis">'+aM.join("")+"</div></div>").appendTo(av)}if(aD.direction=="x"){if(aK==null){aK=Math.floor(G/(aJ.length>0?aJ.length:1))}if(aG==null){aI=[];for(aF=0;aF<aJ.length;++aF){aE=aJ[aF].label;if(aE){aI.push('<div class="tickLabel" style="float:left;width:'+aK+'px">'+aE+"</div>")}}if(aI.length>0){aI.push('<div style="clear:left"></div>');aC=aH(aI,"width:10000px;");aG=aC.height();aC.remove()}}}else{if(aK==null||aG==null){for(aF=0;aF<aJ.length;++aF){aE=aJ[aF].label;if(aE){aI.push('<div class="tickLabel">'+aE+"</div>")}}if(aI.length>0){aC=aH(aI,"");if(aK==null){aK=aC.children().width()}if(aG==null){aG=aC.find("div.tickLabel").height()}aC.remove()}}}if(aK==null){aK=0}if(aG==null){aG=0}aD.labelWidth=aK;aD.labelHeight=aG}function au(aD){var aC=aD.labelWidth,aL=aD.labelHeight,aH=aD.options.position,aF=aD.options.tickLength,aG=O.grid.axisMargin,aJ=O.grid.labelMargin,aK=aD.direction=="x"?p:aw,aE;var aB=c.grep(aK,function(aN){return aN&&aN.options.position==aH&&aN.reserveSpace});if(c.inArray(aD,aB)==aB.length-1){aG=0}if(aF==null){aF="full"}var aI=c.grep(aK,function(aN){return aN&&aN.reserveSpace});var aM=c.inArray(aD,aI)==0;if(!aM&&aF=="full"){aF=5}if(!isNaN(+aF)){aJ+=+aF}if(aD.direction=="x"){aL+=aJ;if(aH=="bottom"){q.bottom+=aL+aG;aD.box={top:I-q.bottom,height:aL}}else{aD.box={top:q.top+aG,height:aL};q.top+=aL+aG}}else{aC+=aJ;if(aH=="left"){aD.box={left:q.left+aG,width:aC};q.left+=aC+aG}else{q.right+=aC+aG;aD.box={left:G-q.right,width:aC}}}aD.position=aH;aD.tickLength=aF;aD.box.padding=aJ;aD.innermost=aM}function U(aB){if(aB.direction=="x"){aB.box.left=q.left;aB.box.width=h}else{aB.box.top=q.top;aB.box.height=w}}function t(){var aC,aE=m();c.each(aE,function(aF,aG){aG.show=aG.options.show;if(aG.show==null){aG.show=aG.used}aG.reserveSpace=aG.show||aG.options.reserveSpace;n(aG)});allocatedAxes=c.grep(aE,function(aF){return aF.reserveSpace});q.left=q.right=q.top=q.bottom=0;if(O.grid.show){c.each(allocatedAxes,function(aF,aG){S(aG);P(aG);ap(aG,aG.ticks);L(aG)});for(aC=allocatedAxes.length-1;aC>=0;--aC){au(allocatedAxes[aC])}var aD=O.grid.minBorderMargin;if(aD==null){aD=0;for(aC=0;aC<Q.length;++aC){aD=Math.max(aD,Q[aC].points.radius+Q[aC].points.lineWidth/2)}}for(var aB in q){q[aB]+=O.grid.borderWidth;q[aB]=Math.max(aD,q[aB])}}h=G-q.left-q.right;w=I-q.bottom-q.top;c.each(aE,function(aF,aG){r(aG)});if(O.grid.show){c.each(allocatedAxes,function(aF,aG){U(aG)});k()}o()}function n(aE){var aF=aE.options,aD=+(aF.min!=null?aF.min:aE.datamin),aB=+(aF.max!=null?aF.max:aE.datamax),aH=aB-aD;if(aH==0){var aC=aB==0?1:0.01;if(aF.min==null){aD-=aC}if(aF.max==null||aF.min!=null){aB+=aC}}else{var aG=aF.autoscaleMargin;if(aG!=null){if(aF.min==null){aD-=aH*aG;if(aD<0&&aE.datamin!=null&&aE.datamin>=0){aD=0}}if(aF.max==null){aB+=aH*aG;if(aB>0&&aE.datamax!=null&&aE.datamax<=0){aB=0}}}}aE.min=aD;aE.max=aB}function S(aG){var aM=aG.options;var aH;if(typeof aM.ticks=="number"&&aM.ticks>0){aH=aM.ticks}else{aH=0.3*Math.sqrt(aG.direction=="x"?G:I)}var aT=(aG.max-aG.min)/aH,aO,aB,aN,aR,aS,aQ,aI;if(aM.mode=="time"){var aJ={second:1000,minute:60*1000,hour:60*60*1000,day:24*60*60*1000,month:30*24*60*60*1000,year:365.2425*24*60*60*1000};var aK=[[1,"second"],[2,"second"],[5,"second"],[10,"second"],[30,"second"],[1,"minute"],[2,"minute"],[5,"minute"],[10,"minute"],[30,"minute"],[1,"hour"],[2,"hour"],[4,"hour"],[8,"hour"],[12,"hour"],[1,"day"],[2,"day"],[3,"day"],[0.25,"month"],[0.5,"month"],[1,"month"],[2,"month"],[3,"month"],[6,"month"],[1,"year"]];var aC=0;if(aM.minTickSize!=null){if(typeof aM.tickSize=="number"){aC=aM.tickSize}else{aC=aM.minTickSize[0]*aJ[aM.minTickSize[1]]}}for(var aS=0;aS<aK.length-1;++aS){if(aT<(aK[aS][0]*aJ[aK[aS][1]]+aK[aS+1][0]*aJ[aK[aS+1][1]])/2&&aK[aS][0]*aJ[aK[aS][1]]>=aC){break}}aO=aK[aS][0];aN=aK[aS][1];if(aN=="year"){aQ=Math.pow(10,Math.floor(Math.log(aT/aJ.year)/Math.LN10));aI=(aT/aJ.year)/aQ;if(aI<1.5){aO=1}else{if(aI<3){aO=2}else{if(aI<7.5){aO=5}else{aO=10}}}aO*=aQ}aG.tickSize=aM.tickSize||[aO,aN];aB=function(aX){var a2=[],a0=aX.tickSize[0],a3=aX.tickSize[1],a1=new Date(aX.min);var aW=a0*aJ[a3];if(a3=="second"){a1.setUTCSeconds(a(a1.getUTCSeconds(),a0))}if(a3=="minute"){a1.setUTCMinutes(a(a1.getUTCMinutes(),a0))}if(a3=="hour"){a1.setUTCHours(a(a1.getUTCHours(),a0))}if(a3=="month"){a1.setUTCMonth(a(a1.getUTCMonth(),a0))}if(a3=="year"){a1.setUTCFullYear(a(a1.getUTCFullYear(),a0))}a1.setUTCMilliseconds(0);if(aW>=aJ.minute){a1.setUTCSeconds(0)}if(aW>=aJ.hour){a1.setUTCMinutes(0)}if(aW>=aJ.day){a1.setUTCHours(0)}if(aW>=aJ.day*4){a1.setUTCDate(1)}if(aW>=aJ.year){a1.setUTCMonth(0)}var a5=0,a4=Number.NaN,aY;do{aY=a4;a4=a1.getTime();a2.push(a4);if(a3=="month"){if(a0<1){a1.setUTCDate(1);var aV=a1.getTime();a1.setUTCMonth(a1.getUTCMonth()+1);var aZ=a1.getTime();a1.setTime(a4+a5*aJ.hour+(aZ-aV)*a0);a5=a1.getUTCHours();a1.setUTCHours(0)}else{a1.setUTCMonth(a1.getUTCMonth()+a0)}}else{if(a3=="year"){a1.setUTCFullYear(a1.getUTCFullYear()+a0)}else{a1.setTime(a4+aW)}}}while(a4<aX.max&&a4!=aY);return a2};aR=function(aV,aY){var a0=new Date(aV);if(aM.timeformat!=null){return c.plot.formatDate(a0,aM.timeformat,aM.monthNames)}var aW=aY.tickSize[0]*aJ[aY.tickSize[1]];var aX=aY.max-aY.min;var aZ=(aM.twelveHourClock)?" %p":"";if(aW<aJ.minute){fmt="%h:%M:%S"+aZ}else{if(aW<aJ.day){if(aX<2*aJ.day){fmt="%h:%M"+aZ}else{fmt="%b %d %h:%M"+aZ}}else{if(aW<aJ.month){fmt="%b %d"}else{if(aW<aJ.year){if(aX<aJ.year){fmt="%b"}else{fmt="%b %y"}}else{fmt="%y"}}}}return c.plot.formatDate(a0,fmt,aM.monthNames)}}else{var aU=aM.tickDecimals;var aP=-Math.floor(Math.log(aT)/Math.LN10);if(aU!=null&&aP>aU){aP=aU}aQ=Math.pow(10,-aP);aI=aT/aQ;if(aI<1.5){aO=1}else{if(aI<3){aO=2;if(aI>2.25&&(aU==null||aP+1<=aU)){aO=2.5;++aP}}else{if(aI<7.5){aO=5}else{aO=10}}}aO*=aQ;if(aM.minTickSize!=null&&aO<aM.minTickSize){aO=aM.minTickSize}aG.tickDecimals=Math.max(0,aU!=null?aU:aP);aG.tickSize=aM.tickSize||aO;aB=function(aX){var aZ=[];var a0=a(aX.min,aX.tickSize),aW=0,aV=Number.NaN,aY;do{aY=aV;aV=a0+aW*aX.tickSize;aZ.push(aV);++aW}while(aV<aX.max&&aV!=aY);return aZ};aR=function(aV,aW){return aV.toFixed(aW.tickDecimals)}}if(aM.alignTicksWithAxis!=null){var aF=(aG.direction=="x"?p:aw)[aM.alignTicksWithAxis-1];if(aF&&aF.used&&aF!=aG){var aL=aB(aG);if(aL.length>0){if(aM.min==null){aG.min=Math.min(aG.min,aL[0])}if(aM.max==null&&aL.length>1){aG.max=Math.max(aG.max,aL[aL.length-1])}}aB=function(aX){var aY=[],aV,aW;for(aW=0;aW<aF.ticks.length;++aW){aV=(aF.ticks[aW].v-aF.min)/(aF.max-aF.min);aV=aX.min+aV*(aX.max-aX.min);aY.push(aV)}return aY};if(aG.mode!="time"&&aM.tickDecimals==null){var aE=Math.max(0,-Math.floor(Math.log(aT)/Math.LN10)+1),aD=aB(aG);if(!(aD.length>1&&/\..*0$/.test((aD[1]-aD[0]).toFixed(aE)))){aG.tickDecimals=aE}}}}aG.tickGenerator=aB;if(c.isFunction(aM.tickFormatter)){aG.tickFormatter=function(aV,aW){return""+aM.tickFormatter(aV,aW)}}else{aG.tickFormatter=aR}}function P(aF){var aH=aF.options.ticks,aG=[];if(aH==null||(typeof aH=="number"&&aH>0)){aG=aF.tickGenerator(aF)}else{if(aH){if(c.isFunction(aH)){aG=aH({min:aF.min,max:aF.max})}else{aG=aH}}}var aE,aB;aF.ticks=[];for(aE=0;aE<aG.length;++aE){var aC=null;var aD=aG[aE];if(typeof aD=="object"){aB=+aD[0];if(aD.length>1){aC=aD[1]}}else{aB=+aD}if(aC==null){aC=aF.tickFormatter(aB,aF)}if(!isNaN(aB)){aF.ticks.push({v:aB,label:aC})}}}function ap(aB,aC){if(aB.options.autoscaleMargin&&aC.length>0){if(aB.options.min==null){aB.min=Math.min(aB.min,aC[0].v)}if(aB.options.max==null&&aC.length>1){aB.max=Math.max(aB.max,aC[aC.length-1].v)}}}function W(){H.clearRect(0,0,G,I);var aC=O.grid;if(aC.show&&aC.backgroundColor){N()}if(aC.show&&!aC.aboveData){ac()}for(var aB=0;aB<Q.length;++aB){an(ak.drawSeries,[H,Q[aB]]);d(Q[aB])}an(ak.draw,[H]);if(aC.show&&aC.aboveData){ac()}}function D(aB,aI){var aE,aH,aG,aD,aF=m();for(i=0;i<aF.length;++i){aE=aF[i];if(aE.direction==aI){aD=aI+aE.n+"axis";if(!aB[aD]&&aE.n==1){aD=aI+"axis"}if(aB[aD]){aH=aB[aD].from;aG=aB[aD].to;break}}}if(!aB[aD]){aE=aI=="x"?p[0]:aw[0];aH=aB[aI+"1"];aG=aB[aI+"2"]}if(aH!=null&&aG!=null&&aH>aG){var aC=aH;aH=aG;aG=aC}return{from:aH,to:aG,axis:aE}}function N(){H.save();H.translate(q.left,q.top);H.fillStyle=am(O.grid.backgroundColor,w,0,"rgba(255, 255, 255, 0)");H.fillRect(0,0,h,w);H.restore()}function ac(){var aF;H.save();H.translate(q.left,q.top);var aH=O.grid.markings;if(aH){if(c.isFunction(aH)){var aK=aq.getAxes();aK.xmin=aK.xaxis.min;aK.xmax=aK.xaxis.max;aK.ymin=aK.yaxis.min;aK.ymax=aK.yaxis.max;aH=aH(aK)}for(aF=0;aF<aH.length;++aF){var aD=aH[aF],aC=D(aD,"x"),aI=D(aD,"y");if(aC.from==null){aC.from=aC.axis.min}if(aC.to==null){aC.to=aC.axis.max}if(aI.from==null){aI.from=aI.axis.min}if(aI.to==null){aI.to=aI.axis.max}if(aC.to<aC.axis.min||aC.from>aC.axis.max||aI.to<aI.axis.min||aI.from>aI.axis.max){continue}aC.from=Math.max(aC.from,aC.axis.min);aC.to=Math.min(aC.to,aC.axis.max);aI.from=Math.max(aI.from,aI.axis.min);aI.to=Math.min(aI.to,aI.axis.max);if(aC.from==aC.to&&aI.from==aI.to){continue}aC.from=aC.axis.p2c(aC.from);aC.to=aC.axis.p2c(aC.to);aI.from=aI.axis.p2c(aI.from);aI.to=aI.axis.p2c(aI.to);if(aC.from==aC.to||aI.from==aI.to){H.beginPath();H.strokeStyle=aD.color||O.grid.markingsColor;H.lineWidth=aD.lineWidth||O.grid.markingsLineWidth;H.moveTo(aC.from,aI.from);H.lineTo(aC.to,aI.to);H.stroke()}else{H.fillStyle=aD.color||O.grid.markingsColor;H.fillRect(aC.from,aI.to,aC.to-aC.from,aI.from-aI.to)}}}var aK=m(),aM=O.grid.borderWidth;for(var aE=0;aE<aK.length;++aE){var aB=aK[aE],aG=aB.box,aQ=aB.tickLength,aN,aL,aP,aJ;if(!aB.show||aB.ticks.length==0){continue}H.strokeStyle=aB.options.tickColor||c.color.parse(aB.options.color).scale("a",0.22).toString();H.lineWidth=1;if(aB.direction=="x"){aN=0;if(aQ=="full"){aL=(aB.position=="top"?0:w)}else{aL=aG.top-q.top+(aB.position=="top"?aG.height:0)}}else{aL=0;if(aQ=="full"){aN=(aB.position=="left"?0:h)}else{aN=aG.left-q.left+(aB.position=="left"?aG.width:0)}}if(!aB.innermost){H.beginPath();aP=aJ=0;if(aB.direction=="x"){aP=h}else{aJ=w}if(H.lineWidth==1){aN=Math.floor(aN)+0.5;aL=Math.floor(aL)+0.5}H.moveTo(aN,aL);H.lineTo(aN+aP,aL+aJ);H.stroke()}H.beginPath();for(aF=0;aF<aB.ticks.length;++aF){var aO=aB.ticks[aF].v;aP=aJ=0;if(aO<aB.min||aO>aB.max||(aQ=="full"&&aM>0&&(aO==aB.min||aO==aB.max))){continue}if(aB.direction=="x"){aN=aB.p2c(aO);aJ=aQ=="full"?-w:aQ;if(aB.position=="top"){aJ=-aJ}}else{aL=aB.p2c(aO);aP=aQ=="full"?-h:aQ;if(aB.position=="left"){aP=-aP}}if(H.lineWidth==1){if(aB.direction=="x"){aN=Math.floor(aN)+0.5}else{aL=Math.floor(aL)+0.5}}H.moveTo(aN,aL);H.lineTo(aN+aP,aL+aJ)}H.stroke()}if(aM){H.lineWidth=aM;H.strokeStyle=O.grid.borderColor;H.strokeRect(-aM/2,-aM/2,h+aM,w+aM)}H.restore()}function k(){av.find(".tickLabels").remove();var aG=['<div class="tickLabels" style="font-size:smaller">'];var aJ=m();for(var aD=0;aD<aJ.length;++aD){var aC=aJ[aD],aF=aC.box;if(!aC.show){continue}aG.push('<div class="'+aC.direction+"Axis "+aC.direction+aC.n+'Axis" style="color:'+aC.options.color+'">');for(var aE=0;aE<aC.ticks.length;++aE){var aH=aC.ticks[aE];if(!aH.label||aH.v<aC.min||aH.v>aC.max){continue}var aK={},aI;if(aC.direction=="x"){aI="center";aK.left=Math.round(q.left+aC.p2c(aH.v)-aC.labelWidth/2);if(aC.position=="bottom"){aK.top=aF.top+aF.padding}else{aK.bottom=I-(aF.top+aF.height-aF.padding)}}else{aK.top=Math.round(q.top+aC.p2c(aH.v)-aC.labelHeight/2);if(aC.position=="left"){aK.right=G-(aF.left+aF.width-aF.padding);aI="right"}else{aK.left=aF.left+aF.padding;aI="left"}}aK.width=aC.labelWidth;var aB=["position:absolute","text-align:"+aI];for(var aL in aK){aB.push(aL+":"+aK[aL]+"px")}aG.push('<div class="tickLabel" style="'+aB.join(";")+'">'+aH.label+"</div>")}aG.push("</div>")}aG.push("</div>");av.append(aG.join(""))}function d(aB){if(aB.lines.show){at(aB)}if(aB.bars.show){e(aB)}if(aB.points.show){ao(aB)}}function at(aE){function aD(aP,aQ,aI,aU,aT){var aV=aP.points,aJ=aP.pointsize,aN=null,aM=null;H.beginPath();for(var aO=aJ;aO<aV.length;aO+=aJ){var aL=aV[aO-aJ],aS=aV[aO-aJ+1],aK=aV[aO],aR=aV[aO+1];if(aL==null||aK==null){continue}if(aS<=aR&&aS<aT.min){if(aR<aT.min){continue}aL=(aT.min-aS)/(aR-aS)*(aK-aL)+aL;aS=aT.min}else{if(aR<=aS&&aR<aT.min){if(aS<aT.min){continue}aK=(aT.min-aS)/(aR-aS)*(aK-aL)+aL;aR=aT.min}}if(aS>=aR&&aS>aT.max){if(aR>aT.max){continue}aL=(aT.max-aS)/(aR-aS)*(aK-aL)+aL;aS=aT.max}else{if(aR>=aS&&aR>aT.max){if(aS>aT.max){continue}aK=(aT.max-aS)/(aR-aS)*(aK-aL)+aL;aR=aT.max}}if(aL<=aK&&aL<aU.min){if(aK<aU.min){continue}aS=(aU.min-aL)/(aK-aL)*(aR-aS)+aS;aL=aU.min}else{if(aK<=aL&&aK<aU.min){if(aL<aU.min){continue}aR=(aU.min-aL)/(aK-aL)*(aR-aS)+aS;aK=aU.min}}if(aL>=aK&&aL>aU.max){if(aK>aU.max){continue}aS=(aU.max-aL)/(aK-aL)*(aR-aS)+aS;aL=aU.max}else{if(aK>=aL&&aK>aU.max){if(aL>aU.max){continue}aR=(aU.max-aL)/(aK-aL)*(aR-aS)+aS;aK=aU.max}}if(aL!=aN||aS!=aM){H.moveTo(aU.p2c(aL)+aQ,aT.p2c(aS)+aI)}aN=aK;aM=aR;H.lineTo(aU.p2c(aK)+aQ,aT.p2c(aR)+aI)}H.stroke()}function aF(aI,aQ,aP){var aW=aI.points,aV=aI.pointsize,aN=Math.min(Math.max(0,aP.min),aP.max),aX=0,aU,aT=false,aM=1,aL=0,aR=0;while(true){if(aV>0&&aX>aW.length+aV){break}aX+=aV;var aZ=aW[aX-aV],aK=aW[aX-aV+aM],aY=aW[aX],aJ=aW[aX+aM];if(aT){if(aV>0&&aZ!=null&&aY==null){aR=aX;aV=-aV;aM=2;continue}if(aV<0&&aX==aL+aV){H.fill();aT=false;aV=-aV;aM=1;aX=aL=aR+aV;continue}}if(aZ==null||aY==null){continue}if(aZ<=aY&&aZ<aQ.min){if(aY<aQ.min){continue}aK=(aQ.min-aZ)/(aY-aZ)*(aJ-aK)+aK;aZ=aQ.min}else{if(aY<=aZ&&aY<aQ.min){if(aZ<aQ.min){continue}aJ=(aQ.min-aZ)/(aY-aZ)*(aJ-aK)+aK;aY=aQ.min}}if(aZ>=aY&&aZ>aQ.max){if(aY>aQ.max){continue}aK=(aQ.max-aZ)/(aY-aZ)*(aJ-aK)+aK;aZ=aQ.max}else{if(aY>=aZ&&aY>aQ.max){if(aZ>aQ.max){continue}aJ=(aQ.max-aZ)/(aY-aZ)*(aJ-aK)+aK;aY=aQ.max}}if(!aT){H.beginPath();H.moveTo(aQ.p2c(aZ),aP.p2c(aN));aT=true}if(aK>=aP.max&&aJ>=aP.max){H.lineTo(aQ.p2c(aZ),aP.p2c(aP.max));H.lineTo(aQ.p2c(aY),aP.p2c(aP.max));continue}else{if(aK<=aP.min&&aJ<=aP.min){H.lineTo(aQ.p2c(aZ),aP.p2c(aP.min));H.lineTo(aQ.p2c(aY),aP.p2c(aP.min));continue}}var aO=aZ,aS=aY;if(aK<=aJ&&aK<aP.min&&aJ>=aP.min){aZ=(aP.min-aK)/(aJ-aK)*(aY-aZ)+aZ;aK=aP.min}else{if(aJ<=aK&&aJ<aP.min&&aK>=aP.min){aY=(aP.min-aK)/(aJ-aK)*(aY-aZ)+aZ;aJ=aP.min}}if(aK>=aJ&&aK>aP.max&&aJ<=aP.max){aZ=(aP.max-aK)/(aJ-aK)*(aY-aZ)+aZ;aK=aP.max}else{if(aJ>=aK&&aJ>aP.max&&aK<=aP.max){aY=(aP.max-aK)/(aJ-aK)*(aY-aZ)+aZ;aJ=aP.max}}if(aZ!=aO){H.lineTo(aQ.p2c(aO),aP.p2c(aK))}H.lineTo(aQ.p2c(aZ),aP.p2c(aK));H.lineTo(aQ.p2c(aY),aP.p2c(aJ));if(aY!=aS){H.lineTo(aQ.p2c(aY),aP.p2c(aJ));H.lineTo(aQ.p2c(aS),aP.p2c(aJ))}}}H.save();H.translate(q.left,q.top);H.lineJoin="round";var aG=aE.lines.lineWidth,aB=aE.shadowSize;if(aG>0&&aB>0){H.lineWidth=aB;H.strokeStyle="rgba(0,0,0,0.1)";var aH=Math.PI/18;aD(aE.datapoints,Math.sin(aH)*(aG/2+aB/2),Math.cos(aH)*(aG/2+aB/2),aE.xaxis,aE.yaxis);H.lineWidth=aB/2;aD(aE.datapoints,Math.sin(aH)*(aG/2+aB/4),Math.cos(aH)*(aG/2+aB/4),aE.xaxis,aE.yaxis)}H.lineWidth=aG;H.strokeStyle=aE.color;var aC=ae(aE.lines,aE.color,0,w);if(aC){H.fillStyle=aC;aF(aE.datapoints,aE.xaxis,aE.yaxis)}if(aG>0){aD(aE.datapoints,0,0,aE.xaxis,aE.yaxis)}H.restore()}function ao(aE){function aH(aN,aM,aU,aK,aS,aT,aQ,aJ){var aR=aN.points,aI=aN.pointsize;for(var aL=0;aL<aR.length;aL+=aI){var aP=aR[aL],aO=aR[aL+1];if(aP==null||aP<aT.min||aP>aT.max||aO<aQ.min||aO>aQ.max){continue}H.beginPath();aP=aT.p2c(aP);aO=aQ.p2c(aO)+aK;if(aJ=="circle"){H.arc(aP,aO,aM,0,aS?Math.PI:Math.PI*2,false)}else{aJ(H,aP,aO,aM,aS)}H.closePath();if(aU){H.fillStyle=aU;H.fill()}H.stroke()}}H.save();H.translate(q.left,q.top);var aG=aE.points.lineWidth,aC=aE.shadowSize,aB=aE.points.radius,aF=aE.points.symbol;if(aG>0&&aC>0){var aD=aC/2;H.lineWidth=aD;H.strokeStyle="rgba(0,0,0,0.1)";aH(aE.datapoints,aB,null,aD+aD/2,true,aE.xaxis,aE.yaxis,aF);H.strokeStyle="rgba(0,0,0,0.2)";aH(aE.datapoints,aB,null,aD/2,true,aE.xaxis,aE.yaxis,aF)}H.lineWidth=aG;H.strokeStyle=aE.color;aH(aE.datapoints,aB,ae(aE.points,aE.color),0,false,aE.xaxis,aE.yaxis,aF);H.restore()}function E(aN,aM,aV,aI,aQ,aF,aD,aL,aK,aU,aR,aC){var aE,aT,aJ,aP,aG,aB,aO,aH,aS;if(aR){aH=aB=aO=true;aG=false;aE=aV;aT=aN;aP=aM+aI;aJ=aM+aQ;if(aT<aE){aS=aT;aT=aE;aE=aS;aG=true;aB=false}}else{aG=aB=aO=true;aH=false;aE=aN+aI;aT=aN+aQ;aJ=aV;aP=aM;if(aP<aJ){aS=aP;aP=aJ;aJ=aS;aH=true;aO=false}}if(aT<aL.min||aE>aL.max||aP<aK.min||aJ>aK.max){return}if(aE<aL.min){aE=aL.min;aG=false}if(aT>aL.max){aT=aL.max;aB=false}if(aJ<aK.min){aJ=aK.min;aH=false}if(aP>aK.max){aP=aK.max;aO=false}aE=aL.p2c(aE);aJ=aK.p2c(aJ);aT=aL.p2c(aT);aP=aK.p2c(aP);if(aD){aU.beginPath();aU.moveTo(aE,aJ);aU.lineTo(aE,aP);aU.lineTo(aT,aP);aU.lineTo(aT,aJ);aU.fillStyle=aD(aJ,aP);aU.fill()}if(aC>0&&(aG||aB||aO||aH)){aU.beginPath();aU.moveTo(aE,aJ+aF);if(aG){aU.lineTo(aE,aP+aF)}else{aU.moveTo(aE,aP+aF)}if(aO){aU.lineTo(aT,aP+aF)}else{aU.moveTo(aT,aP+aF)}if(aB){aU.lineTo(aT,aJ+aF)}else{aU.moveTo(aT,aJ+aF)}if(aH){aU.lineTo(aE,aJ+aF)}else{aU.moveTo(aE,aJ+aF)}aU.stroke()}}function e(aD){function aC(aJ,aI,aL,aG,aK,aN,aM){var aO=aJ.points,aF=aJ.pointsize;for(var aH=0;aH<aO.length;aH+=aF){if(aO[aH]==null){continue}E(aO[aH],aO[aH+1],aO[aH+2],aI,aL,aG,aK,aN,aM,H,aD.bars.horizontal,aD.bars.lineWidth)}}H.save();H.translate(q.left,q.top);H.lineWidth=aD.bars.lineWidth;H.strokeStyle=aD.color;var aB=aD.bars.align=="left"?0:-aD.bars.barWidth/2;var aE=aD.bars.fill?function(aF,aG){return ae(aD.bars,aD.color,aF,aG)}:null;aC(aD.datapoints,aB,aB+aD.bars.barWidth,0,aE,aD.xaxis,aD.yaxis);H.restore()}function ae(aD,aB,aC,aF){var aE=aD.fill;if(!aE){return null}if(aD.fillColor){return am(aD.fillColor,aC,aF,aB)}var aG=c.color.parse(aB);aG.a=typeof aE=="number"?aE:0.4;aG.normalize();return aG.toString()}function o(){av.find(".legend").remove();if(!O.legend.show){return}var aH=[],aF=false,aN=O.legend.labelFormatter,aM,aJ;for(var aE=0;aE<Q.length;++aE){aM=Q[aE];aJ=aM.label;if(!aJ){continue}if(aE%O.legend.noColumns==0){if(aF){aH.push("</tr>")}aH.push("<tr>");aF=true}if(aN){aJ=aN(aJ,aM)}aH.push('<td class="legendColorBox"><div style="border:1px solid '+O.legend.labelBoxBorderColor+';padding:1px"><div style="width:4px;height:0;border:5px solid '+aM.color+';overflow:hidden"></div></div></td><td class="legendLabel">'+aJ+"</td>")}if(aF){aH.push("</tr>")}if(aH.length==0){return}var aL='<table style="font-size:smaller;color:'+O.grid.color+'">'+aH.join("")+"</table>";if(O.legend.container!=null){c(O.legend.container).html(aL)}else{var aI="",aC=O.legend.position,aD=O.legend.margin;if(aD[0]==null){aD=[aD,aD]}if(aC.charAt(0)=="n"){aI+="top:"+(aD[1]+q.top)+"px;"}else{if(aC.charAt(0)=="s"){aI+="bottom:"+(aD[1]+q.bottom)+"px;"}}if(aC.charAt(1)=="e"){aI+="right:"+(aD[0]+q.right)+"px;"}else{if(aC.charAt(1)=="w"){aI+="left:"+(aD[0]+q.left)+"px;"}}var aK=c('<div class="legend">'+aL.replace('style="','style="position:absolute;'+aI+";")+"</div>").appendTo(av);if(O.legend.backgroundOpacity!=0){var aG=O.legend.backgroundColor;if(aG==null){aG=O.grid.backgroundColor;if(aG&&typeof aG=="string"){aG=c.color.parse(aG)}else{aG=c.color.extract(aK,"background-color")}aG.a=1;aG=aG.toString()}var aB=aK.children();c('<div style="position:absolute;width:'+aB.width()+"px;height:"+aB.height()+"px;"+aI+"background-color:"+aG+';"> </div>').prependTo(aK).css("opacity",O.legend.backgroundOpacity)}}}var ab=[],M=null;function K(aI,aG,aD){var aO=O.grid.mouseActiveRadius,a0=aO*aO+1,aY=null,aR=false,aW,aU;for(aW=Q.length-1;aW>=0;--aW){if(!aD(Q[aW])){continue}var aP=Q[aW],aH=aP.xaxis,aF=aP.yaxis,aV=aP.datapoints.points,aT=aP.datapoints.pointsize,aQ=aH.c2p(aI),aN=aF.c2p(aG),aC=aO/aH.scale,aB=aO/aF.scale;if(aH.options.inverseTransform){aC=Number.MAX_VALUE}if(aF.options.inverseTransform){aB=Number.MAX_VALUE}if(aP.lines.show||aP.points.show){for(aU=0;aU<aV.length;aU+=aT){var aK=aV[aU],aJ=aV[aU+1];if(aK==null){continue}if(aK-aQ>aC||aK-aQ<-aC||aJ-aN>aB||aJ-aN<-aB){continue}var aM=Math.abs(aH.p2c(aK)-aI),aL=Math.abs(aF.p2c(aJ)-aG),aS=aM*aM+aL*aL;if(aS<a0){a0=aS;aY=[aW,aU/aT]}}}if(aP.bars.show&&!aY){var aE=aP.bars.align=="left"?0:-aP.bars.barWidth/2,aX=aE+aP.bars.barWidth;for(aU=0;aU<aV.length;aU+=aT){var aK=aV[aU],aJ=aV[aU+1],aZ=aV[aU+2];if(aK==null){continue}if(Q[aW].bars.horizontal?(aQ<=Math.max(aZ,aK)&&aQ>=Math.min(aZ,aK)&&aN>=aJ+aE&&aN<=aJ+aX):(aQ>=aK+aE&&aQ<=aK+aX&&aN>=Math.min(aZ,aJ)&&aN<=Math.max(aZ,aJ))){aY=[aW,aU/aT]}}}}if(aY){aW=aY[0];aU=aY[1];aT=Q[aW].datapoints.pointsize;return{datapoint:Q[aW].datapoints.points.slice(aU*aT,(aU+1)*aT),dataIndex:aU,series:Q[aW],seriesIndex:aW}}return null}function aa(aB){if(O.grid.hoverable){u("plothover",aB,function(aC){return aC.hoverable!=false})}}function l(aB){if(O.grid.hoverable){u("plothover",aB,function(aC){return false})}}function R(aB){u("plotclick",aB,function(aC){return aC.clickable!=false})}function u(aC,aB,aD){var aE=y.offset(),aH=aB.pageX-aE.left-q.left,aF=aB.pageY-aE.top-q.top,aJ=C({left:aH,top:aF});aJ.pageX=aB.pageX;aJ.pageY=aB.pageY;var aK=K(aH,aF,aD);if(aK){aK.pageX=parseInt(aK.series.xaxis.p2c(aK.datapoint[0])+aE.left+q.left);aK.pageY=parseInt(aK.series.yaxis.p2c(aK.datapoint[1])+aE.top+q.top)}if(O.grid.autoHighlight){for(var aG=0;aG<ab.length;++aG){var aI=ab[aG];if(aI.auto==aC&&!(aK&&aI.series==aK.series&&aI.point[0]==aK.datapoint[0]&&aI.point[1]==aK.datapoint[1])){T(aI.series,aI.point)}}if(aK){x(aK.series,aK.datapoint,aC)}}av.trigger(aC,[aJ,aK])}function f(){if(!M){M=setTimeout(s,30)}}function s(){M=null;A.save();A.clearRect(0,0,G,I);A.translate(q.left,q.top);var aC,aB;for(aC=0;aC<ab.length;++aC){aB=ab[aC];if(aB.series.bars.show){v(aB.series,aB.point)}else{ay(aB.series,aB.point)}}A.restore();an(ak.drawOverlay,[A])}function x(aD,aB,aF){if(typeof aD=="number"){aD=Q[aD]}if(typeof aB=="number"){var aE=aD.datapoints.pointsize;aB=aD.datapoints.points.slice(aE*aB,aE*(aB+1))}var aC=al(aD,aB);if(aC==-1){ab.push({series:aD,point:aB,auto:aF});f()}else{if(!aF){ab[aC].auto=false}}}function T(aD,aB){if(aD==null&&aB==null){ab=[];f()}if(typeof aD=="number"){aD=Q[aD]}if(typeof aB=="number"){aB=aD.data[aB]}var aC=al(aD,aB);if(aC!=-1){ab.splice(aC,1);f()}}function al(aD,aE){for(var aB=0;aB<ab.length;++aB){var aC=ab[aB];if(aC.series==aD&&aC.point[0]==aE[0]&&aC.point[1]==aE[1]){return aB}}return -1}function ay(aE,aD){var aC=aD[0],aI=aD[1],aH=aE.xaxis,aG=aE.yaxis;if(aC<aH.min||aC>aH.max||aI<aG.min||aI>aG.max){return}var aF=aE.points.radius+aE.points.lineWidth/2;A.lineWidth=aF;A.strokeStyle=c.color.parse(aE.color).scale("a",0.5).toString();var aB=1.5*aF,aC=aH.p2c(aC),aI=aG.p2c(aI);A.beginPath();if(aE.points.symbol=="circle"){A.arc(aC,aI,aB,0,2*Math.PI,false)}else{aE.points.symbol(A,aC,aI,aB,false)}A.closePath();A.stroke()}function v(aE,aB){A.lineWidth=aE.bars.lineWidth;A.strokeStyle=c.color.parse(aE.color).scale("a",0.5).toString();var aD=c.color.parse(aE.color).scale("a",0.5).toString();var aC=aE.bars.align=="left"?0:-aE.bars.barWidth/2;E(aB[0],aB[1],aB[2]||0,aC,aC+aE.bars.barWidth,0,function(){return aD},aE.xaxis,aE.yaxis,A,aE.bars.horizontal,aE.bars.lineWidth)}function am(aJ,aB,aH,aC){if(typeof aJ=="string"){return aJ}else{var aI=H.createLinearGradient(0,aH,0,aB);for(var aE=0,aD=aJ.colors.length;aE<aD;++aE){var aF=aJ.colors[aE];if(typeof aF!="string"){var aG=c.color.parse(aC);if(aF.brightness!=null){aG=aG.scale("rgb",aF.brightness)}if(aF.opacity!=null){aG.a*=aF.opacity}aF=aG.toString()}aI.addColorStop(aE/(aD-1),aF)}return aI}}}c.plot=function(g,e,d){var f=new b(c(g),e,d,c.plot.plugins);return f};c.plot.version="0.7";c.plot.plugins=[];c.plot.formatDate=function(l,f,h){var o=function(d){d=""+d;return d.length==1?"0"+d:d};var e=[];var p=false,j=false;var n=l.getUTCHours();var k=n<12;if(h==null){h=["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"]}if(f.search(/%p|%P/)!=-1){if(n>12){n=n-12}else{if(n==0){n=12}}}for(var g=0;g<f.length;++g){var m=f.charAt(g);if(p){switch(m){case"h":m=""+n;break;case"H":m=o(n);break;case"M":m=o(l.getUTCMinutes());break;case"S":m=o(l.getUTCSeconds());break;case"d":m=""+l.getUTCDate();break;case"m":m=""+(l.getUTCMonth()+1);break;case"y":m=""+l.getUTCFullYear();break;case"b":m=""+h[l.getUTCMonth()];break;case"p":m=(k)?("am"):("pm");break;case"P":m=(k)?("AM"):("PM");break;case"0":m="";j=true;break}if(m&&j){m=o(m);j=false}e.push(m);if(!j){p=false}}else{if(m=="%"){p=true}else{e.push(m)}}}return e.join("")};function a(e,d){return d*Math.floor(e/d)}})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.navigate.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.navigate.min.js
new file mode 100644
index 0000000..ecf63c9
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.navigate.min.js
@@ -0,0 +1 @@
+(function(i){i.fn.drag=function(j,k,l){if(k){this.bind("dragstart",j)}if(l){this.bind("dragend",l)}return !j?this.trigger("drag"):this.bind("drag",k?k:j)};var d=i.event,c=d.special,h=c.drag={not:":input",distance:0,which:1,dragging:false,setup:function(j){j=i.extend({distance:h.distance,which:h.which,not:h.not},j||{});j.distance=e(j.distance);d.add(this,"mousedown",f,j);if(this.attachEvent){this.attachEvent("ondragstart",a)}},teardown:function(){d.remove(this,"mousedown",f);if(this===h.dragging){h.dragging=h.proxy=false}g(this,true);if(this.detachEvent){this.detachEvent("ondragstart",a)}}};c.dragstart=c.dragend={setup:function(){},teardown:function(){}};function f(j){var k=this,l,m=j.data||{};if(m.elem){k=j.dragTarget=m.elem;j.dragProxy=h.proxy||k;j.cursorOffsetX=m.pageX-m.left;j.cursorOffsetY=m.pageY-m.top;j.offsetX=j.pageX-j.cursorOffsetX;j.offsetY=j.pageY-j.cursorOffsetY}else{if(h.dragging||(m.which>0&&j.which!=m.which)||i(j.target).is(m.not)){return}}switch(j.type){case"mousedown":i.extend(m,i(k).offset(),{elem:k,target:j.target,pageX:j.pageX,pageY:j.pageY});d.add(document,"mousemove mouseup",f,m);g(k,false);h.dragging=null;return false;case !h.dragging&&"mousemove":if(e(j.pageX-m.pageX)+e(j.pageY-m.pageY)<m.distance){break}j.target=m.target;l=b(j,"dragstart",k);if(l!==false){h.dragging=k;h.proxy=j.dragProxy=i(l||k)[0]}case"mousemove":if(h.dragging){l=b(j,"drag",k);if(c.drop){c.drop.allowed=(l!==false);c.drop.handler(j)}if(l!==false){break}j.type="mouseup"}case"mouseup":d.remove(document,"mousemove mouseup",f);if(h.dragging){if(c.drop){c.drop.handler(j)}b(j,"dragend",k)}g(k,true);h.dragging=h.proxy=m.elem=false;break}return true}function b(m,k,j){m.type=k;var l=i.event.handle.call(j,m);return l===false?false:l||m.result}function e(j){return Math.pow(j,2)}function a(){return(h.dragging===false)}function g(j,k){if(!j){return}j.unselectable=k?"off":"on";j.onselectstart=function(){return k};if(j.style){j.style.MozUserSelect=k?"":"none"}}})(jQuery);(function(f){var e=["DOMMouseScroll","mousewheel"];f.event.special.mousewheel={setup:function(){if(this.addEventListener){for(var a=e.length;a;){this.addEventListener(e[--a],d,false)}}else{this.onmousewheel=d}},teardown:function(){if(this.removeEventListener){for(var a=e.length;a;){this.removeEventListener(e[--a],d,false)}}else{this.onmousewheel=null}}};f.fn.extend({mousewheel:function(a){return a?this.bind("mousewheel",a):this.trigger("mousewheel")},unmousewheel:function(a){return this.unbind("mousewheel",a)}});function d(b){var h=[].slice.call(arguments,1),a=0,c=true;b=f.event.fix(b||window.event);b.type="mousewheel";if(b.wheelDelta){a=b.wheelDelta/120}if(b.detail){a=-b.detail/3}h.unshift(b,a);return f.event.handle.apply(this,h)}})(jQuery);(function(b){var a={xaxis:{zoomRange:null,panRange:null},zoom:{interactive:false,trigger:"dblclick",amount:1.5},pan:{interactive:false,cursor:"move",frameRate:20}};function c(o){function m(q,p){var r=o.offset();r.left=q.pageX-r.left;r.top=q.pageY-r.top;if(p){o.zoomOut({center:r})}else{o.zoom({center:r})}}function d(p,q){m(p,q<0);return false}var i="default",g=0,e=0,n=null;function f(p){if(p.which!=1){return false}var q=o.getPlaceholder().css("cursor");if(q){i=q}o.getPlaceholder().css("cursor",o.getOptions().pan.cursor);g=p.pageX;e=p.pageY}function j(q){var p=o.getOptions().pan.frameRate;if(n||!p){return}n=setTimeout(function(){o.pan({left:g-q.pageX,top:e-q.pageY});g=q.pageX;e=q.pageY;n=null},1/p*1000)}function h(p){if(n){clearTimeout(n);n=null}o.getPlaceholder().css("cursor",i);o.pan({left:g-p.pageX,top:e-p.pageY})}function l(q,p){var r=q.getOptions();if(r.zoom.interactive){p[r.zoom.trigger](m);p.mousewheel(d)}if(r.pan.interactive){p.bind("dragstart",{distance:10},f);p.bind("drag",j);p.bind("dragend",h)}}o.zoomOut=function(p){if(!p){p={}}if(!p.amount){p.amount=o.getOptions().zoom.amount}p.amount=1/p.amount;o.zoom(p)};o.zoom=function(q){if(!q){q={}}var x=q.center,r=q.amount||o.getOptions().zoom.amount,p=o.width(),t=o.height();if(!x){x={left:p/2,top:t/2}}var s=x.left/p,v=x.top/t,u={x:{min:x.left-s*p/r,max:x.left+(1-s)*p/r},y:{min:x.top-v*t/r,max:x.top+(1-v)*t/r}};b.each(o.getAxes(),function(z,C){var D=C.options,B=u[C.direction].min,w=u[C.direction].max,E=D.zoomRange;if(E===false){return}B=C.c2p(B);w=C.c2p(w);if(B>w){var A=B;B=w;w=A}var y=w-B;if(E&&((E[0]!=null&&y<E[0])||(E[1]!=null&&y>E[1]))){return}D.min=B;D.max=w});o.setupGrid();o.draw();if(!q.preventEvent){o.getPlaceholder().trigger("plotzoom",[o])}};o.pan=function(p){var q={x:+p.left,y:+p.top};if(isNaN(q.x)){q.x=0}if(isNaN(q.y)){q.y=0}b.each(o.getAxes(),function(s,u){var v=u.options,t,r,w=q[u.direction];t=u.c2p(u.p2c(u.min)+w),r=u.c2p(u.p2c(u.max)+w);var x=v.panRange;if(x===false){return}if(x){if(x[0]!=null&&x[0]>t){w=x[0]-t;t+=w;r+=w}if(x[1]!=null&&x[1]<r){w=x[1]-r;t+=w;r+=w}}v.min=t;v.max=r});o.setupGrid();o.draw();if(!p.preventEvent){o.getPlaceholder().trigger("plotpan",[o])}};function k(q,p){p.unbind(q.getOptions().zoom.trigger,m);p.unbind("mousewheel",d);p.unbind("dragstart",f);p.unbind("drag",j);p.unbind("dragend",h);if(n){clearTimeout(n)}}o.hooks.bindEvents.push(l);o.hooks.shutdown.push(k)}b.plot.plugins.push({init:c,options:a,name:"navigate",version:"1.3"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.pie.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.pie.min.js
new file mode 100644
index 0000000..b7bf870
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.pie.min.js
@@ -0,0 +1 @@
+(function(b){function c(D){var h=null;var L=null;var n=null;var B=null;var p=null;var M=0;var F=true;var o=10;var w=0.95;var A=0;var d=false;var z=false;var j=[];D.hooks.processOptions.push(g);D.hooks.bindEvents.push(e);function g(O,N){if(N.series.pie.show){N.grid.show=false;if(N.series.pie.label.show=="auto"){if(N.legend.show){N.series.pie.label.show=false}else{N.series.pie.label.show=true}}if(N.series.pie.radius=="auto"){if(N.series.pie.label.show){N.series.pie.radius=3/4}else{N.series.pie.radius=1}}if(N.series.pie.tilt>1){N.series.pie.tilt=1}if(N.series.pie.tilt<0){N.series.pie.tilt=0}O.hooks.processDatapoints.push(E);O.hooks.drawOverlay.push(H);O.hooks.draw.push(r)}}function e(P,N){var O=P.getOptions();if(O.series.pie.show&&O.grid.hoverable){N.unbind("mousemove").mousemove(t)}if(O.series.pie.show&&O.grid.clickable){N.unbind("click").click(l)}}function G(O){var P="";function N(S,T){if(!T){T=0}for(var R=0;R<S.length;++R){for(var Q=0;Q<T;Q++){P+="\t"}if(typeof S[R]=="object"){P+=""+R+":\n";N(S[R],T+1)}else{P+=""+R+": "+S[R]+"\n"}}}N(O);alert(P)}function q(P){for(var N=0;N<P.length;++N){var O=parseFloat(P[N].data[0][1]);if(O){M+=O}}}function E(Q,N,O,P){if(!d){d=true;h=Q.getCanvas();L=b(h).parent();a=Q.getOptions();Q.setData(K(Q.getData()))}}function I(){A=L.children().filter(".legend").children().width();n=Math.min(h.width,(h.height/a.series.pie.tilt))/2;p=(h.height/2)+a.series.pie.offset.top;B=(h.width/2);if(a.series.pie.offset.left=="auto"){if(a.legend.position.match("w")){B+=A/2}else{B-=A/2}}else{B+=a.series.pie.offset.left}if(B<n){B=n}else{if(B>h.width-n){B=h.width-n}}}function v(O){for(var N=0;N<O.length;++N){if(typeof(O[N].data)=="number"){O[N].data=[[1,O[N].data]]}else{if(typeof(O[N].data)=="undefined"||typeof(O[N].data[0])=="undefined"){if(typeof(O[N].data)!="undefined"&&typeof(O[N].data.label)!="undefined"){O[N].label=O[N].data.label}O[N].data=[[1,0]]}}}return O}function K(Q){Q=v(Q);q(Q);var P=0;var S=0;var N=a.series.pie.combine.color;var R=[];for(var O=0;O<Q.length;++O){Q[O].data[0][1]=parseFloat(Q[O].data[0][1]);if(!Q[O].data[0][1]){Q[O].data[0][1]=0}if(Q[O].data[0][1]/M<=a.series.pie.combine.threshold){P+=Q[O].data[0][1];S++;if(!N){N=Q[O].color}}else{R.push({data:[[1,Q[O].data[0][1]]],color:Q[O].color,label:Q[O].label,angle:(Q[O].data[0][1]*(Math.PI*2))/M,percent:(Q[O].data[0][1]/M*100)})}}if(S>0){R.push({data:[[1,P]],color:N,label:a.series.pie.combine.label,angle:(P*(Math.PI*2))/M,percent:(P/M*100)})}return R}function r(S,Q){if(!L){return}ctx=Q;I();var T=S.getData();var P=0;while(F&&P<o){F=false;if(P>0){n*=w}P+=1;N();if(a.series.pie.tilt<=0.8){O()}R()}if(P>=o){N();L.prepend('<div class="error">Could not draw pie with labels contained inside canvas</div>')}if(S.setSeries&&S.insertLegend){S.setSeries(T);S.insertLegend()}function N(){ctx.clearRect(0,0,h.width,h.height);L.children().filter(".pieLabel, .pieLabelBackground").remove()}function O(){var Z=5;var Y=15;var W=10;var X=0.02;if(a.series.pie.radius>1){var U=a.series.pie.radius}else{var U=n*a.series.pie.radius}if(U>=(h.width/2)-Z||U*a.series.pie.tilt>=(h.height/2)-Y||U<=W){return}ctx.save();ctx.translate(Z,Y);ctx.globalAlpha=X;ctx.fillStyle="#000";ctx.translate(B,p);ctx.scale(1,a.series.pie.tilt);for(var V=1;V<=W;V++){ctx.beginPath();ctx.arc(0,0,U,0,Math.PI*2,false);ctx.fill();U-=V}ctx.restore()}function R(){startAngle=Math.PI*a.series.pie.startAngle;if(a.series.pie.radius>1){var U=a.series.pie.radius}else{var U=n*a.series.pie.radius}ctx.save();ctx.translate(B,p);ctx.scale(1,a.series.pie.tilt);ctx.save();var Y=startAngle;for(var W=0;W<T.length;++W){T[W].startAngle=Y;X(T[W].angle,T[W].color,true)}ctx.restore();ctx.save();ctx.lineWidth=a.series.pie.stroke.width;Y=startAngle;for(var W=0;W<T.length;++W){X(T[W].angle,a.series.pie.stroke.color,false)}ctx.restore();J(ctx);if(a.series.pie.label.show){V()}ctx.restore();function X(ab,Z,aa){if(ab<=0){return}if(aa){ctx.fillStyle=Z}else{ctx.strokeStyle=Z;ctx.lineJoin="round"}ctx.beginPath();if(Math.abs(ab-Math.PI*2)>1e-9){ctx.moveTo(0,0)}else{if(b.browser.msie){ab-=0.0001}}ctx.arc(0,0,U,Y,Y+ab,false);ctx.closePath();Y+=ab;if(aa){ctx.fill()}else{ctx.stroke()}}function V(){var ac=startAngle;if(a.series.pie.label.radius>1){var Z=a.series.pie.label.radius}else{var Z=n*a.series.pie.label.radius}for(var ab=0;ab<T.length;++ab){if(T[ab].percent>=a.series.pie.label.threshold*100){aa(T[ab],ac,ab)}ac+=T[ab].angle}function aa(ap,ai,ag){if(ap.data[0][1]==0){return}var ar=a.legend.labelFormatter,aq,ae=a.series.pie.label.formatter;if(ar){aq=ar(ap.label,ap)}else{aq=ap.label}if(ae){aq=ae(aq,ap)}var aj=((ai+ap.angle)+ai)/2;var ao=B+Math.round(Math.cos(aj)*Z);var am=p+Math.round(Math.sin(aj)*Z)*a.series.pie.tilt;var af='<span class="pieLabel" id="pieLabel'+ag+'" style="position:absolute;top:'+am+"px;left:"+ao+'px;">'+aq+"</span>";L.append(af);var an=L.children("#pieLabel"+ag);var ad=(am-an.height()/2);var ah=(ao-an.width()/2);an.css("top",ad);an.css("left",ah);if(0-ad>0||0-ah>0||h.height-(ad+an.height())<0||h.width-(ah+an.width())<0){F=true}if(a.series.pie.label.background.opacity!=0){var ak=a.series.pie.label.background.color;if(ak==null){ak=ap.color}var al="top:"+ad+"px;left:"+ah+"px;";b('<div class="pieLabelBackground" style="position:absolute;width:'+an.width()+"px;height:"+an.height()+"px;"+al+"background-color:"+ak+';"> </div>').insertBefore(an).css("opacity",a.series.pie.label.background.opacity)}}}}}function J(N){if(a.series.pie.innerRadius>0){N.save();innerRadius=a.series.pie.innerRadius>1?a.series.pie.innerRadius:n*a.series.pie.innerRadius;N.globalCompositeOperation="destination-out";N.beginPath();N.fillStyle=a.series.pie.stroke.color;N.arc(0,0,innerRadius,0,Math.PI*2,false);N.fill();N.closePath();N.restore();N.save();N.beginPath();N.strokeStyle=a.series.pie.stroke.color;N.arc(0,0,innerRadius,0,Math.PI*2,false);N.stroke();N.closePath();N.restore()}}function s(Q,R){for(var S=false,P=-1,N=Q.length,O=N-1;++P<N;O=P){((Q[P][1]<=R[1]&&R[1]<Q[O][1])||(Q[O][1]<=R[1]&&R[1]<Q[P][1]))&&(R[0]<(Q[O][0]-Q[P][0])*(R[1]-Q[P][1])/(Q[O][1]-Q[P][1])+Q[P][0])&&(S=!S)}return S}function u(R,P){var T=D.getData(),O=D.getOptions(),N=O.series.pie.radius>1?O.series.pie.radius:n*O.series.pie.radius;for(var Q=0;Q<T.length;++Q){var S=T[Q];if(S.pie.show){ctx.save();ctx.beginPath();ctx.moveTo(0,0);ctx.arc(0,0,N,S.startAngle,S.startAngle+S.angle,false);ctx.closePath();x=R-B;y=P-p;if(ctx.isPointInPath){if(ctx.isPointInPath(R-B,P-p)){ctx.restore();return{datapoint:[S.percent,S.data],dataIndex:0,series:S,seriesIndex:Q}}}else{p1X=(N*Math.cos(S.startAngle));p1Y=(N*Math.sin(S.startAngle));p2X=(N*Math.cos(S.startAngle+(S.angle/4)));p2Y=(N*Math.sin(S.startAngle+(S.angle/4)));p3X=(N*Math.cos(S.startAngle+(S.angle/2)));p3Y=(N*Math.sin(S.startAngle+(S.angle/2)));p4X=(N*Math.cos(S.startAngle+(S.angle/1.5)));p4Y=(N*Math.sin(S.startAngle+(S.angle/1.5)));p5X=(N*Math.cos(S.startAngle+S.angle));p5Y=(N*Math.sin(S.startAngle+S.angle));arrPoly=[[0,0],[p1X,p1Y],[p2X,p2Y],[p3X,p3Y],[p4X,p4Y],[p5X,p5Y]];arrPoint=[x,y];if(s(arrPoly,arrPoint)){ctx.restore();return{datapoint:[S.percent,S.data],dataIndex:0,series:S,seriesIndex:Q}}}ctx.restore()}}return null}function t(N){m("plothover",N)}function l(N){m("plotclick",N)}function m(N,T){var O=D.offset(),R=parseInt(T.pageX-O.left),P=parseInt(T.pageY-O.top),V=u(R,P);if(a.grid.autoHighlight){for(var Q=0;Q<j.length;++Q){var S=j[Q];if(S.auto==N&&!(V&&S.series==V.series)){f(S.series)}}}if(V){k(V.series,N)}var U={pageX:T.pageX,pageY:T.pageY};L.trigger(N,[U,V])}function k(O,P){if(typeof O=="number"){O=series[O]}var N=C(O);if(N==-1){j.push({series:O,auto:P});D.triggerRedrawOverlay()}else{if(!P){j[N].auto=false}}}function f(O){if(O==null){j=[];D.triggerRedrawOverlay()}if(typeof O=="number"){O=series[O]}var N=C(O);if(N!=-1){j.splice(N,1);D.triggerRedrawOverlay()}}function C(P){for(var N=0;N<j.length;++N){var O=j[N];if(O.series==P){return N}}return -1}function H(Q,R){var P=Q.getOptions();var N=P.series.pie.radius>1?P.series.pie.radius:n*P.series.pie.radius;R.save();R.translate(B,p);R.scale(1,P.series.pie.tilt);for(i=0;i<j.length;++i){O(j[i].series)}J(R);R.restore();function O(S){if(S.angle<0){return}R.fillStyle="rgba(255, 255, 255, "+P.series.pie.highlight.opacity+")";R.beginPath();if(Math.abs(S.angle-Math.PI*2)>1e-9){R.moveTo(0,0)}R.arc(0,0,N,S.startAngle,S.startAngle+S.angle,false);R.closePath();R.fill()}}}var a={series:{pie:{show:false,radius:"auto",innerRadius:0,startAngle:3/2,tilt:1,offset:{top:0,left:"auto"},stroke:{color:"#FFF",width:1},label:{show:"auto",formatter:function(d,e){return'<div style="font-size:x-small;text-align:center;padding:2px;color:'+e.color+';">'+d+"<br/>"+Math.round(e.percent)+"%</div>"},radius:1,background:{color:null,opacity:0},threshold:0},combine:{threshold:-1,color:null,label:"Other"},highlight:{opacity:0.5}}}};b.plot.plugins.push({init:c,options:a,name:"pie",version:"1.0"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.resize.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.resize.min.js
new file mode 100644
index 0000000..1fa0771
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.resize.min.js
@@ -0,0 +1 @@
+(function(n,p,u){var w=n([]),s=n.resize=n.extend(n.resize,{}),o,l="setTimeout",m="resize",t=m+"-special-event",v="delay",r="throttleWindow";s[v]=250;s[r]=true;n.event.special[m]={setup:function(){if(!s[r]&&this[l]){return false}var a=n(this);w=w.add(a);n.data(this,t,{w:a.width(),h:a.height()});if(w.length===1){q()}},teardown:function(){if(!s[r]&&this[l]){return false}var a=n(this);w=w.not(a);a.removeData(t);if(!w.length){clearTimeout(o)}},add:function(b){if(!s[r]&&this[l]){return false}var c;function a(d,h,g){var f=n(this),e=n.data(this,t);e.w=h!==u?h:f.width();e.h=g!==u?g:f.height();c.apply(this,arguments)}if(n.isFunction(b)){c=b;return a}else{c=b.handler;b.handler=a}}};function q(){o=p[l](function(){w.each(function(){var d=n(this),a=d.width(),b=d.height(),c=n.data(this,t);if(a!==c.w||b!==c.h){d.trigger(m,[c.w=a,c.h=b])}});q()},s[v])}})(jQuery,this);(function(b){var a={};function c(f){function e(){var h=f.getPlaceholder();if(h.width()==0||h.height()==0){return}f.resize();f.setupGrid();f.draw()}function g(i,h){i.getPlaceholder().resize(e)}function d(i,h){i.getPlaceholder().unbind("resize",e)}f.hooks.bindEvents.push(g);f.hooks.shutdown.push(d)}b.plot.plugins.push({init:c,options:a,name:"resize",version:"1.0"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.selection.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.selection.min.js
new file mode 100644
index 0000000..badc005
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.selection.min.js
@@ -0,0 +1 @@
+(function(a){function b(k){var p={first:{x:-1,y:-1},second:{x:-1,y:-1},show:false,active:false};var m={};var r=null;function e(s){if(p.active){l(s);k.getPlaceholder().trigger("plotselecting",[g()])}}function n(s){if(s.which!=1){return}document.body.focus();if(document.onselectstart!==undefined&&m.onselectstart==null){m.onselectstart=document.onselectstart;document.onselectstart=function(){return false}}if(document.ondrag!==undefined&&m.ondrag==null){m.ondrag=document.ondrag;document.ondrag=function(){return false}}d(p.first,s);p.active=true;r=function(t){j(t)};a(document).one("mouseup",r)}function j(s){r=null;if(document.onselectstart!==undefined){document.onselectstart=m.onselectstart}if(document.ondrag!==undefined){document.ondrag=m.ondrag}p.active=false;l(s);if(f()){i()}else{k.getPlaceholder().trigger("plotunselected",[]);k.getPlaceholder().trigger("plotselecting",[null])}return false}function g(){if(!f()){return null}var u={},t=p.first,s=p.second;a.each(k.getAxes(),function(v,w){if(w.used){var y=w.c2p(t[w.direction]),x=w.c2p(s[w.direction]);u[v]={from:Math.min(y,x),to:Math.max(y,x)}}});return u}function i(){var s=g();k.getPlaceholder().trigger("plotselected",[s]);if(s.xaxis&&s.yaxis){k.getPlaceholder().trigger("selected",[{x1:s.xaxis.from,y1:s.yaxis.from,x2:s.xaxis.to,y2:s.yaxis.to}])}}function h(t,u,s){return u<t?t:(u>s?s:u)}function d(w,t){var v=k.getOptions();var u=k.getPlaceholder().offset();var s=k.getPlotOffset();w.x=h(0,t.pageX-u.left-s.left,k.width());w.y=h(0,t.pageY-u.top-s.top,k.height());if(v.selection.mode=="y"){w.x=w==p.first?0:k.width()}if(v.selection.mode=="x"){w.y=w==p.first?0:k.height()}}function l(s){if(s.pageX==null){return}d(p.second,s);if(f()){p.show=true;k.triggerRedrawOverlay()}else{q(true)}}function q(s){if(p.show){p.show=false;k.triggerRedrawOverlay();if(!s){k.getPlaceholder().trigger("plotunselected",[])}}}function c(s,w){var t,y,z,A,x=k.getAxes();for(var u in x){t=x[u];if(t.direction==w){A=w+t.n+"axis";if(!s[A]&&t.n==1){A=w+"axis"}if(s[A]){y=s[A].from;z=s[A].to;break}}}if(!s[A]){t=w=="x"?k.getXAxes()[0]:k.getYAxes()[0];y=s[w+"1"];z=s[w+"2"]}if(y!=null&&z!=null&&y>z){var v=y;y=z;z=v}return{from:y,to:z,axis:t}}function o(t,s){var v,u,w=k.getOptions();if(w.selection.mode=="y"){p.first.x=0;p.second.x=k.width()}else{u=c(t,"x");p.first.x=u.axis.p2c(u.from);p.second.x=u.axis.p2c(u.to)}if(w.selection.mode=="x"){p.first.y=0;p.second.y=k.height()}else{u=c(t,"y");p.first.y=u.axis.p2c(u.from);p.second.y=u.axis.p2c(u.to)}p.show=true;k.triggerRedrawOverlay();if(!s&&f()){i()}}function f(){var s=5;return Math.abs(p.second.x-p.first.x)>=s&&Math.abs(p.second.y-p.first.y)>=s}k.clearSelection=q;k.setSelection=o;k.getSelection=g;k.hooks.bindEvents.push(function(t,s){var u=t.getOptions();if(u.selection.mode!=null){s.mousemove(e);s.mousedown(n)}});k.hooks.drawOverlay.push(function(v,D){if(p.show&&f()){var t=v.getPlotOffset();var s=v.getOptions();D.save();D.translate(t.left,t.top);var z=a.color.parse(s.selection.color);D.strokeStyle=z.scale("a",0.8).toString();D.lineWidth=1;D.lineJoin="round";D.fillStyle=z.scale("a",0.4).toString();var B=Math.min(p.first.x,p.second.x),A=Math.min(p.first.y,p.second.y),C=Math.abs(p.second.x-p.first.x),u=Math.abs(p.second.y-p.first.y);D.fillRect(B,A,C,u);D.strokeRect(B,A,C,u);D.restore()}});k.hooks.shutdown.push(function(t,s){s.unbind("mousemove",e);s.unbind("mousedown",n);if(r){a(document).unbind("mouseup",r)}})}a.plot.plugins.push({init:b,options:{selection:{mode:null,color:"#e8cfac"}},name:"selection",version:"1.1"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.stack.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.stack.min.js
new file mode 100644
index 0000000..bba2a0e
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.stack.min.js
@@ -0,0 +1 @@
+(function(b){var a={series:{stack:null}};function c(f){function d(k,j){var h=null;for(var g=0;g<j.length;++g){if(k==j[g]){break}if(j[g].stack==k.stack){h=j[g]}}return h}function e(C,v,g){if(v.stack==null){return}var p=d(v,C.getData());if(!p){return}var z=g.pointsize,F=g.points,h=p.datapoints.pointsize,y=p.datapoints.points,t=[],x,w,k,J,I,r,u=v.lines.show,G=v.bars.horizontal,o=z>2&&(G?g.format[2].x:g.format[2].y),n=u&&v.lines.steps,E=true,q=G?1:0,H=G?0:1,D=0,B=0,A;while(true){if(D>=F.length){break}A=t.length;if(F[D]==null){for(m=0;m<z;++m){t.push(F[D+m])}D+=z}else{if(B>=y.length){if(!u){for(m=0;m<z;++m){t.push(F[D+m])}}D+=z}else{if(y[B]==null){for(m=0;m<z;++m){t.push(null)}E=true;B+=h}else{x=F[D+q];w=F[D+H];J=y[B+q];I=y[B+H];r=0;if(x==J){for(m=0;m<z;++m){t.push(F[D+m])}t[A+H]+=I;r=I;D+=z;B+=h}else{if(x>J){if(u&&D>0&&F[D-z]!=null){k=w+(F[D-z+H]-w)*(J-x)/(F[D-z+q]-x);t.push(J);t.push(k+I);for(m=2;m<z;++m){t.push(F[D+m])}r=I}B+=h}else{if(E&&u){D+=z;continue}for(m=0;m<z;++m){t.push(F[D+m])}if(u&&B>0&&y[B-h]!=null){r=I+(y[B-h+H]-I)*(x-J)/(y[B-h+q]-J)}t[A+H]+=r;D+=z}}E=false;if(A!=t.length&&o){t[A+2]+=r}}}}if(n&&A!=t.length&&A>0&&t[A]!=null&&t[A]!=t[A-z]&&t[A+1]!=t[A-z+1]){for(m=0;m<z;++m){t[A+z+m]=t[A+m]}t[A+1]=t[A-z+1]}}g.points=t}f.hooks.processDatapoints.push(e)}b.plot.plugins.push({init:c,options:a,name:"stack",version:"1.2"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.symbol.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.symbol.min.js
new file mode 100644
index 0000000..272e003
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.symbol.min.js
@@ -0,0 +1 @@
+(function(b){function a(h,e,g){var d={square:function(k,j,n,i,m){var l=i*Math.sqrt(Math.PI)/2;k.rect(j-l,n-l,l+l,l+l)},diamond:function(k,j,n,i,m){var l=i*Math.sqrt(Math.PI/2);k.moveTo(j-l,n);k.lineTo(j,n-l);k.lineTo(j+l,n);k.lineTo(j,n+l);k.lineTo(j-l,n)},triangle:function(l,k,o,j,n){var m=j*Math.sqrt(2*Math.PI/Math.sin(Math.PI/3));var i=m*Math.sin(Math.PI/3);l.moveTo(k-m/2,o+i/2);l.lineTo(k+m/2,o+i/2);if(!n){l.lineTo(k,o-i/2);l.lineTo(k-m/2,o+i/2)}},cross:function(k,j,n,i,m){var l=i*Math.sqrt(Math.PI)/2;k.moveTo(j-l,n-l);k.lineTo(j+l,n+l);k.moveTo(j-l,n+l);k.lineTo(j+l,n-l)}};var f=e.points.symbol;if(d[f]){e.points.symbol=d[f]}}function c(d){d.hooks.processDatapoints.push(a)}b.plot.plugins.push({init:c,name:"symbols",version:"1.0"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js
new file mode 100644
index 0000000..d8b79df
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/flot/jquery.flot.threshold.min.js
@@ -0,0 +1 @@
+(function(B){var A={series:{threshold:null}};function C(D){function E(L,S,M){if(!S.threshold){return }var F=M.pointsize,I,O,N,G,K,H=B.extend({},S);H.datapoints={points:[],pointsize:F};H.label=null;H.color=S.threshold.color;H.threshold=null;H.originSeries=S;H.data=[];var P=S.threshold.below,Q=M.points,R=S.lines.show;threspoints=[];newpoints=[];for(I=0;I<Q.length;I+=F){O=Q[I];N=Q[I+1];K=G;if(N<P){G=threspoints}else{G=newpoints}if(R&&K!=G&&O!=null&&I>0&&Q[I-F]!=null){var J=(O-Q[I-F])/(N-Q[I-F+1])*(P-N)+O;K.push(J);K.push(P);for(m=2;m<F;++m){K.push(Q[I+m])}G.push(null);G.push(null);for(m=2;m<F;++m){G.push(Q[I+m])}G.push(J);G.push(P);for(m=2;m<F;++m){G.push(Q[I+m])}}G.push(O);G.push(N)}M.points=newpoints;H.datapoints.points=threspoints;if(H.datapoints.points.length>0){L.getData().push(H)}}D.hooks.processDatapoints.push(E)}B.plot.plugins.push({init:C,options:A,name:"threshold",version:"1.0"})})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/jquery/jquery.min.js b/hyracks-control-cc/src/main/resources/static/javascript/jquery/jquery.min.js
new file mode 100644
index 0000000..628ed9b
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/jquery/jquery.min.js
@@ -0,0 +1,4 @@
+/*! jQuery v1.6.4 http://jquery.com/ | http://jquery.org/license */
+(function(a,b){function cu(a){return f.isWindow(a)?a:a.nodeType===9?a.defaultView||a.parentWindow:!1}function cr(a){if(!cg[a]){var b=c.body,d=f("<"+a+">").appendTo(b),e=d.css("display");d.remove();if(e==="none"||e===""){ch||(ch=c.createElement("iframe"),ch.frameBorder=ch.width=ch.height=0),b.appendChild(ch);if(!ci||!ch.createElement)ci=(ch.contentWindow||ch.contentDocument).document,ci.write((c.compatMode==="CSS1Compat"?"<!doctype html>":"")+"<html><body>"),ci.close();d=ci.createElement(a),ci.body.appendChild(d),e=f.css(d,"display"),b.removeChild(ch)}cg[a]=e}return cg[a]}function cq(a,b){var c={};f.each(cm.concat.apply([],cm.slice(0,b)),function(){c[this]=a});return c}function cp(){cn=b}function co(){setTimeout(cp,0);return cn=f.now()}function cf(){try{return new a.ActiveXObject("Microsoft.XMLHTTP")}catch(b){}}function ce(){try{return new a.XMLHttpRequest}catch(b){}}function b$(a,c){a.dataFilter&&(c=a.dataFilter(c,a.dataType));var d=a.dataTypes,e={},g,h,i=d.length,j,k=d[0],l,m,n,o,p;for(g=1;g<i;g++){if(g===1)for(h in a.converters)typeof h=="string"&&(e[h.toLowerCase()]=a.converters[h]);l=k,k=d[g];if(k==="*")k=l;else if(l!=="*"&&l!==k){m=l+" "+k,n=e[m]||e["* "+k];if(!n){p=b;for(o in e){j=o.split(" ");if(j[0]===l||j[0]==="*"){p=e[j[1]+" "+k];if(p){o=e[o],o===!0?n=p:p===!0&&(n=o);break}}}}!n&&!p&&f.error("No conversion from "+m.replace(" "," to ")),n!==!0&&(c=n?n(c):p(o(c)))}}return c}function bZ(a,c,d){var e=a.contents,f=a.dataTypes,g=a.responseFields,h,i,j,k;for(i in g)i in d&&(c[g[i]]=d[i]);while(f[0]==="*")f.shift(),h===b&&(h=a.mimeType||c.getResponseHeader("content-type"));if(h)for(i in e)if(e[i]&&e[i].test(h)){f.unshift(i);break}if(f[0]in d)j=f[0];else{for(i in d){if(!f[0]||a.converters[i+" "+f[0]]){j=i;break}k||(k=i)}j=j||k}if(j){j!==f[0]&&f.unshift(j);return d[j]}}function bY(a,b,c,d){if(f.isArray(b))f.each(b,function(b,e){c||bA.test(a)?d(a,e):bY(a+"["+(typeof e=="object"||f.isArray(e)?b:"")+"]",e,c,d)});else if(!c&&b!=null&&typeof b=="object")for(var e in b)bY(a+"["+e+"]",b[e],c,d);else d(a,b)}function bX(a,c){var d,e,g=f.ajaxSettings.flatOptions||{};for(d in c)c[d]!==b&&((g[d]?a:e||(e={}))[d]=c[d]);e&&f.extend(!0,a,e)}function bW(a,c,d,e,f,g){f=f||c.dataTypes[0],g=g||{},g[f]=!0;var h=a[f],i=0,j=h?h.length:0,k=a===bP,l;for(;i<j&&(k||!l);i++)l=h[i](c,d,e),typeof l=="string"&&(!k||g[l]?l=b:(c.dataTypes.unshift(l),l=bW(a,c,d,e,l,g)));(k||!l)&&!g["*"]&&(l=bW(a,c,d,e,"*",g));return l}function bV(a){return function(b,c){typeof b!="string"&&(c=b,b="*");if(f.isFunction(c)){var d=b.toLowerCase().split(bL),e=0,g=d.length,h,i,j;for(;e<g;e++)h=d[e],j=/^\+/.test(h),j&&(h=h.substr(1)||"*"),i=a[h]=a[h]||[],i[j?"unshift":"push"](c)}}}function by(a,b,c){var d=b==="width"?a.offsetWidth:a.offsetHeight,e=b==="width"?bt:bu;if(d>0){c!=="border"&&f.each(e,function(){c||(d-=parseFloat(f.css(a,"padding"+this))||0),c==="margin"?d+=parseFloat(f.css(a,c+this))||0:d-=parseFloat(f.css(a,"border"+this+"Width"))||0});return d+"px"}d=bv(a,b,b);if(d<0||d==null)d=a.style[b]||0;d=parseFloat(d)||0,c&&f.each(e,function(){d+=parseFloat(f.css(a,"padding"+this))||0,c!=="padding"&&(d+=parseFloat(f.css(a,"border"+this+"Width"))||0),c==="margin"&&(d+=parseFloat(f.css(a,c+this))||0)});return d+"px"}function bl(a,b){b.src?f.ajax({url:b.src,async:!1,dataType:"script"}):f.globalEval((b.text||b.textContent||b.innerHTML||"").replace(bd,"/*$0*/")),b.parentNode&&b.parentNode.removeChild(b)}function bk(a){f.nodeName(a,"input")?bj(a):"getElementsByTagName"in a&&f.grep(a.getElementsByTagName("input"),bj)}function bj(a){if(a.type==="checkbox"||a.type==="radio")a.defaultChecked=a.checked}function bi(a){return"getElementsByTagName"in a?a.getElementsByTagName("*"):"querySelectorAll"in a?a.querySelectorAll("*"):[]}function bh(a,b){var c;if(b.nodeType===1){b.clearAttributes&&b.clearAttributes(),b.mergeAttributes&&b.mergeAttributes(a),c=b.nodeName.toLowerCase();if(c==="object")b.outerHTML=a.outerHTML;else if(c!=="input"||a.type!=="checkbox"&&a.type!=="radio"){if(c==="option")b.selected=a.defaultSelected;else if(c==="input"||c==="textarea")b.defaultValue=a.defaultValue}else a.checked&&(b.defaultChecked=b.checked=a.checked),b.value!==a.value&&(b.value=a.value);b.removeAttribute(f.expando)}}function bg(a,b){if(b.nodeType===1&&!!f.hasData(a)){var c=f.expando,d=f.data(a),e=f.data(b,d);if(d=d[c]){var g=d.events;e=e[c]=f.extend({},d);if(g){delete e.handle,e.events={};for(var h in g)for(var i=0,j=g[h].length;i<j;i++)f.event.add(b,h+(g[h][i].namespace?".":"")+g[h][i].namespace,g[h][i],g[h][i].data)}}}}function bf(a,b){return f.nodeName(a,"table")?a.getElementsByTagName("tbody")[0]||a.appendChild(a.ownerDocument.createElement("tbody")):a}function V(a,b,c){b=b||0;if(f.isFunction(b))return f.grep(a,function(a,d){var e=!!b.call(a,d,a);return e===c});if(b.nodeType)return f.grep(a,function(a,d){return a===b===c});if(typeof b=="string"){var d=f.grep(a,function(a){return a.nodeType===1});if(Q.test(b))return f.filter(b,d,!c);b=f.filter(b,d)}return f.grep(a,function(a,d){return f.inArray(a,b)>=0===c})}function U(a){return!a||!a.parentNode||a.parentNode.nodeType===11}function M(a,b){return(a&&a!=="*"?a+".":"")+b.replace(y,"`").replace(z,"&")}function L(a){var b,c,d,e,g,h,i,j,k,l,m,n,o,p=[],q=[],r=f._data(this,"events");if(!(a.liveFired===this||!r||!r.live||a.target.disabled||a.button&&a.type==="click")){a.namespace&&(n=new RegExp("(^|\\.)"+a.namespace.split(".").join("\\.(?:.*\\.)?")+"(\\.|$)")),a.liveFired=this;var s=r.live.slice(0);for(i=0;i<s.length;i++)g=s[i],g.origType.replace(w,"")===a.type?q.push(g.selector):s.splice(i--,1);e=f(a.target).closest(q,a.currentTarget);for(j=0,k=e.length;j<k;j++){m=e[j];for(i=0;i<s.length;i++){g=s[i];if(m.selector===g.selector&&(!n||n.test(g.namespace))&&!m.elem.disabled){h=m.elem,d=null;if(g.preType==="mouseenter"||g.preType==="mouseleave")a.type=g.preType,d=f(a.relatedTarget).closest(g.selector)[0],d&&f.contains(h,d)&&(d=h);(!d||d!==h)&&p.push({elem:h,handleObj:g,level:m.level})}}}for(j=0,k=p.length;j<k;j++){e=p[j];if(c&&e.level>c)break;a.currentTarget=e.elem,a.data=e.handleObj.data,a.handleObj=e.handleObj,o=e.handleObj.origHandler.apply(e.elem,arguments);if(o===!1||a.isPropagationStopped()){c=e.level,o===!1&&(b=!1);if(a.isImmediatePropagationStopped())break}}return b}}function J(a,c,d){var e=f.extend({},d[0]);e.type=a,e.originalEvent={},e.liveFired=b,f.event.handle.call(c,e),e.isDefaultPrevented()&&d[0].preventDefault()}function D(){return!0}function C(){return!1}function m(a,c,d){var e=c+"defer",g=c+"queue",h=c+"mark",i=f.data(a,e,b,!0);i&&(d==="queue"||!f.data(a,g,b,!0))&&(d==="mark"||!f.data(a,h,b,!0))&&setTimeout(function(){!f.data(a,g,b,!0)&&!f.data(a,h,b,!0)&&(f.removeData(a,e,!0),i.resolve())},0)}function l(a){for(var b in a)if(b!=="toJSON")return!1;return!0}function k(a,c,d){if(d===b&&a.nodeType===1){var e="data-"+c.replace(j,"-$1").toLowerCase();d=a.getAttribute(e);if(typeof d=="string"){try{d=d==="true"?!0:d==="false"?!1:d==="null"?null:f.isNaN(d)?i.test(d)?f.parseJSON(d):d:parseFloat(d)}catch(g){}f.data(a,c,d)}else d=b}return d}var c=a.document,d=a.navigator,e=a.location,f=function(){function K(){if(!e.isReady){try{c.documentElement.doScroll("left")}catch(a){setTimeout(K,1);return}e.ready()}}var e=function(a,b){return new e.fn.init(a,b,h)},f=a.jQuery,g=a.$,h,i=/^(?:[^#<]*(<[\w\W]+>)[^>]*$|#([\w\-]*)$)/,j=/\S/,k=/^\s+/,l=/\s+$/,m=/\d/,n=/^<(\w+)\s*\/?>(?:<\/\1>)?$/,o=/^[\],:{}\s]*$/,p=/\\(?:["\\\/bfnrt]|u[0-9a-fA-F]{4})/g,q=/"[^"\\\n\r]*"|true|false|null|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?/g,r=/(?:^|:|,)(?:\s*\[)+/g,s=/(webkit)[ \/]([\w.]+)/,t=/(opera)(?:.*version)?[ \/]([\w.]+)/,u=/(msie) ([\w.]+)/,v=/(mozilla)(?:.*? rv:([\w.]+))?/,w=/-([a-z]|[0-9])/ig,x=/^-ms-/,y=function(a,b){return(b+"").toUpperCase()},z=d.userAgent,A,B,C,D=Object.prototype.toString,E=Object.prototype.hasOwnProperty,F=Array.prototype.push,G=Array.prototype.slice,H=String.prototype.trim,I=Array.prototype.indexOf,J={};e.fn=e.prototype={constructor:e,init:function(a,d,f){var g,h,j,k;if(!a)return this;if(a.nodeType){this.context=this[0]=a,this.length=1;return this}if(a==="body"&&!d&&c.body){this.context=c,this[0]=c.body,this.selector=a,this.length=1;return this}if(typeof a=="string"){a.charAt(0)!=="<"||a.charAt(a.length-1)!==">"||a.length<3?g=i.exec(a):g=[null,a,null];if(g&&(g[1]||!d)){if(g[1]){d=d instanceof e?d[0]:d,k=d?d.ownerDocument||d:c,j=n.exec(a),j?e.isPlainObject(d)?(a=[c.createElement(j[1])],e.fn.attr.call(a,d,!0)):a=[k.createElement(j[1])]:(j=e.buildFragment([g[1]],[k]),a=(j.cacheable?e.clone(j.fragment):j.fragment).childNodes);return e.merge(this,a)}h=c.getElementById(g[2]);if(h&&h.parentNode){if(h.id!==g[2])return f.find(a);this.length=1,this[0]=h}this.context=c,this.selector=a;return this}return!d||d.jquery?(d||f).find(a):this.constructor(d).find(a)}if(e.isFunction(a))return f.ready(a);a.selector!==b&&(this.selector=a.selector,this.context=a.context);return e.makeArray(a,this)},selector:"",jquery:"1.6.4",length:0,size:function(){return this.length},toArray:function(){return G.call(this,0)},get:function(a){return a==null?this.toArray():a<0?this[this.length+a]:this[a]},pushStack:function(a,b,c){var d=this.constructor();e.isArray(a)?F.apply(d,a):e.merge(d,a),d.prevObject=this,d.context=this.context,b==="find"?d.selector=this.selector+(this.selector?" ":"")+c:b&&(d.selector=this.selector+"."+b+"("+c+")");return d},each:function(a,b){return e.each(this,a,b)},ready:function(a){e.bindReady(),B.done(a);return this},eq:function(a){return a===-1?this.slice(a):this.slice(a,+a+1)},first:function(){return this.eq(0)},last:function(){return this.eq(-1)},slice:function(){return this.pushStack(G.apply(this,arguments),"slice",G.call(arguments).join(","))},map:function(a){return this.pushStack(e.map(this,function(b,c){return a.call(b,c,b)}))},end:function(){return this.prevObject||this.constructor(null)},push:F,sort:[].sort,splice:[].splice},e.fn.init.prototype=e.fn,e.extend=e.fn.extend=function(){var a,c,d,f,g,h,i=arguments[0]||{},j=1,k=arguments.length,l=!1;typeof i=="boolean"&&(l=i,i=arguments[1]||{},j=2),typeof i!="object"&&!e.isFunction(i)&&(i={}),k===j&&(i=this,--j);for(;j<k;j++)if((a=arguments[j])!=null)for(c in a){d=i[c],f=a[c];if(i===f)continue;l&&f&&(e.isPlainObject(f)||(g=e.isArray(f)))?(g?(g=!1,h=d&&e.isArray(d)?d:[]):h=d&&e.isPlainObject(d)?d:{},i[c]=e.extend(l,h,f)):f!==b&&(i[c]=f)}return i},e.extend({noConflict:function(b){a.$===e&&(a.$=g),b&&a.jQuery===e&&(a.jQuery=f);return e},isReady:!1,readyWait:1,holdReady:function(a){a?e.readyWait++:e.ready(!0)},ready:function(a){if(a===!0&&!--e.readyWait||a!==!0&&!e.isReady){if(!c.body)return setTimeout(e.ready,1);e.isReady=!0;if(a!==!0&&--e.readyWait>0)return;B.resolveWith(c,[e]),e.fn.trigger&&e(c).trigger("ready").unbind("ready")}},bindReady:function(){if(!B){B=e._Deferred();if(c.readyState==="complete")return setTimeout(e.ready,1);if(c.addEventListener)c.addEventListener("DOMContentLoaded",C,!1),a.addEventListener("load",e.ready,!1);else if(c.attachEvent){c.attachEvent("onreadystatechange",C),a.attachEvent("onload",e.ready);var b=!1;try{b=a.frameElement==null}catch(d){}c.documentElement.doScroll&&b&&K()}}},isFunction:function(a){return e.type(a)==="function"},isArray:Array.isArray||function(a){return e.type(a)==="array"},isWindow:function(a){return a&&typeof a=="object"&&"setInterval"in a},isNaN:function(a){return a==null||!m.test(a)||isNaN(a)},type:function(a){return a==null?String(a):J[D.call(a)]||"object"},isPlainObject:function(a){if(!a||e.type(a)!=="object"||a.nodeType||e.isWindow(a))return!1;try{if(a.constructor&&!E.call(a,"constructor")&&!E.call(a.constructor.prototype,"isPrototypeOf"))return!1}catch(c){return!1}var d;for(d in a);return d===b||E.call(a,d)},isEmptyObject:function(a){for(var b in a)return!1;return!0},error:function(a){throw a},parseJSON:function(b){if(typeof b!="string"||!b)return null;b=e.trim(b);if(a.JSON&&a.JSON.parse)return a.JSON.parse(b);if(o.test(b.replace(p,"@").replace(q,"]").replace(r,"")))return(new Function("return "+b))();e.error("Invalid JSON: "+b)},parseXML:function(c){var d,f;try{a.DOMParser?(f=new DOMParser,d=f.parseFromString(c,"text/xml")):(d=new ActiveXObject("Microsoft.XMLDOM"),d.async="false",d.loadXML(c))}catch(g){d=b}(!d||!d.documentElement||d.getElementsByTagName("parsererror").length)&&e.error("Invalid XML: "+c);return d},noop:function(){},globalEval:function(b){b&&j.test(b)&&(a.execScript||function(b){a.eval.call(a,b)})(b)},camelCase:function(a){return a.replace(x,"ms-").replace(w,y)},nodeName:function(a,b){return a.nodeName&&a.nodeName.toUpperCase()===b.toUpperCase()},each:function(a,c,d){var f,g=0,h=a.length,i=h===b||e.isFunction(a);if(d){if(i){for(f in a)if(c.apply(a[f],d)===!1)break}else for(;g<h;)if(c.apply(a[g++],d)===!1)break}else if(i){for(f in a)if(c.call(a[f],f,a[f])===!1)break}else for(;g<h;)if(c.call(a[g],g,a[g++])===!1)break;return a},trim:H?function(a){return a==null?"":H.call(a)}:function(a){return a==null?"":(a+"").replace(k,"").replace(l,"")},makeArray:function(a,b){var c=b||[];if(a!=null){var d=e.type(a);a.length==null||d==="string"||d==="function"||d==="regexp"||e.isWindow(a)?F.call(c,a):e.merge(c,a)}return c},inArray:function(a,b){if(!b)return-1;if(I)return I.call(b,a);for(var c=0,d=b.length;c<d;c++)if(b[c]===a)return c;return-1},merge:function(a,c){var d=a.length,e=0;if(typeof c.length=="number")for(var f=c.length;e<f;e++)a[d++]=c[e];else while(c[e]!==b)a[d++]=c[e++];a.length=d;return a},grep:function(a,b,c){var d=[],e;c=!!c;for(var f=0,g=a.length;f<g;f++)e=!!b(a[f],f),c!==e&&d.push(a[f]);return d},map:function(a,c,d){var f,g,h=[],i=0,j=a.length,k=a instanceof e||j!==b&&typeof j=="number"&&(j>0&&a[0]&&a[j-1]||j===0||e.isArray(a));if(k)for(;i<j;i++)f=c(a[i],i,d),f!=null&&(h[h.length]=f);else for(g in a)f=c(a[g],g,d),f!=null&&(h[h.length]=f);return h.concat.apply([],h)},guid:1,proxy:function(a,c){if(typeof c=="string"){var d=a[c];c=a,a=d}if(!e.isFunction(a))return b;var f=G.call(arguments,2),g=function(){return a.apply(c,f.concat(G.call(arguments)))};g.guid=a.guid=a.guid||g.guid||e.guid++;return g},access:function(a,c,d,f,g,h){var i=a.length;if(typeof c=="object"){for(var j in c)e.access(a,j,c[j],f,g,d);return a}if(d!==b){f=!h&&f&&e.isFunction(d);for(var k=0;k<i;k++)g(a[k],c,f?d.call(a[k],k,g(a[k],c)):d,h);return a}return i?g(a[0],c):b},now:function(){return(new Date).getTime()},uaMatch:function(a){a=a.toLowerCase();var b=s.exec(a)||t.exec(a)||u.exec(a)||a.indexOf("compatible")<0&&v.exec(a)||[];return{browser:b[1]||"",version:b[2]||"0"}},sub:function(){function a(b,c){return new a.fn.init(b,c)}e.extend(!0,a,this),a.superclass=this,a.fn=a.prototype=this(),a.fn.constructor=a,a.sub=this.sub,a.fn.init=function(d,f){f&&f instanceof e&&!(f instanceof a)&&(f=a(f));return e.fn.init.call(this,d,f,b)},a.fn.init.prototype=a.fn;var b=a(c);return a},browser:{}}),e.each("Boolean Number String Function Array Date RegExp Object".split(" "),function(a,b){J["[object "+b+"]"]=b.toLowerCase()}),A=e.uaMatch(z),A.browser&&(e.browser[A.browser]=!0,e.browser.version=A.version),e.browser.webkit&&(e.browser.safari=!0),j.test(" ")&&(k=/^[\s\xA0]+/,l=/[\s\xA0]+$/),h=e(c),c.addEventListener?C=function(){c.removeEventListener("DOMContentLoaded",C,!1),e.ready()}:c.attachEvent&&(C=function(){c.readyState==="complete"&&(c.detachEvent("onreadystatechange",C),e.ready())});return e}(),g="done fail isResolved isRejected promise then always pipe".split(" "),h=[].slice;f.extend({_Deferred:function(){var a=[],b,c,d,e={done:function(){if(!d){var c=arguments,g,h,i,j,k;b&&(k=b,b=0);for(g=0,h=c.length;g<h;g++)i=c[g],j=f.type(i),j==="array"?e.done.apply(e,i):j==="function"&&a.push(i);k&&e.resolveWith(k[0],k[1])}return this},resolveWith:function(e,f){if(!d&&!b&&!c){f=f||[],c=1;try{while(a[0])a.shift().apply(e,f)}finally{b=[e,f],c=0}}return this},resolve:function(){e.resolveWith(this,arguments);return this},isResolved:function(){return!!c||!!b},cancel:function(){d=1,a=[];return this}};return e},Deferred:function(a){var b=f._Deferred(),c=f._Deferred(),d;f.extend(b,{then:function(a,c){b.done(a).fail(c);return this},always:function(){return b.done.apply(b,arguments).fail.apply(this,arguments)},fail:c.done,rejectWith:c.resolveWith,reject:c.resolve,isRejected:c.isResolved,pipe:function(a,c){return f.Deferred(function(d){f.each({done:[a,"resolve"],fail:[c,"reject"]},function(a,c){var e=c[0],g=c[1],h;f.isFunction(e)?b[a](function(){h=e.apply(this,arguments),h&&f.isFunction(h.promise)?h.promise().then(d.resolve,d.reject):d[g+"With"](this===b?d:this,[h])}):b[a](d[g])})}).promise()},promise:function(a){if(a==null){if(d)return d;d=a={}}var c=g.length;while(c--)a[g[c]]=b[g[c]];return a}}),b.done(c.cancel).fail(b.cancel),delete b.cancel,a&&a.call(b,b);return b},when:function(a){function i(a){return function(c){b[a]=arguments.length>1?h.call(arguments,0):c,--e||g.resolveWith(g,h.call(b,0))}}var b=arguments,c=0,d=b.length,e=d,g=d<=1&&a&&f.isFunction(a.promise)?a:f.Deferred();if(d>1){for(;c<d;c++)b[c]&&f.isFunction(b[c].promise)?b[c].promise().then(i(c),g.reject):--e;e||g.resolveWith(g,b)}else g!==a&&g.resolveWith(g,d?[a]:[]);return g.promise()}}),f.support=function(){var a=c.createElement("div"),b=c.documentElement,d,e,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u;a.setAttribute("className","t"),a.innerHTML=" <link/><table></table><a href='/a' style='top:1px;float:left;opacity:.55;'>a</a><input type='checkbox'/>",d=a.getElementsByTagName("*"),e=a.getElementsByTagName("a")[0];if(!d||!d.length||!e)return{};g=c.createElement("select"),h=g.appendChild(c.createElement("option")),i=a.getElementsByTagName("input")[0],k={leadingWhitespace:a.firstChild.nodeType===3,tbody:!a.getElementsByTagName("tbody").length,htmlSerialize:!!a.getElementsByTagName("link").length,style:/top/.test(e.getAttribute("style")),hrefNormalized:e.getAttribute("href")==="/a",opacity:/^0.55$/.test(e.style.opacity),cssFloat:!!e.style.cssFloat,checkOn:i.value==="on",optSelected:h.selected,getSetAttribute:a.className!=="t",submitBubbles:!0,changeBubbles:!0,focusinBubbles:!1,deleteExpando:!0,noCloneEvent:!0,inlineBlockNeedsLayout:!1,shrinkWrapBlocks:!1,reliableMarginRight:!0},i.checked=!0,k.noCloneChecked=i.cloneNode(!0).checked,g.disabled=!0,k.optDisabled=!h.disabled;try{delete a.test}catch(v){k.deleteExpando=!1}!a.addEventListener&&a.attachEvent&&a.fireEvent&&(a.attachEvent("onclick",function(){k.noCloneEvent=!1}),a.cloneNode(!0).fireEvent("onclick")),i=c.createElement("input"),i.value="t",i.setAttribute("type","radio"),k.radioValue=i.value==="t",i.setAttribute("checked","checked"),a.appendChild(i),l=c.createDocumentFragment(),l.appendChild(a.firstChild),k.checkClone=l.cloneNode(!0).cloneNode(!0).lastChild.checked,a.innerHTML="",a.style.width=a.style.paddingLeft="1px",m=c.getElementsByTagName("body")[0],o=c.createElement(m?"div":"body"),p={visibility:"hidden",width:0,height:0,border:0,margin:0,background:"none"},m&&f.extend(p,{position:"absolute",left:"-1000px",top:"-1000px"});for(t in p)o.style[t]=p[t];o.appendChild(a),n=m||b,n.insertBefore(o,n.firstChild),k.appendChecked=i.checked,k.boxModel=a.offsetWidth===2,"zoom"in a.style&&(a.style.display="inline",a.style.zoom=1,k.inlineBlockNeedsLayout=a.offsetWidth===2,a.style.display="",a.innerHTML="<div style='width:4px;'></div>",k.shrinkWrapBlocks=a.offsetWidth!==2),a.innerHTML="<table><tr><td style='padding:0;border:0;display:none'></td><td>t</td></tr></table>",q=a.getElementsByTagName("td"),u=q[0].offsetHeight===0,q[0].style.display="",q[1].style.display="none",k.reliableHiddenOffsets=u&&q[0].offsetHeight===0,a.innerHTML="",c.defaultView&&c.defaultView.getComputedStyle&&(j=c.createElement("div"),j.style.width="0",j.style.marginRight="0",a.appendChild(j),k.reliableMarginRight=(parseInt((c.defaultView.getComputedStyle(j,null)||{marginRight:0}).marginRight,10)||0)===0),o.innerHTML="",n.removeChild(o);if(a.attachEvent)for(t in{submit:1,change:1,focusin:1})s="on"+t,u=s in a,u||(a.setAttribute(s,"return;"),u=typeof a[s]=="function"),k[t+"Bubbles"]=u;o=l=g=h=m=j=a=i=null;return k}(),f.boxModel=f.support.boxModel;var i=/^(?:\{.*\}|\[.*\])$/,j=/([A-Z])/g;f.extend({cache:{},uuid:0,expando:"jQuery"+(f.fn.jquery+Math.random()).replace(/\D/g,""),noData:{embed:!0,object:"clsid:D27CDB6E-AE6D-11cf-96B8-444553540000",applet:!0},hasData:function(a){a=a.nodeType?f.cache[a[f.expando]]:a[f.expando];return!!a&&!l(a)},data:function(a,c,d,e){if(!!f.acceptData(a)){var g,h,i=f.expando,j=typeof c=="string",k=a.nodeType,l=k?f.cache:a,m=k?a[f.expando]:a[f.expando]&&f.expando;if((!m||e&&m&&l[m]&&!l[m][i])&&j&&d===b)return;m||(k?a[f.expando]=m=++f.uuid:m=f.expando),l[m]||(l[m]={},k||(l[m].toJSON=f.noop));if(typeof c=="object"||typeof c=="function")e?l[m][i]=f.extend(l[m][i],c):l[m]=f.extend(l[m],c);g=l[m],e&&(g[i]||(g[i]={}),g=g[i]),d!==b&&(g[f.camelCase(c)]=d);if(c==="events"&&!g[c])return g[i]&&g[i].events;j?(h=g[c],h==null&&(h=g[f.camelCase(c)])):h=g;return h}},removeData:function(a,b,c){if(!!f.acceptData(a)){var d,e=f.expando,g=a.nodeType,h=g?f.cache:a,i=g?a[f.expando]:f.expando;if(!h[i])return;if(b){d=c?h[i][e]:h[i];if(d){d[b]||(b=f.camelCase(b)),delete d[b];if(!l(d))return}}if(c){delete h[i][e];if(!l(h[i]))return}var j=h[i][e];f.support.deleteExpando||!h.setInterval?delete h[i]:h[i]=null,j?(h[i]={},g||(h[i].toJSON=f.noop),h[i][e]=j):g&&(f.support.deleteExpando?delete a[f.expando]:a.removeAttribute?a.removeAttribute(f.expando):a[f.expando]=null)}},_data:function(a,b,c){return f.data(a,b,c,!0)},acceptData:function(a){if(a.nodeName){var b=f.noData[a.nodeName.toLowerCase()];if(b)return b!==!0&&a.getAttribute("classid")===b}return!0}}),f.fn.extend({data:function(a,c){var d=null;if(typeof a=="undefined"){if(this.length){d=f.data(this[0]);if(this[0].nodeType===1){var e=this[0].attributes,g;for(var h=0,i=e.length;h<i;h++)g=e[h].name,g.indexOf("data-")===0&&(g=f.camelCase(g.substring(5)),k(this[0],g,d[g]))}}return d}if(typeof a=="object")return this.each(function(){f.data(this,a)});var j=a.split(".");j[1]=j[1]?"."+j[1]:"";if(c===b){d=this.triggerHandler("getData"+j[1]+"!",[j[0]]),d===b&&this.length&&(d=f.data(this[0],a),d=k(this[0],a,d));return d===b&&j[1]?this.data(j[0]):d}return this.each(function(){var b=f(this),d=[j[0],c];b.triggerHandler("setData"+j[1]+"!",d),f.data(this,a,c),b.triggerHandler("changeData"+j[1]+"!",d)})},removeData:function(a){return this.each(function(){f.removeData(this,a)})}}),f.extend({_mark:function(a,c){a&&(c=(c||"fx")+"mark",f.data(a,c,(f.data(a,c,b,!0)||0)+1,!0))},_unmark:function(a,c,d){a!==!0&&(d=c,c=a,a=!1);if(c){d=d||"fx";var e=d+"mark",g=a?0:(f.data(c,e,b,!0)||1)-1;g?f.data(c,e,g,!0):(f.removeData(c,e,!0),m(c,d,"mark"))}},queue:function(a,c,d){if(a){c=(c||"fx")+"queue";var e=f.data(a,c,b,!0);d&&(!e||f.isArray(d)?e=f.data(a,c,f.makeArray(d),!0):e.push(d));return e||[]}},dequeue:function(a,b){b=b||"fx";var c=f.queue(a,b),d=c.shift(),e;d==="inprogress"&&(d=c.shift()),d&&(b==="fx"&&c.unshift("inprogress"),d.call(a,function(){f.dequeue(a,b)})),c.length||(f.removeData(a,b+"queue",!0),m(a,b,"queue"))}}),f.fn.extend({queue:function(a,c){typeof a!="string"&&(c=a,a="fx");if(c===b)return f.queue(this[0],a);return this.each(function(){var b=f.queue(this,a,c);a==="fx"&&b[0]!=="inprogress"&&f.dequeue(this,a)})},dequeue:function(a){return this.each(function(){f.dequeue(this,a)})},delay:function(a,b){a=f.fx?f.fx.speeds[a]||a:a,b=b||"fx";return this.queue(b,function(){var c=this;setTimeout(function(){f.dequeue(c,b)},a)})},clearQueue:function(a){return this.queue(a||"fx",[])},promise:function(a,c){function m(){--h||d.resolveWith(e,[e])}typeof a!="string"&&(c=a,a=b),a=a||"fx";var d=f.Deferred(),e=this,g=e.length,h=1,i=a+"defer",j=a+"queue",k=a+"mark",l;while(g--)if(l=f.data(e[g],i,b,!0)||(f.data(e[g],j,b,!0)||f.data(e[g],k,b,!0))&&f.data(e[g],i,f._Deferred(),!0))h++,l.done(m);m();return d.promise()}});var n=/[\n\t\r]/g,o=/\s+/,p=/\r/g,q=/^(?:button|input)$/i,r=/^(?:button|input|object|select|textarea)$/i,s=/^a(?:rea)?$/i,t=/^(?:autofocus|autoplay|async|checked|controls|defer|disabled|hidden|loop|multiple|open|readonly|required|scoped|selected)$/i,u,v;f.fn.extend({attr:function(a,b){return f.access(this,a,b,!0,f.attr)},removeAttr:function(a){return this.each(function(){f.removeAttr(this,a)})},prop:function(a,b){return f.access(this,a,b,!0,f.prop)},removeProp:function(a){a=f.propFix[a]||a;return this.each(function(){try{this[a]=b,delete this[a]}catch(c){}})},addClass:function(a){var b,c,d,e,g,h,i;if(f.isFunction(a))return this.each(function(b){f(this).addClass(a.call(this,b,this.className))});if(a&&typeof a=="string"){b=a.split(o);for(c=0,d=this.length;c<d;c++){e=this[c];if(e.nodeType===1)if(!e.className&&b.length===1)e.className=a;else{g=" "+e.className+" ";for(h=0,i=b.length;h<i;h++)~g.indexOf(" "+b[h]+" ")||(g+=b[h]+" ");e.className=f.trim(g)}}}return this},removeClass:function(a){var c,d,e,g,h,i,j;if(f.isFunction(a))return this.each(function(b){f(this).removeClass(a.call(this,b,this.className))});if(a&&typeof a=="string"||a===b){c=(a||"").split(o);for(d=0,e=this.length;d<e;d++){g=this[d];if(g.nodeType===1&&g.className)if(a){h=(" "+g.className+" ").replace(n," ");for(i=0,j=c.length;i<j;i++)h=h.replace(" "+c[i]+" "," ");g.className=f.trim(h)}else g.className=""}}return this},toggleClass:function(a,b){var c=typeof a,d=typeof b=="boolean";if(f.isFunction(a))return this.each(function(c){f(this).toggleClass(a.call(this,c,this.className,b),b)});return this.each(function(){if(c==="string"){var e,g=0,h=f(this),i=b,j=a.split(o);while(e=j[g++])i=d?i:!h.hasClass(e),h[i?"addClass":"removeClass"](e)}else if(c==="undefined"||c==="boolean")this.className&&f._data(this,"__className__",this.className),this.className=this.className||a===!1?"":f._data(this,"__className__")||""})},hasClass:function(a){var b=" "+a+" ";for(var c=0,d=this.length;c<d;c++)if(this[c].nodeType===1&&(" "+this[c].className+" ").replace(n," ").indexOf(b)>-1)return!0;return!1},val:function(a){var c,d,e=this[0];if(!arguments.length){if(e){c=f.valHooks[e.nodeName.toLowerCase()]||f.valHooks[e.type];if(c&&"get"in c&&(d=c.get(e,"value"))!==b)return d;d=e.value;return typeof d=="string"?d.replace(p,""):d==null?"":d}return b}var g=f.isFunction(a);return this.each(function(d){var e=f(this),h;if(this.nodeType===1){g?h=a.call(this,d,e.val()):h=a,h==null?h="":typeof h=="number"?h+="":f.isArray(h)&&(h=f.map(h,function(a){return a==null?"":a+""})),c=f.valHooks[this.nodeName.toLowerCase()]||f.valHooks[this.type];if(!c||!("set"in c)||c.set(this,h,"value")===b)this.value=h}})}}),f.extend({valHooks:{option:{get:function(a){var b=a.attributes.value;return!b||b.specified?a.value:a.text}},select:{get:function(a){var b,c=a.selectedIndex,d=[],e=a.options,g=a.type==="select-one";if(c<0)return null;for(var h=g?c:0,i=g?c+1:e.length;h<i;h++){var j=e[h];if(j.selected&&(f.support.optDisabled?!j.disabled:j.getAttribute("disabled")===null)&&(!j.parentNode.disabled||!f.nodeName(j.parentNode,"optgroup"))){b=f(j).val();if(g)return b;d.push(b)}}if(g&&!d.length&&e.length)return f(e[c]).val();return d},set:function(a,b){var c=f.makeArray(b);f(a).find("option").each(function(){this.selected=f.inArray(f(this).val(),c)>=0}),c.length||(a.selectedIndex=-1);return c}}},attrFn:{val:!0,css:!0,html:!0,text:!0,data:!0,width:!0,height:!0,offset:!0},attrFix:{tabindex:"tabIndex"},attr:function(a,c,d,e){var g=a.nodeType;if(!a||g===3||g===8||g===2)return b;if(e&&c in f.attrFn)return f(a)[c](d);if(!("getAttribute"in a))return f.prop(a,c,d);var h,i,j=g!==1||!f.isXMLDoc(a);j&&(c=f.attrFix[c]||c,i=f.attrHooks[c],i||(t.test(c)?i=v:u&&(i=u)));if(d!==b){if(d===null){f.removeAttr(a,c);return b}if(i&&"set"in i&&j&&(h=i.set(a,d,c))!==b)return h;a.setAttribute(c,""+d);return d}if(i&&"get"in i&&j&&(h=i.get(a,c))!==null)return h;h=a.getAttribute(c);return h===null?b:h},removeAttr:function(a,b){var c;a.nodeType===1&&(b=f.attrFix[b]||b,f.attr(a,b,""),a.removeAttribute(b),t.test(b)&&(c=f.propFix[b]||b)in a&&(a[c]=!1))},attrHooks:{type:{set:function(a,b){if(q.test(a.nodeName)&&a.parentNode)f.error("type property can't be changed");else if(!f.support.radioValue&&b==="radio"&&f.nodeName(a,"input")){var c=a.value;a.setAttribute("type",b),c&&(a.value=c);return b}}},value:{get:function(a,b){if(u&&f.nodeName(a,"button"))return u.get(a,b);return b in a?a.value:null},set:function(a,b,c){if(u&&f.nodeName(a,"button"))return u.set(a,b,c);a.value=b}}},propFix:{tabindex:"tabIndex",readonly:"readOnly","for":"htmlFor","class":"className",maxlength:"maxLength",cellspacing:"cellSpacing",cellpadding:"cellPadding",rowspan:"rowSpan",colspan:"colSpan",usemap:"useMap",frameborder:"frameBorder",contenteditable:"contentEditable"},prop:function(a,c,d){var e=a.nodeType;if(!a||e===3||e===8||e===2)return b;var g,h,i=e!==1||!f.isXMLDoc(a);i&&(c=f.propFix[c]||c,h=f.propHooks[c]);return d!==b?h&&"set"in h&&(g=h.set(a,d,c))!==b?g:a[c]=d:h&&"get"in h&&(g=h.get(a,c))!==null?g:a[c]},propHooks:{tabIndex:{get:function(a){var c=a.getAttributeNode("tabindex");return c&&c.specified?parseInt(c.value,10):r.test(a.nodeName)||s.test(a.nodeName)&&a.href?0:b}}}}),f.attrHooks.tabIndex=f.propHooks.tabIndex,v={get:function(a,c){var d;return f.prop(a,c)===!0||(d=a.getAttributeNode(c))&&d.nodeValue!==!1?c.toLowerCase():b},set:function(a,b,c){var d;b===!1?f.removeAttr(a,c):(d=f.propFix[c]||c,d in a&&(a[d]=!0),a.setAttribute(c,c.toLowerCase()));return c}},f.support.getSetAttribute||(u=f.valHooks.button={get:function(a,c){var d;d=a.getAttributeNode(c);return d&&d.nodeValue!==""?d.nodeValue:b},set:function(a,b,d){var e=a.getAttributeNode(d);e||(e=c.createAttribute(d),a.setAttributeNode(e));return e.nodeValue=b+""}},f.each(["width","height"],function(a,b){f.attrHooks[b]=f.extend(f.attrHooks[b],{set:function(a,c){if(c===""){a.setAttribute(b,"auto");return c}}})})),f.support.hrefNormalized||f.each(["href","src","width","height"],function(a,c){f.attrHooks[c]=f.extend(f.attrHooks[c],{get:function(a){var d=a.getAttribute(c,2);return d===null?b:d}})}),f.support.style||(f.attrHooks.style={get:function(a){return a.style.cssText.toLowerCase()||b},set:function(a,b){return a.style.cssText=""+b}}),f.support.optSelected||(f.propHooks.selected=f.extend(f.propHooks.selected,{get:function(a){var b=a.parentNode;b&&(b.selectedIndex,b.parentNode&&b.parentNode.selectedIndex);return null}})),f.support.checkOn||f.each(["radio","checkbox"],function(){f.valHooks[this]={get:function(a){return a.getAttribute("value")===null?"on":a.value}}}),f.each(["radio","checkbox"],function(){f.valHooks[this]=f.extend(f.valHooks[this],{set:function(a,b){if(f.isArray(b))return a.checked=f.inArray(f(a).val(),b)>=0}})});var w=/\.(.*)$/,x=/^(?:textarea|input|select)$/i,y=/\./g,z=/ /g,A=/[^\w\s.|`]/g,B=function(a){return a.replace(A,"\\$&")};f.event={add:function(a,c,d,e){if(a.nodeType!==3&&a.nodeType!==8){if(d===!1)d=C;else if(!d)return;var g,h;d.handler&&(g=d,d=g.handler),d.guid||(d.guid=f.guid++);var i=f._data(a);if(!i)return;var j=i.events,k=i.handle;j||(i.events=j={}),k||(i.handle=k=function(a){return typeof f!="undefined"&&(!a||f.event.triggered!==a.type)?f.event.handle.apply(k.elem,arguments):b}),k.elem=a,c=c.split(" ");var l,m=0,n;while(l=c[m++]){h=g?f.extend({},g):{handler:d,data:e},l.indexOf(".")>-1?(n=l.split("."),l=n.shift(),h.namespace=n.slice(0).sort().join(".")):(n=[],h.namespace=""),h.type=l,h.guid||(h.guid=d.guid);var o=j[l],p=f.event.special[l]||{};if(!o){o=j[l]=[];if(!p.setup||p.setup.call(a,e,n,k)===!1)a.addEventListener?a.addEventListener(l,k,!1):a.attachEvent&&a.attachEvent("on"+l,k)}p.add&&(p.add.call(a,h),h.handler.guid||(h.handler.guid=d.guid)),o.push(h),f.event.global[l]=!0}a=null}},global:{},remove:function(a,c,d,e){if(a.nodeType!==3&&a.nodeType!==8){d===!1&&(d=C);var g,h,i,j,k=0,l,m,n,o,p,q,r,s=f.hasData(a)&&f._data(a),t=s&&s.events;if(!s||!t)return;c&&c.type&&(d=c.handler,c=c.type);if(!c||typeof c=="string"&&c.charAt(0)==="."){c=c||"";for(h in t)f.event.remove(a,h+c);return}c=c.split(" ");while(h=c[k++]){r=h,q=null,l=h.indexOf(".")<0,m=[],l||(m=h.split("."),h=m.shift(),n=new RegExp("(^|\\.)"+f.map(m.slice(0).sort(),B).join("\\.(?:.*\\.)?")+"(\\.|$)")),p=t[h];if(!p)continue;if(!d){for(j=0;j<p.length;j++){q=p[j];if(l||n.test(q.namespace))f.event.remove(a,r,q.handler,j),p.splice(j--,1)}continue}o=f.event.special[h]||{};for(j=e||0;j<p.length;j++){q=p[j];if(d.guid===q.guid){if(l||n.test(q.namespace))e==null&&p.splice(j--,1),o.remove&&o.remove.call(a,q);if(e!=null)break}}if(p.length===0||e!=null&&p.length===1)(!o.teardown||o.teardown.call(a,m)===!1)&&f.removeEvent(a,h,s.handle),g=null,delete
+t[h]}if(f.isEmptyObject(t)){var u=s.handle;u&&(u.elem=null),delete s.events,delete s.handle,f.isEmptyObject(s)&&f.removeData(a,b,!0)}}},customEvent:{getData:!0,setData:!0,changeData:!0},trigger:function(c,d,e,g){var h=c.type||c,i=[],j;h.indexOf("!")>=0&&(h=h.slice(0,-1),j=!0),h.indexOf(".")>=0&&(i=h.split("."),h=i.shift(),i.sort());if(!!e&&!f.event.customEvent[h]||!!f.event.global[h]){c=typeof c=="object"?c[f.expando]?c:new f.Event(h,c):new f.Event(h),c.type=h,c.exclusive=j,c.namespace=i.join("."),c.namespace_re=new RegExp("(^|\\.)"+i.join("\\.(?:.*\\.)?")+"(\\.|$)");if(g||!e)c.preventDefault(),c.stopPropagation();if(!e){f.each(f.cache,function(){var a=f.expando,b=this[a];b&&b.events&&b.events[h]&&f.event.trigger(c,d,b.handle.elem)});return}if(e.nodeType===3||e.nodeType===8)return;c.result=b,c.target=e,d=d!=null?f.makeArray(d):[],d.unshift(c);var k=e,l=h.indexOf(":")<0?"on"+h:"";do{var m=f._data(k,"handle");c.currentTarget=k,m&&m.apply(k,d),l&&f.acceptData(k)&&k[l]&&k[l].apply(k,d)===!1&&(c.result=!1,c.preventDefault()),k=k.parentNode||k.ownerDocument||k===c.target.ownerDocument&&a}while(k&&!c.isPropagationStopped());if(!c.isDefaultPrevented()){var n,o=f.event.special[h]||{};if((!o._default||o._default.call(e.ownerDocument,c)===!1)&&(h!=="click"||!f.nodeName(e,"a"))&&f.acceptData(e)){try{l&&e[h]&&(n=e[l],n&&(e[l]=null),f.event.triggered=h,e[h]())}catch(p){}n&&(e[l]=n),f.event.triggered=b}}return c.result}},handle:function(c){c=f.event.fix(c||a.event);var d=((f._data(this,"events")||{})[c.type]||[]).slice(0),e=!c.exclusive&&!c.namespace,g=Array.prototype.slice.call(arguments,0);g[0]=c,c.currentTarget=this;for(var h=0,i=d.length;h<i;h++){var j=d[h];if(e||c.namespace_re.test(j.namespace)){c.handler=j.handler,c.data=j.data,c.handleObj=j;var k=j.handler.apply(this,g);k!==b&&(c.result=k,k===!1&&(c.preventDefault(),c.stopPropagation()));if(c.isImmediatePropagationStopped())break}}return c.result},props:"altKey attrChange attrName bubbles button cancelable charCode clientX clientY ctrlKey currentTarget data detail eventPhase fromElement handler keyCode layerX layerY metaKey newValue offsetX offsetY pageX pageY prevValue relatedNode relatedTarget screenX screenY shiftKey srcElement target toElement view wheelDelta which".split(" "),fix:function(a){if(a[f.expando])return a;var d=a;a=f.Event(d);for(var e=this.props.length,g;e;)g=this.props[--e],a[g]=d[g];a.target||(a.target=a.srcElement||c),a.target.nodeType===3&&(a.target=a.target.parentNode),!a.relatedTarget&&a.fromElement&&(a.relatedTarget=a.fromElement===a.target?a.toElement:a.fromElement);if(a.pageX==null&&a.clientX!=null){var h=a.target.ownerDocument||c,i=h.documentElement,j=h.body;a.pageX=a.clientX+(i&&i.scrollLeft||j&&j.scrollLeft||0)-(i&&i.clientLeft||j&&j.clientLeft||0),a.pageY=a.clientY+(i&&i.scrollTop||j&&j.scrollTop||0)-(i&&i.clientTop||j&&j.clientTop||0)}a.which==null&&(a.charCode!=null||a.keyCode!=null)&&(a.which=a.charCode!=null?a.charCode:a.keyCode),!a.metaKey&&a.ctrlKey&&(a.metaKey=a.ctrlKey),!a.which&&a.button!==b&&(a.which=a.button&1?1:a.button&2?3:a.button&4?2:0);return a},guid:1e8,proxy:f.proxy,special:{ready:{setup:f.bindReady,teardown:f.noop},live:{add:function(a){f.event.add(this,M(a.origType,a.selector),f.extend({},a,{handler:L,guid:a.handler.guid}))},remove:function(a){f.event.remove(this,M(a.origType,a.selector),a)}},beforeunload:{setup:function(a,b,c){f.isWindow(this)&&(this.onbeforeunload=c)},teardown:function(a,b){this.onbeforeunload===b&&(this.onbeforeunload=null)}}}},f.removeEvent=c.removeEventListener?function(a,b,c){a.removeEventListener&&a.removeEventListener(b,c,!1)}:function(a,b,c){a.detachEvent&&a.detachEvent("on"+b,c)},f.Event=function(a,b){if(!this.preventDefault)return new f.Event(a,b);a&&a.type?(this.originalEvent=a,this.type=a.type,this.isDefaultPrevented=a.defaultPrevented||a.returnValue===!1||a.getPreventDefault&&a.getPreventDefault()?D:C):this.type=a,b&&f.extend(this,b),this.timeStamp=f.now(),this[f.expando]=!0},f.Event.prototype={preventDefault:function(){this.isDefaultPrevented=D;var a=this.originalEvent;!a||(a.preventDefault?a.preventDefault():a.returnValue=!1)},stopPropagation:function(){this.isPropagationStopped=D;var a=this.originalEvent;!a||(a.stopPropagation&&a.stopPropagation(),a.cancelBubble=!0)},stopImmediatePropagation:function(){this.isImmediatePropagationStopped=D,this.stopPropagation()},isDefaultPrevented:C,isPropagationStopped:C,isImmediatePropagationStopped:C};var E=function(a){var b=a.relatedTarget,c=!1,d=a.type;a.type=a.data,b!==this&&(b&&(c=f.contains(this,b)),c||(f.event.handle.apply(this,arguments),a.type=d))},F=function(a){a.type=a.data,f.event.handle.apply(this,arguments)};f.each({mouseenter:"mouseover",mouseleave:"mouseout"},function(a,b){f.event.special[a]={setup:function(c){f.event.add(this,b,c&&c.selector?F:E,a)},teardown:function(a){f.event.remove(this,b,a&&a.selector?F:E)}}}),f.support.submitBubbles||(f.event.special.submit={setup:function(a,b){if(!f.nodeName(this,"form"))f.event.add(this,"click.specialSubmit",function(a){var b=a.target,c=f.nodeName(b,"input")||f.nodeName(b,"button")?b.type:"";(c==="submit"||c==="image")&&f(b).closest("form").length&&J("submit",this,arguments)}),f.event.add(this,"keypress.specialSubmit",function(a){var b=a.target,c=f.nodeName(b,"input")||f.nodeName(b,"button")?b.type:"";(c==="text"||c==="password")&&f(b).closest("form").length&&a.keyCode===13&&J("submit",this,arguments)});else return!1},teardown:function(a){f.event.remove(this,".specialSubmit")}});if(!f.support.changeBubbles){var G,H=function(a){var b=f.nodeName(a,"input")?a.type:"",c=a.value;b==="radio"||b==="checkbox"?c=a.checked:b==="select-multiple"?c=a.selectedIndex>-1?f.map(a.options,function(a){return a.selected}).join("-"):"":f.nodeName(a,"select")&&(c=a.selectedIndex);return c},I=function(c){var d=c.target,e,g;if(!!x.test(d.nodeName)&&!d.readOnly){e=f._data(d,"_change_data"),g=H(d),(c.type!=="focusout"||d.type!=="radio")&&f._data(d,"_change_data",g);if(e===b||g===e)return;if(e!=null||g)c.type="change",c.liveFired=b,f.event.trigger(c,arguments[1],d)}};f.event.special.change={filters:{focusout:I,beforedeactivate:I,click:function(a){var b=a.target,c=f.nodeName(b,"input")?b.type:"";(c==="radio"||c==="checkbox"||f.nodeName(b,"select"))&&I.call(this,a)},keydown:function(a){var b=a.target,c=f.nodeName(b,"input")?b.type:"";(a.keyCode===13&&!f.nodeName(b,"textarea")||a.keyCode===32&&(c==="checkbox"||c==="radio")||c==="select-multiple")&&I.call(this,a)},beforeactivate:function(a){var b=a.target;f._data(b,"_change_data",H(b))}},setup:function(a,b){if(this.type==="file")return!1;for(var c in G)f.event.add(this,c+".specialChange",G[c]);return x.test(this.nodeName)},teardown:function(a){f.event.remove(this,".specialChange");return x.test(this.nodeName)}},G=f.event.special.change.filters,G.focus=G.beforeactivate}f.support.focusinBubbles||f.each({focus:"focusin",blur:"focusout"},function(a,b){function e(a){var c=f.event.fix(a);c.type=b,c.originalEvent={},f.event.trigger(c,null,c.target),c.isDefaultPrevented()&&a.preventDefault()}var d=0;f.event.special[b]={setup:function(){d++===0&&c.addEventListener(a,e,!0)},teardown:function(){--d===0&&c.removeEventListener(a,e,!0)}}}),f.each(["bind","one"],function(a,c){f.fn[c]=function(a,d,e){var g;if(typeof a=="object"){for(var h in a)this[c](h,d,a[h],e);return this}if(arguments.length===2||d===!1)e=d,d=b;c==="one"?(g=function(a){f(this).unbind(a,g);return e.apply(this,arguments)},g.guid=e.guid||f.guid++):g=e;if(a==="unload"&&c!=="one")this.one(a,d,e);else for(var i=0,j=this.length;i<j;i++)f.event.add(this[i],a,g,d);return this}}),f.fn.extend({unbind:function(a,b){if(typeof a=="object"&&!a.preventDefault)for(var c in a)this.unbind(c,a[c]);else for(var d=0,e=this.length;d<e;d++)f.event.remove(this[d],a,b);return this},delegate:function(a,b,c,d){return this.live(b,c,d,a)},undelegate:function(a,b,c){return arguments.length===0?this.unbind("live"):this.die(b,null,c,a)},trigger:function(a,b){return this.each(function(){f.event.trigger(a,b,this)})},triggerHandler:function(a,b){if(this[0])return f.event.trigger(a,b,this[0],!0)},toggle:function(a){var b=arguments,c=a.guid||f.guid++,d=0,e=function(c){var e=(f.data(this,"lastToggle"+a.guid)||0)%d;f.data(this,"lastToggle"+a.guid,e+1),c.preventDefault();return b[e].apply(this,arguments)||!1};e.guid=c;while(d<b.length)b[d++].guid=c;return this.click(e)},hover:function(a,b){return this.mouseenter(a).mouseleave(b||a)}});var K={focus:"focusin",blur:"focusout",mouseenter:"mouseover",mouseleave:"mouseout"};f.each(["live","die"],function(a,c){f.fn[c]=function(a,d,e,g){var h,i=0,j,k,l,m=g||this.selector,n=g?this:f(this.context);if(typeof a=="object"&&!a.preventDefault){for(var o in a)n[c](o,d,a[o],m);return this}if(c==="die"&&!a&&g&&g.charAt(0)==="."){n.unbind(g);return this}if(d===!1||f.isFunction(d))e=d||C,d=b;a=(a||"").split(" ");while((h=a[i++])!=null){j=w.exec(h),k="",j&&(k=j[0],h=h.replace(w,""));if(h==="hover"){a.push("mouseenter"+k,"mouseleave"+k);continue}l=h,K[h]?(a.push(K[h]+k),h=h+k):h=(K[h]||h)+k;if(c==="live")for(var p=0,q=n.length;p<q;p++)f.event.add(n[p],"live."+M(h,m),{data:d,selector:m,handler:e,origType:h,origHandler:e,preType:l});else n.unbind("live."+M(h,m),e)}return this}}),f.each("blur focus focusin focusout load resize scroll unload click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup error".split(" "),function(a,b){f.fn[b]=function(a,c){c==null&&(c=a,a=null);return arguments.length>0?this.bind(b,a,c):this.trigger(b)},f.attrFn&&(f.attrFn[b]=!0)}),function(){function u(a,b,c,d,e,f){for(var g=0,h=d.length;g<h;g++){var i=d[g];if(i){var j=!1;i=i[a];while(i){if(i.sizcache===c){j=d[i.sizset];break}if(i.nodeType===1){f||(i.sizcache=c,i.sizset=g);if(typeof b!="string"){if(i===b){j=!0;break}}else if(k.filter(b,[i]).length>0){j=i;break}}i=i[a]}d[g]=j}}}function t(a,b,c,d,e,f){for(var g=0,h=d.length;g<h;g++){var i=d[g];if(i){var j=!1;i=i[a];while(i){if(i.sizcache===c){j=d[i.sizset];break}i.nodeType===1&&!f&&(i.sizcache=c,i.sizset=g);if(i.nodeName.toLowerCase()===b){j=i;break}i=i[a]}d[g]=j}}}var a=/((?:\((?:\([^()]+\)|[^()]+)+\)|\[(?:\[[^\[\]]*\]|['"][^'"]*['"]|[^\[\]'"]+)+\]|\\.|[^ >+~,(\[\\]+)+|[>+~])(\s*,\s*)?((?:.|\r|\n)*)/g,d=0,e=Object.prototype.toString,g=!1,h=!0,i=/\\/g,j=/\W/;[0,0].sort(function(){h=!1;return 0});var k=function(b,d,f,g){f=f||[],d=d||c;var h=d;if(d.nodeType!==1&&d.nodeType!==9)return[];if(!b||typeof b!="string")return f;var i,j,n,o,q,r,s,t,u=!0,w=k.isXML(d),x=[],y=b;do{a.exec(""),i=a.exec(y);if(i){y=i[3],x.push(i[1]);if(i[2]){o=i[3];break}}}while(i);if(x.length>1&&m.exec(b))if(x.length===2&&l.relative[x[0]])j=v(x[0]+x[1],d);else{j=l.relative[x[0]]?[d]:k(x.shift(),d);while(x.length)b=x.shift(),l.relative[b]&&(b+=x.shift()),j=v(b,j)}else{!g&&x.length>1&&d.nodeType===9&&!w&&l.match.ID.test(x[0])&&!l.match.ID.test(x[x.length-1])&&(q=k.find(x.shift(),d,w),d=q.expr?k.filter(q.expr,q.set)[0]:q.set[0]);if(d){q=g?{expr:x.pop(),set:p(g)}:k.find(x.pop(),x.length===1&&(x[0]==="~"||x[0]==="+")&&d.parentNode?d.parentNode:d,w),j=q.expr?k.filter(q.expr,q.set):q.set,x.length>0?n=p(j):u=!1;while(x.length)r=x.pop(),s=r,l.relative[r]?s=x.pop():r="",s==null&&(s=d),l.relative[r](n,s,w)}else n=x=[]}n||(n=j),n||k.error(r||b);if(e.call(n)==="[object Array]")if(!u)f.push.apply(f,n);else if(d&&d.nodeType===1)for(t=0;n[t]!=null;t++)n[t]&&(n[t]===!0||n[t].nodeType===1&&k.contains(d,n[t]))&&f.push(j[t]);else for(t=0;n[t]!=null;t++)n[t]&&n[t].nodeType===1&&f.push(j[t]);else p(n,f);o&&(k(o,h,f,g),k.uniqueSort(f));return f};k.uniqueSort=function(a){if(r){g=h,a.sort(r);if(g)for(var b=1;b<a.length;b++)a[b]===a[b-1]&&a.splice(b--,1)}return a},k.matches=function(a,b){return k(a,null,null,b)},k.matchesSelector=function(a,b){return k(b,null,null,[a]).length>0},k.find=function(a,b,c){var d;if(!a)return[];for(var e=0,f=l.order.length;e<f;e++){var g,h=l.order[e];if(g=l.leftMatch[h].exec(a)){var j=g[1];g.splice(1,1);if(j.substr(j.length-1)!=="\\"){g[1]=(g[1]||"").replace(i,""),d=l.find[h](g,b,c);if(d!=null){a=a.replace(l.match[h],"");break}}}}d||(d=typeof b.getElementsByTagName!="undefined"?b.getElementsByTagName("*"):[]);return{set:d,expr:a}},k.filter=function(a,c,d,e){var f,g,h=a,i=[],j=c,m=c&&c[0]&&k.isXML(c[0]);while(a&&c.length){for(var n in l.filter)if((f=l.leftMatch[n].exec(a))!=null&&f[2]){var o,p,q=l.filter[n],r=f[1];g=!1,f.splice(1,1);if(r.substr(r.length-1)==="\\")continue;j===i&&(i=[]);if(l.preFilter[n]){f=l.preFilter[n](f,j,d,i,e,m);if(!f)g=o=!0;else if(f===!0)continue}if(f)for(var s=0;(p=j[s])!=null;s++)if(p){o=q(p,f,s,j);var t=e^!!o;d&&o!=null?t?g=!0:j[s]=!1:t&&(i.push(p),g=!0)}if(o!==b){d||(j=i),a=a.replace(l.match[n],"");if(!g)return[];break}}if(a===h)if(g==null)k.error(a);else break;h=a}return j},k.error=function(a){throw"Syntax error, unrecognized expression: "+a};var l=k.selectors={order:["ID","NAME","TAG"],match:{ID:/#((?:[\w\u00c0-\uFFFF\-]|\\.)+)/,CLASS:/\.((?:[\w\u00c0-\uFFFF\-]|\\.)+)/,NAME:/\[name=['"]*((?:[\w\u00c0-\uFFFF\-]|\\.)+)['"]*\]/,ATTR:/\[\s*((?:[\w\u00c0-\uFFFF\-]|\\.)+)\s*(?:(\S?=)\s*(?:(['"])(.*?)\3|(#?(?:[\w\u00c0-\uFFFF\-]|\\.)*)|)|)\s*\]/,TAG:/^((?:[\w\u00c0-\uFFFF\*\-]|\\.)+)/,CHILD:/:(only|nth|last|first)-child(?:\(\s*(even|odd|(?:[+\-]?\d+|(?:[+\-]?\d*)?n\s*(?:[+\-]\s*\d+)?))\s*\))?/,POS:/:(nth|eq|gt|lt|first|last|even|odd)(?:\((\d*)\))?(?=[^\-]|$)/,PSEUDO:/:((?:[\w\u00c0-\uFFFF\-]|\\.)+)(?:\((['"]?)((?:\([^\)]+\)|[^\(\)]*)+)\2\))?/},leftMatch:{},attrMap:{"class":"className","for":"htmlFor"},attrHandle:{href:function(a){return a.getAttribute("href")},type:function(a){return a.getAttribute("type")}},relative:{"+":function(a,b){var c=typeof b=="string",d=c&&!j.test(b),e=c&&!d;d&&(b=b.toLowerCase());for(var f=0,g=a.length,h;f<g;f++)if(h=a[f]){while((h=h.previousSibling)&&h.nodeType!==1);a[f]=e||h&&h.nodeName.toLowerCase()===b?h||!1:h===b}e&&k.filter(b,a,!0)},">":function(a,b){var c,d=typeof b=="string",e=0,f=a.length;if(d&&!j.test(b)){b=b.toLowerCase();for(;e<f;e++){c=a[e];if(c){var g=c.parentNode;a[e]=g.nodeName.toLowerCase()===b?g:!1}}}else{for(;e<f;e++)c=a[e],c&&(a[e]=d?c.parentNode:c.parentNode===b);d&&k.filter(b,a,!0)}},"":function(a,b,c){var e,f=d++,g=u;typeof b=="string"&&!j.test(b)&&(b=b.toLowerCase(),e=b,g=t),g("parentNode",b,f,a,e,c)},"~":function(a,b,c){var e,f=d++,g=u;typeof b=="string"&&!j.test(b)&&(b=b.toLowerCase(),e=b,g=t),g("previousSibling",b,f,a,e,c)}},find:{ID:function(a,b,c){if(typeof b.getElementById!="undefined"&&!c){var d=b.getElementById(a[1]);return d&&d.parentNode?[d]:[]}},NAME:function(a,b){if(typeof b.getElementsByName!="undefined"){var c=[],d=b.getElementsByName(a[1]);for(var e=0,f=d.length;e<f;e++)d[e].getAttribute("name")===a[1]&&c.push(d[e]);return c.length===0?null:c}},TAG:function(a,b){if(typeof b.getElementsByTagName!="undefined")return b.getElementsByTagName(a[1])}},preFilter:{CLASS:function(a,b,c,d,e,f){a=" "+a[1].replace(i,"")+" ";if(f)return a;for(var g=0,h;(h=b[g])!=null;g++)h&&(e^(h.className&&(" "+h.className+" ").replace(/[\t\n\r]/g," ").indexOf(a)>=0)?c||d.push(h):c&&(b[g]=!1));return!1},ID:function(a){return a[1].replace(i,"")},TAG:function(a,b){return a[1].replace(i,"").toLowerCase()},CHILD:function(a){if(a[1]==="nth"){a[2]||k.error(a[0]),a[2]=a[2].replace(/^\+|\s*/g,"");var b=/(-?)(\d*)(?:n([+\-]?\d*))?/.exec(a[2]==="even"&&"2n"||a[2]==="odd"&&"2n+1"||!/\D/.test(a[2])&&"0n+"+a[2]||a[2]);a[2]=b[1]+(b[2]||1)-0,a[3]=b[3]-0}else a[2]&&k.error(a[0]);a[0]=d++;return a},ATTR:function(a,b,c,d,e,f){var g=a[1]=a[1].replace(i,"");!f&&l.attrMap[g]&&(a[1]=l.attrMap[g]),a[4]=(a[4]||a[5]||"").replace(i,""),a[2]==="~="&&(a[4]=" "+a[4]+" ");return a},PSEUDO:function(b,c,d,e,f){if(b[1]==="not")if((a.exec(b[3])||"").length>1||/^\w/.test(b[3]))b[3]=k(b[3],null,null,c);else{var g=k.filter(b[3],c,d,!0^f);d||e.push.apply(e,g);return!1}else if(l.match.POS.test(b[0])||l.match.CHILD.test(b[0]))return!0;return b},POS:function(a){a.unshift(!0);return a}},filters:{enabled:function(a){return a.disabled===!1&&a.type!=="hidden"},disabled:function(a){return a.disabled===!0},checked:function(a){return a.checked===!0},selected:function(a){a.parentNode&&a.parentNode.selectedIndex;return a.selected===!0},parent:function(a){return!!a.firstChild},empty:function(a){return!a.firstChild},has:function(a,b,c){return!!k(c[3],a).length},header:function(a){return/h\d/i.test(a.nodeName)},text:function(a){var b=a.getAttribute("type"),c=a.type;return a.nodeName.toLowerCase()==="input"&&"text"===c&&(b===c||b===null)},radio:function(a){return a.nodeName.toLowerCase()==="input"&&"radio"===a.type},checkbox:function(a){return a.nodeName.toLowerCase()==="input"&&"checkbox"===a.type},file:function(a){return a.nodeName.toLowerCase()==="input"&&"file"===a.type},password:function(a){return a.nodeName.toLowerCase()==="input"&&"password"===a.type},submit:function(a){var b=a.nodeName.toLowerCase();return(b==="input"||b==="button")&&"submit"===a.type},image:function(a){return a.nodeName.toLowerCase()==="input"&&"image"===a.type},reset:function(a){var b=a.nodeName.toLowerCase();return(b==="input"||b==="button")&&"reset"===a.type},button:function(a){var b=a.nodeName.toLowerCase();return b==="input"&&"button"===a.type||b==="button"},input:function(a){return/input|select|textarea|button/i.test(a.nodeName)},focus:function(a){return a===a.ownerDocument.activeElement}},setFilters:{first:function(a,b){return b===0},last:function(a,b,c,d){return b===d.length-1},even:function(a,b){return b%2===0},odd:function(a,b){return b%2===1},lt:function(a,b,c){return b<c[3]-0},gt:function(a,b,c){return b>c[3]-0},nth:function(a,b,c){return c[3]-0===b},eq:function(a,b,c){return c[3]-0===b}},filter:{PSEUDO:function(a,b,c,d){var e=b[1],f=l.filters[e];if(f)return f(a,c,b,d);if(e==="contains")return(a.textContent||a.innerText||k.getText([a])||"").indexOf(b[3])>=0;if(e==="not"){var g=b[3];for(var h=0,i=g.length;h<i;h++)if(g[h]===a)return!1;return!0}k.error(e)},CHILD:function(a,b){var c=b[1],d=a;switch(c){case"only":case"first":while(d=d.previousSibling)if(d.nodeType===1)return!1;if(c==="first")return!0;d=a;case"last":while(d=d.nextSibling)if(d.nodeType===1)return!1;return!0;case"nth":var e=b[2],f=b[3];if(e===1&&f===0)return!0;var g=b[0],h=a.parentNode;if(h&&(h.sizcache!==g||!a.nodeIndex)){var i=0;for(d=h.firstChild;d;d=d.nextSibling)d.nodeType===1&&(d.nodeIndex=++i);h.sizcache=g}var j=a.nodeIndex-f;return e===0?j===0:j%e===0&&j/e>=0}},ID:function(a,b){return a.nodeType===1&&a.getAttribute("id")===b},TAG:function(a,b){return b==="*"&&a.nodeType===1||a.nodeName.toLowerCase()===b},CLASS:function(a,b){return(" "+(a.className||a.getAttribute("class"))+" ").indexOf(b)>-1},ATTR:function(a,b){var c=b[1],d=l.attrHandle[c]?l.attrHandle[c](a):a[c]!=null?a[c]:a.getAttribute(c),e=d+"",f=b[2],g=b[4];return d==null?f==="!=":f==="="?e===g:f==="*="?e.indexOf(g)>=0:f==="~="?(" "+e+" ").indexOf(g)>=0:g?f==="!="?e!==g:f==="^="?e.indexOf(g)===0:f==="$="?e.substr(e.length-g.length)===g:f==="|="?e===g||e.substr(0,g.length+1)===g+"-":!1:e&&d!==!1},POS:function(a,b,c,d){var e=b[2],f=l.setFilters[e];if(f)return f(a,c,b,d)}}},m=l.match.POS,n=function(a,b){return"\\"+(b-0+1)};for(var o in l.match)l.match[o]=new RegExp(l.match[o].source+/(?![^\[]*\])(?![^\(]*\))/.source),l.leftMatch[o]=new RegExp(/(^(?:.|\r|\n)*?)/.source+l.match[o].source.replace(/\\(\d+)/g,n));var p=function(a,b){a=Array.prototype.slice.call(a,0);if(b){b.push.apply(b,a);return b}return a};try{Array.prototype.slice.call(c.documentElement.childNodes,0)[0].nodeType}catch(q){p=function(a,b){var c=0,d=b||[];if(e.call(a)==="[object Array]")Array.prototype.push.apply(d,a);else if(typeof a.length=="number")for(var f=a.length;c<f;c++)d.push(a[c]);else for(;a[c];c++)d.push(a[c]);return d}}var r,s;c.documentElement.compareDocumentPosition?r=function(a,b){if(a===b){g=!0;return 0}if(!a.compareDocumentPosition||!b.compareDocumentPosition)return a.compareDocumentPosition?-1:1;return a.compareDocumentPosition(b)&4?-1:1}:(r=function(a,b){if(a===b){g=!0;return 0}if(a.sourceIndex&&b.sourceIndex)return a.sourceIndex-b.sourceIndex;var c,d,e=[],f=[],h=a.parentNode,i=b.parentNode,j=h;if(h===i)return s(a,b);if(!h)return-1;if(!i)return 1;while(j)e.unshift(j),j=j.parentNode;j=i;while(j)f.unshift(j),j=j.parentNode;c=e.length,d=f.length;for(var k=0;k<c&&k<d;k++)if(e[k]!==f[k])return s(e[k],f[k]);return k===c?s(a,f[k],-1):s(e[k],b,1)},s=function(a,b,c){if(a===b)return c;var d=a.nextSibling;while(d){if(d===b)return-1;d=d.nextSibling}return 1}),k.getText=function(a){var b="",c;for(var d=0;a[d];d++)c=a[d],c.nodeType===3||c.nodeType===4?b+=c.nodeValue:c.nodeType!==8&&(b+=k.getText(c.childNodes));return b},function(){var a=c.createElement("div"),d="script"+(new Date).getTime(),e=c.documentElement;a.innerHTML="<a name='"+d+"'/>",e.insertBefore(a,e.firstChild),c.getElementById(d)&&(l.find.ID=function(a,c,d){if(typeof c.getElementById!="undefined"&&!d){var e=c.getElementById(a[1]);return e?e.id===a[1]||typeof e.getAttributeNode!="undefined"&&e.getAttributeNode("id").nodeValue===a[1]?[e]:b:[]}},l.filter.ID=function(a,b){var c=typeof a.getAttributeNode!="undefined"&&a.getAttributeNode("id");return a.nodeType===1&&c&&c.nodeValue===b}),e.removeChild(a),e=a=null}(),function(){var a=c.createElement("div");a.appendChild(c.createComment("")),a.getElementsByTagName("*").length>0&&(l.find.TAG=function(a,b){var c=b.getElementsByTagName(a[1]);if(a[1]==="*"){var d=[];for(var e=0;c[e];e++)c[e].nodeType===1&&d.push(c[e]);c=d}return c}),a.innerHTML="<a href='#'></a>",a.firstChild&&typeof a.firstChild.getAttribute!="undefined"&&a.firstChild.getAttribute("href")!=="#"&&(l.attrHandle.href=function(a){return a.getAttribute("href",2)}),a=null}(),c.querySelectorAll&&function(){var a=k,b=c.createElement("div"),d="__sizzle__";b.innerHTML="<p class='TEST'></p>";if(!b.querySelectorAll||b.querySelectorAll(".TEST").length!==0){k=function(b,e,f,g){e=e||c;if(!g&&!k.isXML(e)){var h=/^(\w+$)|^\.([\w\-]+$)|^#([\w\-]+$)/.exec(b);if(h&&(e.nodeType===1||e.nodeType===9)){if(h[1])return p(e.getElementsByTagName(b),f);if(h[2]&&l.find.CLASS&&e.getElementsByClassName)return p(e.getElementsByClassName(h[2]),f)}if(e.nodeType===9){if(b==="body"&&e.body)return p([e.body],f);if(h&&h[3]){var i=e.getElementById(h[3]);if(!i||!i.parentNode)return p([],f);if(i.id===h[3])return p([i],f)}try{return p(e.querySelectorAll(b),f)}catch(j){}}else if(e.nodeType===1&&e.nodeName.toLowerCase()!=="object"){var m=e,n=e.getAttribute("id"),o=n||d,q=e.parentNode,r=/^\s*[+~]/.test(b);n?o=o.replace(/'/g,"\\$&"):e.setAttribute("id",o),r&&q&&(e=e.parentNode);try{if(!r||q)return p(e.querySelectorAll("[id='"+o+"'] "+b),f)}catch(s){}finally{n||m.removeAttribute("id")}}}return a(b,e,f,g)};for(var e in a)k[e]=a[e];b=null}}(),function(){var a=c.documentElement,b=a.matchesSelector||a.mozMatchesSelector||a.webkitMatchesSelector||a.msMatchesSelector;if(b){var d=!b.call(c.createElement("div"),"div"),e=!1;try{b.call(c.documentElement,"[test!='']:sizzle")}catch(f){e=!0}k.matchesSelector=function(a,c){c=c.replace(/\=\s*([^'"\]]*)\s*\]/g,"='$1']");if(!k.isXML(a))try{if(e||!l.match.PSEUDO.test(c)&&!/!=/.test(c)){var f=b.call(a,c);if(f||!d||a.document&&a.document.nodeType!==11)return f}}catch(g){}return k(c,null,null,[a]).length>0}}}(),function(){var a=c.createElement("div");a.innerHTML="<div class='test e'></div><div class='test'></div>";if(!!a.getElementsByClassName&&a.getElementsByClassName("e").length!==0){a.lastChild.className="e";if(a.getElementsByClassName("e").length===1)return;l.order.splice(1,0,"CLASS"),l.find.CLASS=function(a,b,c){if(typeof b.getElementsByClassName!="undefined"&&!c)return b.getElementsByClassName(a[1])},a=null}}(),c.documentElement.contains?k.contains=function(a,b){return a!==b&&(a.contains?a.contains(b):!0)}:c.documentElement.compareDocumentPosition?k.contains=function(a,b){return!!(a.compareDocumentPosition(b)&16)}:k.contains=function(){return!1},k.isXML=function(a){var b=(a?a.ownerDocument||a:0).documentElement;return b?b.nodeName!=="HTML":!1};var v=function(a,b){var c,d=[],e="",f=b.nodeType?[b]:b;while(c=l.match.PSEUDO.exec(a))e+=c[0],a=a.replace(l.match.PSEUDO,"");a=l.relative[a]?a+"*":a;for(var g=0,h=f.length;g<h;g++)k(a,f[g],d);return k.filter(e,d)};f.find=k,f.expr=k.selectors,f.expr[":"]=f.expr.filters,f.unique=k.uniqueSort,f.text=k.getText,f.isXMLDoc=k.isXML,f.contains=k.contains}();var N=/Until$/,O=/^(?:parents|prevUntil|prevAll)/,P=/,/,Q=/^.[^:#\[\.,]*$/,R=Array.prototype.slice,S=f.expr.match.POS,T={children:!0,contents:!0,next:!0,prev:!0};f.fn.extend({find:function(a){var b=this,c,d;if(typeof a!="string")return f(a).filter(function(){for(c=0,d=b.length;c<d;c++)if(f.contains(b[c],this))return!0});var e=this.pushStack("","find",a),g,h,i;for(c=0,d=this.length;c<d;c++){g=e.length,f.find(a,this[c],e);if(c>0)for(h=g;h<e.length;h++)for(i=0;i<g;i++)if(e[i]===e[h]){e.splice(h--,1);break}}return e},has:function(a){var b=f(a);return this.filter(function(){for(var a=0,c=b.length;a<c;a++)if(f.contains(this,b[a]))return!0})},not:function(a){return this.pushStack(V(this,a,!1),"not",a)},filter:function(a){return this.pushStack(V(this,a,!0),"filter",a)},is:function(a){return!!a&&(typeof a=="string"?f.filter(a,this).length>0:this.filter(a).length>0)},closest:function(a,b){var c=[],d,e,g=this[0];if(f.isArray(a)){var h,i,j={},k=1;if(g&&a.length){for(d=0,e=a.length;d<e;d++)i=a[d],j[i]||(j[i]=S.test(i)?f(i,b||this.context):i);while(g&&g.ownerDocument&&g!==b){for(i in j)h=j[i],(h.jquery?h.index(g)>-1:f(g).is(h))&&c.push({selector:i,elem:g,level:k});g=g.parentNode,k++}}return c}var l=S.test(a)||typeof a!="string"?f(a,b||this.context):0;for(d=0,e=this.length;d<e;d++){g=this[d];while(g){if(l?l.index(g)>-1:f.find.matchesSelector(g,a)){c.push(g);break}g=g.parentNode;if(!g||!g.ownerDocument||g===b||g.nodeType===11)break}}c=c.length>1?f.unique(c):c;return this.pushStack(c,"closest",a)},index:function(a){if(!a)return this[0]&&this[0].parentNode?this.prevAll().length:-1;if(typeof a=="string")return f.inArray(this[0],f(a));return f.inArray(a.jquery?a[0]:a,this)},add:function(a,b){var c=typeof a=="string"?f(a,b):f.makeArray(a&&a.nodeType?[a]:a),d=f.merge(this.get(),c);return this.pushStack(U(c[0])||U(d[0])?d:f.unique(d))},andSelf:function(){return this.add(this.prevObject)}}),f.each({parent:function(a){var b=a.parentNode;return b&&b.nodeType!==11?b:null},parents:function(a){return f.dir(a,"parentNode")},parentsUntil:function(a,b,c){return f.dir(a,"parentNode",c)},next:function(a){return f.nth(a,2,"nextSibling")},prev:function(a){return f.nth(a,2,"previousSibling")},nextAll:function(a){return f.dir(a,"nextSibling")},prevAll:function(a){return f.dir(a,"previousSibling")},nextUntil:function(a,b,c){return f.dir(a,"nextSibling",c)},prevUntil:function(a,b,c){return f.dir(a,"previousSibling",c)},siblings:function(a){return f.sibling(a.parentNode.firstChild,a)},children:function(a){return f.sibling(a.firstChild)},contents:function(a){return f.nodeName(a,"iframe")?a.contentDocument||a.contentWindow.document:f.makeArray(a.childNodes)}},function(a,b){f.fn[a]=function(c,d){var e=f.map(this,b,c),g=R.call(arguments);N.test(a)||(d=c),d&&typeof d=="string"&&(e=f.filter(d,e)),e=this.length>1&&!T[a]?f.unique(e):e,(this.length>1||P.test(d))&&O.test(a)&&(e=e.reverse());return this.pushStack(e,a,g.join(","))}}),f.extend({filter:function(a,b,c){c&&(a=":not("+a+")");return b.length===1?f.find.matchesSelector(b[0],a)?[b[0]]:[]:f.find.matches(a,b)},dir:function(a,c,d){var e=[],g=a[c];while(g&&g.nodeType!==9&&(d===b||g.nodeType!==1||!f(g).is(d)))g.nodeType===1&&e.push(g),g=g[c];return e},nth:function(a,b,c,d){b=b||1;var e=0;for(;a;a=a[c])if(a.nodeType===1&&++e===b)break;return a},sibling:function(a,b){var c=[];for(;a;a=a.nextSibling)a.nodeType===1&&a!==b&&c.push(a);return c}});var W=/ jQuery\d+="(?:\d+|null)"/g,X=/^\s+/,Y=/<(?!area|br|col|embed|hr|img|input|link|meta|param)(([\w:]+)[^>]*)\/>/ig,Z=/<([\w:]+)/,$=/<tbody/i,_=/<|&#?\w+;/,ba=/<(?:script|object|embed|option|style)/i,bb=/checked\s*(?:[^=]|=\s*.checked.)/i,bc=/\/(java|ecma)script/i,bd=/^\s*<!(?:\[CDATA\[|\-\-)/,be={option:[1,"<select multiple='multiple'>","</select>"],legend:[1,"<fieldset>","</fieldset>"],thead:[1,"<table>","</table>"],tr:[2,"<table><tbody>","</tbody></table>"],td:[3,"<table><tbody><tr>","</tr></tbody></table>"],col:[2,"<table><tbody></tbody><colgroup>","</colgroup></table>"],area:[1,"<map>","</map>"],_default:[0,"",""]};be.optgroup=be.option,be.tbody=be.tfoot=be.colgroup=be.caption=be.thead,be.th=be.td,f.support.htmlSerialize||(be._default=[1,"div<div>","</div>"]),f.fn.extend({text:function(a){if(f.isFunction(a))return this.each(function(b){var c=f(this);c.text(a.call(this,b,c.text()))});if(typeof a!="object"&&a!==b)return this.empty().append((this[0]&&this[0].ownerDocument||c).createTextNode(a));return f.text(this)},wrapAll:function(a){if(f.isFunction(a))return this.each(function(b){f(this).wrapAll(a.call(this,b))});if(this[0]){var b=f(a,this[0].ownerDocument).eq(0).clone(!0);this[0].parentNode&&b.insertBefore(this[0]),b.map(function(){var a=this;while(a.firstChild&&a.firstChild.nodeType===1)a=a.firstChild;return a}).append(this)}return this},wrapInner:function(a){if(f.isFunction(a))return this.each(function(b){f(this).wrapInner(a.call(this,b))});return this.each(function(){var b=f(this),c=b.contents();c.length?c.wrapAll(a):b.append(a)})},wrap:function(a){return this.each(function(){f(this).wrapAll(a)})},unwrap:function(){return this.parent().each(function(){f.nodeName(this,"body")||f(this).replaceWith(this.childNodes)}).end()},append:function(){return this.domManip(arguments,!0,function(a){this.nodeType===1&&this.appendChild(a)})},prepend:function(){return this.domManip(arguments,!0,function(a){this.nodeType===1&&this.insertBefore(a,this.firstChild)})},before:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,!1,function(a){this.parentNode.insertBefore(a,this)});if(arguments.length){var a=f(arguments[0]);a.push.apply(a,this.toArray());return this.pushStack(a,"before",arguments)}},after:function(){if(this[0]&&this[0].parentNode)return this.domManip(arguments,!1,function(a){this.parentNode.insertBefore(a,this.nextSibling)});if(arguments.length){var a=this.pushStack(this,"after",arguments);a.push.apply(a,f(arguments[0]).toArray());return a}},remove:function(a,b){for(var c=0,d;(d=this[c])!=null;c++)if(!a||f.filter(a,[d]).length)!b&&d.nodeType===1&&(f.cleanData(d.getElementsByTagName("*")),f.cleanData([d])),d.parentNode&&d.parentNode.removeChild(d);return this},empty:function(){for(var a=0,b;(b=this[a])!=null;a++){b.nodeType===1&&f.cleanData(b.getElementsByTagName("*"));while(b.firstChild)b.removeChild(b.firstChild)}return this},clone:function(a,b){a=a==null?!1:a,b=b==null?a:b;return this.map(function(){return f.clone(this,a,b)})},html:function(a){if(a===b)return this[0]&&this[0].nodeType===1?this[0].innerHTML.replace(W,""):null;if(typeof a=="string"&&!ba.test(a)&&(f.support.leadingWhitespace||!X.test(a))&&!be[(Z.exec(a)||["",""])[1].toLowerCase()]){a=a.replace(Y,"<$1></$2>");try{for(var c=0,d=this.length;c<d;c++)this[c].nodeType===1&&(f.cleanData(this[c].getElementsByTagName("*")),this[c].innerHTML=a)}catch(e){this.empty().append(a)}}else f.isFunction(a)?this.each(function(b){var c=f(this);c.html(a.call(this,b,c.html()))}):this.empty().append(a);return this},replaceWith:function(a){if(this[0]&&this[0].parentNode){if(f.isFunction(a))return this.each(function(b){var c=f(this),d=c.html();c.replaceWith(a.call(this,b,d))});typeof a!="string"&&(a=f(a).detach());return this.each(function(){var b=this.nextSibling,c=this.parentNode;f(this).remove(),b?f(b).before(a):f(c).append(a)})}return this.length?this.pushStack(f(f.isFunction(a)?a():a),"replaceWith",a):this},detach:function(a){return this.remove(a,!0)},domManip:function(a,c,d){var e,g,h,i,j=a[0],k=[];if(!f.support.checkClone&&arguments.length===3&&typeof j=="string"&&bb.test(j))return this.each(function(){f(this).domManip(a,c,d,!0)});if(f.isFunction(j))return this.each(function(e){var g=f(this);a[0]=j.call(this,e,c?g.html():b),g.domManip(a,c,d)});if(this[0]){i=j&&j.parentNode,f.support.parentNode&&i&&i.nodeType===11&&i.childNodes.length===this.length?e={fragment:i}:e=f.buildFragment(a,this,k),h=e.fragment,h.childNodes.length===1?g=h=h.firstChild:g=h.firstChild;if(g){c=c&&f.nodeName(g,"tr");for(var l=0,m=this.length,n=m-1;l<m;l++)d.call(c?bf(this[l],g):this[l],e.cacheable||m>1&&l<n?f.clone(h,!0,!0):h)}k.length&&f.each(k,bl)}return this}}),f.buildFragment=function(a,b,d){var e,g,h,i;b&&b[0]&&(i=b[0].ownerDocument||b[0]),i.createDocumentFragment||(i=c),a.length===1&&typeof a[0]=="string"&&a[0].length<512&&i===c&&a[0].charAt(0)==="<"&&!ba.test(a[0])&&(f.support.checkClone||!bb.test(a[0]))&&(g=!0,h=f.fragments[a[0]],h&&h!==1&&(e=h)),e||(e=i.createDocumentFragment(),f.clean
+(a,i,e,d)),g&&(f.fragments[a[0]]=h?e:1);return{fragment:e,cacheable:g}},f.fragments={},f.each({appendTo:"append",prependTo:"prepend",insertBefore:"before",insertAfter:"after",replaceAll:"replaceWith"},function(a,b){f.fn[a]=function(c){var d=[],e=f(c),g=this.length===1&&this[0].parentNode;if(g&&g.nodeType===11&&g.childNodes.length===1&&e.length===1){e[b](this[0]);return this}for(var h=0,i=e.length;h<i;h++){var j=(h>0?this.clone(!0):this).get();f(e[h])[b](j),d=d.concat(j)}return this.pushStack(d,a,e.selector)}}),f.extend({clone:function(a,b,c){var d=a.cloneNode(!0),e,g,h;if((!f.support.noCloneEvent||!f.support.noCloneChecked)&&(a.nodeType===1||a.nodeType===11)&&!f.isXMLDoc(a)){bh(a,d),e=bi(a),g=bi(d);for(h=0;e[h];++h)g[h]&&bh(e[h],g[h])}if(b){bg(a,d);if(c){e=bi(a),g=bi(d);for(h=0;e[h];++h)bg(e[h],g[h])}}e=g=null;return d},clean:function(a,b,d,e){var g;b=b||c,typeof b.createElement=="undefined"&&(b=b.ownerDocument||b[0]&&b[0].ownerDocument||c);var h=[],i;for(var j=0,k;(k=a[j])!=null;j++){typeof k=="number"&&(k+="");if(!k)continue;if(typeof k=="string")if(!_.test(k))k=b.createTextNode(k);else{k=k.replace(Y,"<$1></$2>");var l=(Z.exec(k)||["",""])[1].toLowerCase(),m=be[l]||be._default,n=m[0],o=b.createElement("div");o.innerHTML=m[1]+k+m[2];while(n--)o=o.lastChild;if(!f.support.tbody){var p=$.test(k),q=l==="table"&&!p?o.firstChild&&o.firstChild.childNodes:m[1]==="<table>"&&!p?o.childNodes:[];for(i=q.length-1;i>=0;--i)f.nodeName(q[i],"tbody")&&!q[i].childNodes.length&&q[i].parentNode.removeChild(q[i])}!f.support.leadingWhitespace&&X.test(k)&&o.insertBefore(b.createTextNode(X.exec(k)[0]),o.firstChild),k=o.childNodes}var r;if(!f.support.appendChecked)if(k[0]&&typeof (r=k.length)=="number")for(i=0;i<r;i++)bk(k[i]);else bk(k);k.nodeType?h.push(k):h=f.merge(h,k)}if(d){g=function(a){return!a.type||bc.test(a.type)};for(j=0;h[j];j++)if(e&&f.nodeName(h[j],"script")&&(!h[j].type||h[j].type.toLowerCase()==="text/javascript"))e.push(h[j].parentNode?h[j].parentNode.removeChild(h[j]):h[j]);else{if(h[j].nodeType===1){var s=f.grep(h[j].getElementsByTagName("script"),g);h.splice.apply(h,[j+1,0].concat(s))}d.appendChild(h[j])}}return h},cleanData:function(a){var b,c,d=f.cache,e=f.expando,g=f.event.special,h=f.support.deleteExpando;for(var i=0,j;(j=a[i])!=null;i++){if(j.nodeName&&f.noData[j.nodeName.toLowerCase()])continue;c=j[f.expando];if(c){b=d[c]&&d[c][e];if(b&&b.events){for(var k in b.events)g[k]?f.event.remove(j,k):f.removeEvent(j,k,b.handle);b.handle&&(b.handle.elem=null)}h?delete j[f.expando]:j.removeAttribute&&j.removeAttribute(f.expando),delete d[c]}}}});var bm=/alpha\([^)]*\)/i,bn=/opacity=([^)]*)/,bo=/([A-Z]|^ms)/g,bp=/^-?\d+(?:px)?$/i,bq=/^-?\d/,br=/^([\-+])=([\-+.\de]+)/,bs={position:"absolute",visibility:"hidden",display:"block"},bt=["Left","Right"],bu=["Top","Bottom"],bv,bw,bx;f.fn.css=function(a,c){if(arguments.length===2&&c===b)return this;return f.access(this,a,c,!0,function(a,c,d){return d!==b?f.style(a,c,d):f.css(a,c)})},f.extend({cssHooks:{opacity:{get:function(a,b){if(b){var c=bv(a,"opacity","opacity");return c===""?"1":c}return a.style.opacity}}},cssNumber:{fillOpacity:!0,fontWeight:!0,lineHeight:!0,opacity:!0,orphans:!0,widows:!0,zIndex:!0,zoom:!0},cssProps:{"float":f.support.cssFloat?"cssFloat":"styleFloat"},style:function(a,c,d,e){if(!!a&&a.nodeType!==3&&a.nodeType!==8&&!!a.style){var g,h,i=f.camelCase(c),j=a.style,k=f.cssHooks[i];c=f.cssProps[i]||i;if(d===b){if(k&&"get"in k&&(g=k.get(a,!1,e))!==b)return g;return j[c]}h=typeof d,h==="string"&&(g=br.exec(d))&&(d=+(g[1]+1)*+g[2]+parseFloat(f.css(a,c)),h="number");if(d==null||h==="number"&&isNaN(d))return;h==="number"&&!f.cssNumber[i]&&(d+="px");if(!k||!("set"in k)||(d=k.set(a,d))!==b)try{j[c]=d}catch(l){}}},css:function(a,c,d){var e,g;c=f.camelCase(c),g=f.cssHooks[c],c=f.cssProps[c]||c,c==="cssFloat"&&(c="float");if(g&&"get"in g&&(e=g.get(a,!0,d))!==b)return e;if(bv)return bv(a,c)},swap:function(a,b,c){var d={};for(var e in b)d[e]=a.style[e],a.style[e]=b[e];c.call(a);for(e in b)a.style[e]=d[e]}}),f.curCSS=f.css,f.each(["height","width"],function(a,b){f.cssHooks[b]={get:function(a,c,d){var e;if(c){if(a.offsetWidth!==0)return by(a,b,d);f.swap(a,bs,function(){e=by(a,b,d)});return e}},set:function(a,b){if(!bp.test(b))return b;b=parseFloat(b);if(b>=0)return b+"px"}}}),f.support.opacity||(f.cssHooks.opacity={get:function(a,b){return bn.test((b&&a.currentStyle?a.currentStyle.filter:a.style.filter)||"")?parseFloat(RegExp.$1)/100+"":b?"1":""},set:function(a,b){var c=a.style,d=a.currentStyle,e=f.isNaN(b)?"":"alpha(opacity="+b*100+")",g=d&&d.filter||c.filter||"";c.zoom=1;if(b>=1&&f.trim(g.replace(bm,""))===""){c.removeAttribute("filter");if(d&&!d.filter)return}c.filter=bm.test(g)?g.replace(bm,e):g+" "+e}}),f(function(){f.support.reliableMarginRight||(f.cssHooks.marginRight={get:function(a,b){var c;f.swap(a,{display:"inline-block"},function(){b?c=bv(a,"margin-right","marginRight"):c=a.style.marginRight});return c}})}),c.defaultView&&c.defaultView.getComputedStyle&&(bw=function(a,c){var d,e,g;c=c.replace(bo,"-$1").toLowerCase();if(!(e=a.ownerDocument.defaultView))return b;if(g=e.getComputedStyle(a,null))d=g.getPropertyValue(c),d===""&&!f.contains(a.ownerDocument.documentElement,a)&&(d=f.style(a,c));return d}),c.documentElement.currentStyle&&(bx=function(a,b){var c,d=a.currentStyle&&a.currentStyle[b],e=a.runtimeStyle&&a.runtimeStyle[b],f=a.style;!bp.test(d)&&bq.test(d)&&(c=f.left,e&&(a.runtimeStyle.left=a.currentStyle.left),f.left=b==="fontSize"?"1em":d||0,d=f.pixelLeft+"px",f.left=c,e&&(a.runtimeStyle.left=e));return d===""?"auto":d}),bv=bw||bx,f.expr&&f.expr.filters&&(f.expr.filters.hidden=function(a){var b=a.offsetWidth,c=a.offsetHeight;return b===0&&c===0||!f.support.reliableHiddenOffsets&&(a.style.display||f.css(a,"display"))==="none"},f.expr.filters.visible=function(a){return!f.expr.filters.hidden(a)});var bz=/%20/g,bA=/\[\]$/,bB=/\r?\n/g,bC=/#.*$/,bD=/^(.*?):[ \t]*([^\r\n]*)\r?$/mg,bE=/^(?:color|date|datetime|datetime-local|email|hidden|month|number|password|range|search|tel|text|time|url|week)$/i,bF=/^(?:about|app|app\-storage|.+\-extension|file|res|widget):$/,bG=/^(?:GET|HEAD)$/,bH=/^\/\//,bI=/\?/,bJ=/<script\b[^<]*(?:(?!<\/script>)<[^<]*)*<\/script>/gi,bK=/^(?:select|textarea)/i,bL=/\s+/,bM=/([?&])_=[^&]*/,bN=/^([\w\+\.\-]+:)(?:\/\/([^\/?#:]*)(?::(\d+))?)?/,bO=f.fn.load,bP={},bQ={},bR,bS,bT=["*/"]+["*"];try{bR=e.href}catch(bU){bR=c.createElement("a"),bR.href="",bR=bR.href}bS=bN.exec(bR.toLowerCase())||[],f.fn.extend({load:function(a,c,d){if(typeof a!="string"&&bO)return bO.apply(this,arguments);if(!this.length)return this;var e=a.indexOf(" ");if(e>=0){var g=a.slice(e,a.length);a=a.slice(0,e)}var h="GET";c&&(f.isFunction(c)?(d=c,c=b):typeof c=="object"&&(c=f.param(c,f.ajaxSettings.traditional),h="POST"));var i=this;f.ajax({url:a,type:h,dataType:"html",data:c,complete:function(a,b,c){c=a.responseText,a.isResolved()&&(a.done(function(a){c=a}),i.html(g?f("<div>").append(c.replace(bJ,"")).find(g):c)),d&&i.each(d,[c,b,a])}});return this},serialize:function(){return f.param(this.serializeArray())},serializeArray:function(){return this.map(function(){return this.elements?f.makeArray(this.elements):this}).filter(function(){return this.name&&!this.disabled&&(this.checked||bK.test(this.nodeName)||bE.test(this.type))}).map(function(a,b){var c=f(this).val();return c==null?null:f.isArray(c)?f.map(c,function(a,c){return{name:b.name,value:a.replace(bB,"\r\n")}}):{name:b.name,value:c.replace(bB,"\r\n")}}).get()}}),f.each("ajaxStart ajaxStop ajaxComplete ajaxError ajaxSuccess ajaxSend".split(" "),function(a,b){f.fn[b]=function(a){return this.bind(b,a)}}),f.each(["get","post"],function(a,c){f[c]=function(a,d,e,g){f.isFunction(d)&&(g=g||e,e=d,d=b);return f.ajax({type:c,url:a,data:d,success:e,dataType:g})}}),f.extend({getScript:function(a,c){return f.get(a,b,c,"script")},getJSON:function(a,b,c){return f.get(a,b,c,"json")},ajaxSetup:function(a,b){b?bX(a,f.ajaxSettings):(b=a,a=f.ajaxSettings),bX(a,b);return a},ajaxSettings:{url:bR,isLocal:bF.test(bS[1]),global:!0,type:"GET",contentType:"application/x-www-form-urlencoded",processData:!0,async:!0,accepts:{xml:"application/xml, text/xml",html:"text/html",text:"text/plain",json:"application/json, text/javascript","*":bT},contents:{xml:/xml/,html:/html/,json:/json/},responseFields:{xml:"responseXML",text:"responseText"},converters:{"* text":a.String,"text html":!0,"text json":f.parseJSON,"text xml":f.parseXML},flatOptions:{context:!0,url:!0}},ajaxPrefilter:bV(bP),ajaxTransport:bV(bQ),ajax:function(a,c){function w(a,c,l,m){if(s!==2){s=2,q&&clearTimeout(q),p=b,n=m||"",v.readyState=a>0?4:0;var o,r,u,w=c,x=l?bZ(d,v,l):b,y,z;if(a>=200&&a<300||a===304){if(d.ifModified){if(y=v.getResponseHeader("Last-Modified"))f.lastModified[k]=y;if(z=v.getResponseHeader("Etag"))f.etag[k]=z}if(a===304)w="notmodified",o=!0;else try{r=b$(d,x),w="success",o=!0}catch(A){w="parsererror",u=A}}else{u=w;if(!w||a)w="error",a<0&&(a=0)}v.status=a,v.statusText=""+(c||w),o?h.resolveWith(e,[r,w,v]):h.rejectWith(e,[v,w,u]),v.statusCode(j),j=b,t&&g.trigger("ajax"+(o?"Success":"Error"),[v,d,o?r:u]),i.resolveWith(e,[v,w]),t&&(g.trigger("ajaxComplete",[v,d]),--f.active||f.event.trigger("ajaxStop"))}}typeof a=="object"&&(c=a,a=b),c=c||{};var d=f.ajaxSetup({},c),e=d.context||d,g=e!==d&&(e.nodeType||e instanceof f)?f(e):f.event,h=f.Deferred(),i=f._Deferred(),j=d.statusCode||{},k,l={},m={},n,o,p,q,r,s=0,t,u,v={readyState:0,setRequestHeader:function(a,b){if(!s){var c=a.toLowerCase();a=m[c]=m[c]||a,l[a]=b}return this},getAllResponseHeaders:function(){return s===2?n:null},getResponseHeader:function(a){var c;if(s===2){if(!o){o={};while(c=bD.exec(n))o[c[1].toLowerCase()]=c[2]}c=o[a.toLowerCase()]}return c===b?null:c},overrideMimeType:function(a){s||(d.mimeType=a);return this},abort:function(a){a=a||"abort",p&&p.abort(a),w(0,a);return this}};h.promise(v),v.success=v.done,v.error=v.fail,v.complete=i.done,v.statusCode=function(a){if(a){var b;if(s<2)for(b in a)j[b]=[j[b],a[b]];else b=a[v.status],v.then(b,b)}return this},d.url=((a||d.url)+"").replace(bC,"").replace(bH,bS[1]+"//"),d.dataTypes=f.trim(d.dataType||"*").toLowerCase().split(bL),d.crossDomain==null&&(r=bN.exec(d.url.toLowerCase()),d.crossDomain=!(!r||r[1]==bS[1]&&r[2]==bS[2]&&(r[3]||(r[1]==="http:"?80:443))==(bS[3]||(bS[1]==="http:"?80:443)))),d.data&&d.processData&&typeof d.data!="string"&&(d.data=f.param(d.data,d.traditional)),bW(bP,d,c,v);if(s===2)return!1;t=d.global,d.type=d.type.toUpperCase(),d.hasContent=!bG.test(d.type),t&&f.active++===0&&f.event.trigger("ajaxStart");if(!d.hasContent){d.data&&(d.url+=(bI.test(d.url)?"&":"?")+d.data,delete d.data),k=d.url;if(d.cache===!1){var x=f.now(),y=d.url.replace(bM,"$1_="+x);d.url=y+(y===d.url?(bI.test(d.url)?"&":"?")+"_="+x:"")}}(d.data&&d.hasContent&&d.contentType!==!1||c.contentType)&&v.setRequestHeader("Content-Type",d.contentType),d.ifModified&&(k=k||d.url,f.lastModified[k]&&v.setRequestHeader("If-Modified-Since",f.lastModified[k]),f.etag[k]&&v.setRequestHeader("If-None-Match",f.etag[k])),v.setRequestHeader("Accept",d.dataTypes[0]&&d.accepts[d.dataTypes[0]]?d.accepts[d.dataTypes[0]]+(d.dataTypes[0]!=="*"?", "+bT+"; q=0.01":""):d.accepts["*"]);for(u in d.headers)v.setRequestHeader(u,d.headers[u]);if(d.beforeSend&&(d.beforeSend.call(e,v,d)===!1||s===2)){v.abort();return!1}for(u in{success:1,error:1,complete:1})v[u](d[u]);p=bW(bQ,d,c,v);if(!p)w(-1,"No Transport");else{v.readyState=1,t&&g.trigger("ajaxSend",[v,d]),d.async&&d.timeout>0&&(q=setTimeout(function(){v.abort("timeout")},d.timeout));try{s=1,p.send(l,w)}catch(z){s<2?w(-1,z):f.error(z)}}return v},param:function(a,c){var d=[],e=function(a,b){b=f.isFunction(b)?b():b,d[d.length]=encodeURIComponent(a)+"="+encodeURIComponent(b)};c===b&&(c=f.ajaxSettings.traditional);if(f.isArray(a)||a.jquery&&!f.isPlainObject(a))f.each(a,function(){e(this.name,this.value)});else for(var g in a)bY(g,a[g],c,e);return d.join("&").replace(bz,"+")}}),f.extend({active:0,lastModified:{},etag:{}});var b_=f.now(),ca=/(\=)\?(&|$)|\?\?/i;f.ajaxSetup({jsonp:"callback",jsonpCallback:function(){return f.expando+"_"+b_++}}),f.ajaxPrefilter("json jsonp",function(b,c,d){var e=b.contentType==="application/x-www-form-urlencoded"&&typeof b.data=="string";if(b.dataTypes[0]==="jsonp"||b.jsonp!==!1&&(ca.test(b.url)||e&&ca.test(b.data))){var g,h=b.jsonpCallback=f.isFunction(b.jsonpCallback)?b.jsonpCallback():b.jsonpCallback,i=a[h],j=b.url,k=b.data,l="$1"+h+"$2";b.jsonp!==!1&&(j=j.replace(ca,l),b.url===j&&(e&&(k=k.replace(ca,l)),b.data===k&&(j+=(/\?/.test(j)?"&":"?")+b.jsonp+"="+h))),b.url=j,b.data=k,a[h]=function(a){g=[a]},d.always(function(){a[h]=i,g&&f.isFunction(i)&&a[h](g[0])}),b.converters["script json"]=function(){g||f.error(h+" was not called");return g[0]},b.dataTypes[0]="json";return"script"}}),f.ajaxSetup({accepts:{script:"text/javascript, application/javascript, application/ecmascript, application/x-ecmascript"},contents:{script:/javascript|ecmascript/},converters:{"text script":function(a){f.globalEval(a);return a}}}),f.ajaxPrefilter("script",function(a){a.cache===b&&(a.cache=!1),a.crossDomain&&(a.type="GET",a.global=!1)}),f.ajaxTransport("script",function(a){if(a.crossDomain){var d,e=c.head||c.getElementsByTagName("head")[0]||c.documentElement;return{send:function(f,g){d=c.createElement("script"),d.async="async",a.scriptCharset&&(d.charset=a.scriptCharset),d.src=a.url,d.onload=d.onreadystatechange=function(a,c){if(c||!d.readyState||/loaded|complete/.test(d.readyState))d.onload=d.onreadystatechange=null,e&&d.parentNode&&e.removeChild(d),d=b,c||g(200,"success")},e.insertBefore(d,e.firstChild)},abort:function(){d&&d.onload(0,1)}}}});var cb=a.ActiveXObject?function(){for(var a in cd)cd[a](0,1)}:!1,cc=0,cd;f.ajaxSettings.xhr=a.ActiveXObject?function(){return!this.isLocal&&ce()||cf()}:ce,function(a){f.extend(f.support,{ajax:!!a,cors:!!a&&"withCredentials"in a})}(f.ajaxSettings.xhr()),f.support.ajax&&f.ajaxTransport(function(c){if(!c.crossDomain||f.support.cors){var d;return{send:function(e,g){var h=c.xhr(),i,j;c.username?h.open(c.type,c.url,c.async,c.username,c.password):h.open(c.type,c.url,c.async);if(c.xhrFields)for(j in c.xhrFields)h[j]=c.xhrFields[j];c.mimeType&&h.overrideMimeType&&h.overrideMimeType(c.mimeType),!c.crossDomain&&!e["X-Requested-With"]&&(e["X-Requested-With"]="XMLHttpRequest");try{for(j in e)h.setRequestHeader(j,e[j])}catch(k){}h.send(c.hasContent&&c.data||null),d=function(a,e){var j,k,l,m,n;try{if(d&&(e||h.readyState===4)){d=b,i&&(h.onreadystatechange=f.noop,cb&&delete cd[i]);if(e)h.readyState!==4&&h.abort();else{j=h.status,l=h.getAllResponseHeaders(),m={},n=h.responseXML,n&&n.documentElement&&(m.xml=n),m.text=h.responseText;try{k=h.statusText}catch(o){k=""}!j&&c.isLocal&&!c.crossDomain?j=m.text?200:404:j===1223&&(j=204)}}}catch(p){e||g(-1,p)}m&&g(j,k,m,l)},!c.async||h.readyState===4?d():(i=++cc,cb&&(cd||(cd={},f(a).unload(cb)),cd[i]=d),h.onreadystatechange=d)},abort:function(){d&&d(0,1)}}}});var cg={},ch,ci,cj=/^(?:toggle|show|hide)$/,ck=/^([+\-]=)?([\d+.\-]+)([a-z%]*)$/i,cl,cm=[["height","marginTop","marginBottom","paddingTop","paddingBottom"],["width","marginLeft","marginRight","paddingLeft","paddingRight"],["opacity"]],cn;f.fn.extend({show:function(a,b,c){var d,e;if(a||a===0)return this.animate(cq("show",3),a,b,c);for(var g=0,h=this.length;g<h;g++)d=this[g],d.style&&(e=d.style.display,!f._data(d,"olddisplay")&&e==="none"&&(e=d.style.display=""),e===""&&f.css(d,"display")==="none"&&f._data(d,"olddisplay",cr(d.nodeName)));for(g=0;g<h;g++){d=this[g];if(d.style){e=d.style.display;if(e===""||e==="none")d.style.display=f._data(d,"olddisplay")||""}}return this},hide:function(a,b,c){if(a||a===0)return this.animate(cq("hide",3),a,b,c);for(var d=0,e=this.length;d<e;d++)if(this[d].style){var g=f.css(this[d],"display");g!=="none"&&!f._data(this[d],"olddisplay")&&f._data(this[d],"olddisplay",g)}for(d=0;d<e;d++)this[d].style&&(this[d].style.display="none");return this},_toggle:f.fn.toggle,toggle:function(a,b,c){var d=typeof a=="boolean";f.isFunction(a)&&f.isFunction(b)?this._toggle.apply(this,arguments):a==null||d?this.each(function(){var b=d?a:f(this).is(":hidden");f(this)[b?"show":"hide"]()}):this.animate(cq("toggle",3),a,b,c);return this},fadeTo:function(a,b,c,d){return this.filter(":hidden").css("opacity",0).show().end().animate({opacity:b},a,c,d)},animate:function(a,b,c,d){var e=f.speed(b,c,d);if(f.isEmptyObject(a))return this.each(e.complete,[!1]);a=f.extend({},a);return this[e.queue===!1?"each":"queue"](function(){e.queue===!1&&f._mark(this);var b=f.extend({},e),c=this.nodeType===1,d=c&&f(this).is(":hidden"),g,h,i,j,k,l,m,n,o;b.animatedProperties={};for(i in a){g=f.camelCase(i),i!==g&&(a[g]=a[i],delete a[i]),h=a[g],f.isArray(h)?(b.animatedProperties[g]=h[1],h=a[g]=h[0]):b.animatedProperties[g]=b.specialEasing&&b.specialEasing[g]||b.easing||"swing";if(h==="hide"&&d||h==="show"&&!d)return b.complete.call(this);c&&(g==="height"||g==="width")&&(b.overflow=[this.style.overflow,this.style.overflowX,this.style.overflowY],f.css(this,"display")==="inline"&&f.css(this,"float")==="none"&&(f.support.inlineBlockNeedsLayout?(j=cr(this.nodeName),j==="inline"?this.style.display="inline-block":(this.style.display="inline",this.style.zoom=1)):this.style.display="inline-block"))}b.overflow!=null&&(this.style.overflow="hidden");for(i in a)k=new f.fx(this,b,i),h=a[i],cj.test(h)?k[h==="toggle"?d?"show":"hide":h]():(l=ck.exec(h),m=k.cur(),l?(n=parseFloat(l[2]),o=l[3]||(f.cssNumber[i]?"":"px"),o!=="px"&&(f.style(this,i,(n||1)+o),m=(n||1)/k.cur()*m,f.style(this,i,m+o)),l[1]&&(n=(l[1]==="-="?-1:1)*n+m),k.custom(m,n,o)):k.custom(m,h,""));return!0})},stop:function(a,b){a&&this.queue([]),this.each(function(){var a=f.timers,c=a.length;b||f._unmark(!0,this);while(c--)a[c].elem===this&&(b&&a[c](!0),a.splice(c,1))}),b||this.dequeue();return this}}),f.each({slideDown:cq("show",1),slideUp:cq("hide",1),slideToggle:cq("toggle",1),fadeIn:{opacity:"show"},fadeOut:{opacity:"hide"},fadeToggle:{opacity:"toggle"}},function(a,b){f.fn[a]=function(a,c,d){return this.animate(b,a,c,d)}}),f.extend({speed:function(a,b,c){var d=a&&typeof a=="object"?f.extend({},a):{complete:c||!c&&b||f.isFunction(a)&&a,duration:a,easing:c&&b||b&&!f.isFunction(b)&&b};d.duration=f.fx.off?0:typeof d.duration=="number"?d.duration:d.duration in f.fx.speeds?f.fx.speeds[d.duration]:f.fx.speeds._default,d.old=d.complete,d.complete=function(a){f.isFunction(d.old)&&d.old.call(this),d.queue!==!1?f.dequeue(this):a!==!1&&f._unmark(this)};return d},easing:{linear:function(a,b,c,d){return c+d*a},swing:function(a,b,c,d){return(-Math.cos(a*Math.PI)/2+.5)*d+c}},timers:[],fx:function(a,b,c){this.options=b,this.elem=a,this.prop=c,b.orig=b.orig||{}}}),f.fx.prototype={update:function(){this.options.step&&this.options.step.call(this.elem,this.now,this),(f.fx.step[this.prop]||f.fx.step._default)(this)},cur:function(){if(this.elem[this.prop]!=null&&(!this.elem.style||this.elem.style[this.prop]==null))return this.elem[this.prop];var a,b=f.css(this.elem,this.prop);return isNaN(a=parseFloat(b))?!b||b==="auto"?0:b:a},custom:function(a,b,c){function g(a){return d.step(a)}var d=this,e=f.fx;this.startTime=cn||co(),this.start=a,this.end=b,this.unit=c||this.unit||(f.cssNumber[this.prop]?"":"px"),this.now=this.start,this.pos=this.state=0,g.elem=this.elem,g()&&f.timers.push(g)&&!cl&&(cl=setInterval(e.tick,e.interval))},show:function(){this.options.orig[this.prop]=f.style(this.elem,this.prop),this.options.show=!0,this.custom(this.prop==="width"||this.prop==="height"?1:0,this.cur()),f(this.elem).show()},hide:function(){this.options.orig[this.prop]=f.style(this.elem,this.prop),this.options.hide=!0,this.custom(this.cur(),0)},step:function(a){var b=cn||co(),c=!0,d=this.elem,e=this.options,g,h;if(a||b>=e.duration+this.startTime){this.now=this.end,this.pos=this.state=1,this.update(),e.animatedProperties[this.prop]=!0;for(g in e.animatedProperties)e.animatedProperties[g]!==!0&&(c=!1);if(c){e.overflow!=null&&!f.support.shrinkWrapBlocks&&f.each(["","X","Y"],function(a,b){d.style["overflow"+b]=e.overflow[a]}),e.hide&&f(d).hide();if(e.hide||e.show)for(var i in e.animatedProperties)f.style(d,i,e.orig[i]);e.complete.call(d)}return!1}e.duration==Infinity?this.now=b:(h=b-this.startTime,this.state=h/e.duration,this.pos=f.easing[e.animatedProperties[this.prop]](this.state,h,0,1,e.duration),this.now=this.start+(this.end-this.start)*this.pos),this.update();return!0}},f.extend(f.fx,{tick:function(){for(var a=f.timers,b=0;b<a.length;++b)a[b]()||a.splice(b--,1);a.length||f.fx.stop()},interval:13,stop:function(){clearInterval(cl),cl=null},speeds:{slow:600,fast:200,_default:400},step:{opacity:function(a){f.style(a.elem,"opacity",a.now)},_default:function(a){a.elem.style&&a.elem.style[a.prop]!=null?a.elem.style[a.prop]=(a.prop==="width"||a.prop==="height"?Math.max(0,a.now):a.now)+a.unit:a.elem[a.prop]=a.now}}}),f.expr&&f.expr.filters&&(f.expr.filters.animated=function(a){return f.grep(f.timers,function(b){return a===b.elem}).length});var cs=/^t(?:able|d|h)$/i,ct=/^(?:body|html)$/i;"getBoundingClientRect"in c.documentElement?f.fn.offset=function(a){var b=this[0],c;if(a)return this.each(function(b){f.offset.setOffset(this,a,b)});if(!b||!b.ownerDocument)return null;if(b===b.ownerDocument.body)return f.offset.bodyOffset(b);try{c=b.getBoundingClientRect()}catch(d){}var e=b.ownerDocument,g=e.documentElement;if(!c||!f.contains(g,b))return c?{top:c.top,left:c.left}:{top:0,left:0};var h=e.body,i=cu(e),j=g.clientTop||h.clientTop||0,k=g.clientLeft||h.clientLeft||0,l=i.pageYOffset||f.support.boxModel&&g.scrollTop||h.scrollTop,m=i.pageXOffset||f.support.boxModel&&g.scrollLeft||h.scrollLeft,n=c.top+l-j,o=c.left+m-k;return{top:n,left:o}}:f.fn.offset=function(a){var b=this[0];if(a)return this.each(function(b){f.offset.setOffset(this,a,b)});if(!b||!b.ownerDocument)return null;if(b===b.ownerDocument.body)return f.offset.bodyOffset(b);f.offset.initialize();var c,d=b.offsetParent,e=b,g=b.ownerDocument,h=g.documentElement,i=g.body,j=g.defaultView,k=j?j.getComputedStyle(b,null):b.currentStyle,l=b.offsetTop,m=b.offsetLeft;while((b=b.parentNode)&&b!==i&&b!==h){if(f.offset.supportsFixedPosition&&k.position==="fixed")break;c=j?j.getComputedStyle(b,null):b.currentStyle,l-=b.scrollTop,m-=b.scrollLeft,b===d&&(l+=b.offsetTop,m+=b.offsetLeft,f.offset.doesNotAddBorder&&(!f.offset.doesAddBorderForTableAndCells||!cs.test(b.nodeName))&&(l+=parseFloat(c.borderTopWidth)||0,m+=parseFloat(c.borderLeftWidth)||0),e=d,d=b.offsetParent),f.offset.subtractsBorderForOverflowNotVisible&&c.overflow!=="visible"&&(l+=parseFloat(c.borderTopWidth)||0,m+=parseFloat(c.borderLeftWidth)||0),k=c}if(k.position==="relative"||k.position==="static")l+=i.offsetTop,m+=i.offsetLeft;f.offset.supportsFixedPosition&&k.position==="fixed"&&(l+=Math.max(h.scrollTop,i.scrollTop),m+=Math.max(h.scrollLeft,i.scrollLeft));return{top:l,left:m}},f.offset={initialize:function(){var a=c.body,b=c.createElement("div"),d,e,g,h,i=parseFloat(f.css(a,"marginTop"))||0,j="<div style='position:absolute;top:0;left:0;margin:0;border:5px solid #000;padding:0;width:1px;height:1px;'><div></div></div><table style='position:absolute;top:0;left:0;margin:0;border:5px solid #000;padding:0;width:1px;height:1px;' cellpadding='0' cellspacing='0'><tr><td></td></tr></table>";f.extend(b.style,{position:"absolute",top:0,left:0,margin:0,border:0,width:"1px",height:"1px",visibility:"hidden"}),b.innerHTML=j,a.insertBefore(b,a.firstChild),d=b.firstChild,e=d.firstChild,h=d.nextSibling.firstChild.firstChild,this.doesNotAddBorder=e.offsetTop!==5,this.doesAddBorderForTableAndCells=h.offsetTop===5,e.style.position="fixed",e.style.top="20px",this.supportsFixedPosition=e.offsetTop===20||e.offsetTop===15,e.style.position=e.style.top="",d.style.overflow="hidden",d.style.position="relative",this.subtractsBorderForOverflowNotVisible=e.offsetTop===-5,this.doesNotIncludeMarginInBodyOffset=a.offsetTop!==i,a.removeChild(b),f.offset.initialize=f.noop},bodyOffset:function(a){var b=a.offsetTop,c=a.offsetLeft;f.offset.initialize(),f.offset.doesNotIncludeMarginInBodyOffset&&(b+=parseFloat(f.css(a,"marginTop"))||0,c+=parseFloat(f.css(a,"marginLeft"))||0);return{top:b,left:c}},setOffset:function(a,b,c){var d=f.css(a,"position");d==="static"&&(a.style.position="relative");var e=f(a),g=e.offset(),h=f.css(a,"top"),i=f.css(a,"left"),j=(d==="absolute"||d==="fixed")&&f.inArray("auto",[h,i])>-1,k={},l={},m,n;j?(l=e.position(),m=l.top,n=l.left):(m=parseFloat(h)||0,n=parseFloat(i)||0),f.isFunction(b)&&(b=b.call(a,c,g)),b.top!=null&&(k.top=b.top-g.top+m),b.left!=null&&(k.left=b.left-g.left+n),"using"in b?b.using.call(a,k):e.css(k)}},f.fn.extend({position:function(){if(!this[0])return null;var a=this[0],b=this.offsetParent(),c=this.offset(),d=ct.test(b[0].nodeName)?{top:0,left:0}:b.offset();c.top-=parseFloat(f.css(a,"marginTop"))||0,c.left-=parseFloat(f.css(a,"marginLeft"))||0,d.top+=parseFloat(f.css(b[0],"borderTopWidth"))||0,d.left+=parseFloat(f.css(b[0],"borderLeftWidth"))||0;return{top:c.top-d.top,left:c.left-d.left}},offsetParent:function(){return this.map(function(){var a=this.offsetParent||c.body;while(a&&!ct.test(a.nodeName)&&f.css(a,"position")==="static")a=a.offsetParent;return a})}}),f.each(["Left","Top"],function(a,c){var d="scroll"+c;f.fn[d]=function(c){var e,g;if(c===b){e=this[0];if(!e)return null;g=cu(e);return g?"pageXOffset"in g?g[a?"pageYOffset":"pageXOffset"]:f.support.boxModel&&g.document.documentElement[d]||g.document.body[d]:e[d]}return this.each(function(){g=cu(this),g?g.scrollTo(a?f(g).scrollLeft():c,a?c:f(g).scrollTop()):this[d]=c})}}),f.each(["Height","Width"],function(a,c){var d=c.toLowerCase();f.fn["inner"+c]=function(){var a=this[0];return a&&a.style?parseFloat(f.css(a,d,"padding")):null},f.fn["outer"+c]=function(a){var b=this[0];return b&&b.style?parseFloat(f.css(b,d,a?"margin":"border")):null},f.fn[d]=function(a){var e=this[0];if(!e)return a==null?null:this;if(f.isFunction(a))return this.each(function(b){var c=f(this);c[d](a.call(this,b,c[d]()))});if(f.isWindow(e)){var g=e.document.documentElement["client"+c],h=e.document.body;return e.document.compatMode==="CSS1Compat"&&g||h&&h["client"+c]||g}if(e.nodeType===9)return Math.max(e.documentElement["client"+c],e.body["scroll"+c],e.documentElement["scroll"+c],e.body["offset"+c],e.documentElement["offset"+c]);if(a===b){var i=f.css(e,d),j=parseFloat(i);return f.isNaN(j)?i:j}return this.css(d,typeof a=="string"?a:a+"px")}}),a.jQuery=a.$=f})(window);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/jquery/plugins/jquery-ui.min.js b/hyracks-control-cc/src/main/resources/static/javascript/jquery/plugins/jquery-ui.min.js
new file mode 100644
index 0000000..991cb8d
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/jquery/plugins/jquery-ui.min.js
@@ -0,0 +1,356 @@
+/*!
+ * jQuery UI 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI
+ */(function(a,b){function d(b){return!a(b).parents().andSelf().filter(function(){return a.curCSS(this,"visibility")==="hidden"||a.expr.filters.hidden(this)}).length}function c(b,c){var e=b.nodeName.toLowerCase();if("area"===e){var f=b.parentNode,g=f.name,h;if(!b.href||!g||f.nodeName.toLowerCase()!=="map")return!1;h=a("img[usemap=#"+g+"]")[0];return!!h&&d(h)}return(/input|select|textarea|button|object/.test(e)?!b.disabled:"a"==e?b.href||c:c)&&d(b)}a.ui=a.ui||{};a.ui.version||(a.extend(a.ui,{version:"1.8.17",keyCode:{ALT:18,BACKSPACE:8,CAPS_LOCK:20,COMMA:188,COMMAND:91,COMMAND_LEFT:91,COMMAND_RIGHT:93,CONTROL:17,DELETE:46,DOWN:40,END:35,ENTER:13,ESCAPE:27,HOME:36,INSERT:45,LEFT:37,MENU:93,NUMPAD_ADD:107,NUMPAD_DECIMAL:110,NUMPAD_DIVIDE:111,NUMPAD_ENTER:108,NUMPAD_MULTIPLY:106,NUMPAD_SUBTRACT:109,PAGE_DOWN:34,PAGE_UP:33,PERIOD:190,RIGHT:39,SHIFT:16,SPACE:32,TAB:9,UP:38,WINDOWS:91}}),a.fn.extend({propAttr:a.fn.prop||a.fn.attr,_focus:a.fn.focus,focus:function(b,c){return typeof b=="number"?this.each(function(){var d=this;setTimeout(function(){a(d).focus(),c&&c.call(d)},b)}):this._focus.apply(this,arguments)},scrollParent:function(){var b;a.browser.msie&&/(static|relative)/.test(this.css("position"))||/absolute/.test(this.css("position"))?b=this.parents().filter(function(){return/(relative|absolute|fixed)/.test(a.curCSS(this,"position",1))&&/(auto|scroll)/.test(a.curCSS(this,"overflow",1)+a.curCSS(this,"overflow-y",1)+a.curCSS(this,"overflow-x",1))}).eq(0):b=this.parents().filter(function(){return/(auto|scroll)/.test(a.curCSS(this,"overflow",1)+a.curCSS(this,"overflow-y",1)+a.curCSS(this,"overflow-x",1))}).eq(0);return/fixed/.test(this.css("position"))||!b.length?a(document):b},zIndex:function(c){if(c!==b)return this.css("zIndex",c);if(this.length){var d=a(this[0]),e,f;while(d.length&&d[0]!==document){e=d.css("position");if(e==="absolute"||e==="relative"||e==="fixed"){f=parseInt(d.css("zIndex"),10);if(!isNaN(f)&&f!==0)return f}d=d.parent()}}return 0},disableSelection:function(){return this.bind((a.support.selectstart?"selectstart":"mousedown")+".ui-disableSelection",function(a){a.preventDefault()})},enableSelection:function(){return this.unbind(".ui-disableSelection")}}),a.each(["Width","Height"],function(c,d){function h(b,c,d,f){a.each(e,function(){c-=parseFloat(a.curCSS(b,"padding"+this,!0))||0,d&&(c-=parseFloat(a.curCSS(b,"border"+this+"Width",!0))||0),f&&(c-=parseFloat(a.curCSS(b,"margin"+this,!0))||0)});return c}var e=d==="Width"?["Left","Right"]:["Top","Bottom"],f=d.toLowerCase(),g={innerWidth:a.fn.innerWidth,innerHeight:a.fn.innerHeight,outerWidth:a.fn.outerWidth,outerHeight:a.fn.outerHeight};a.fn["inner"+d]=function(c){if(c===b)return g["inner"+d].call(this);return this.each(function(){a(this).css(f,h(this,c)+"px")})},a.fn["outer"+d]=function(b,c){if(typeof b!="number")return g["outer"+d].call(this,b);return this.each(function(){a(this).css(f,h(this,b,!0,c)+"px")})}}),a.extend(a.expr[":"],{data:function(b,c,d){return!!a.data(b,d[3])},focusable:function(b){return c(b,!isNaN(a.attr(b,"tabindex")))},tabbable:function(b){var d=a.attr(b,"tabindex"),e=isNaN(d);return(e||d>=0)&&c(b,!e)}}),a(function(){var b=document.body,c=b.appendChild(c=document.createElement("div"));a.extend(c.style,{minHeight:"100px",height:"auto",padding:0,borderWidth:0}),a.support.minHeight=c.offsetHeight===100,a.support.selectstart="onselectstart"in c,b.removeChild(c).style.display="none"}),a.extend(a.ui,{plugin:{add:function(b,c,d){var e=a.ui[b].prototype;for(var f in d)e.plugins[f]=e.plugins[f]||[],e.plugins[f].push([c,d[f]])},call:function(a,b,c){var d=a.plugins[b];if(!!d&&!!a.element[0].parentNode)for(var e=0;e<d.length;e++)a.options[d[e][0]]&&d[e][1].apply(a.element,c)}},contains:function(a,b){return document.compareDocumentPosition?a.compareDocumentPosition(b)&16:a!==b&&a.contains(b)},hasScroll:function(b,c){if(a(b).css("overflow")==="hidden")return!1;var d=c&&c==="left"?"scrollLeft":"scrollTop",e=!1;if(b[d]>0)return!0;b[d]=1,e=b[d]>0,b[d]=0;return e},isOverAxis:function(a,b,c){return a>b&&a<b+c},isOver:function(b,c,d,e,f,g){return a.ui.isOverAxis(b,d,f)&&a.ui.isOverAxis(c,e,g)}}))})(jQuery);/*!
+ * jQuery UI Widget 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Widget
+ */(function(a,b){if(a.cleanData){var c=a.cleanData;a.cleanData=function(b){for(var d=0,e;(e=b[d])!=null;d++)try{a(e).triggerHandler("remove")}catch(f){}c(b)}}else{var d=a.fn.remove;a.fn.remove=function(b,c){return this.each(function(){c||(!b||a.filter(b,[this]).length)&&a("*",this).add([this]).each(function(){try{a(this).triggerHandler("remove")}catch(b){}});return d.call(a(this),b,c)})}}a.widget=function(b,c,d){var e=b.split(".")[0],f;b=b.split(".")[1],f=e+"-"+b,d||(d=c,c=a.Widget),a.expr[":"][f]=function(c){return!!a.data(c,b)},a[e]=a[e]||{},a[e][b]=function(a,b){arguments.length&&this._createWidget(a,b)};var g=new c;g.options=a.extend(!0,{},g.options),a[e][b].prototype=a.extend(!0,g,{namespace:e,widgetName:b,widgetEventPrefix:a[e][b].prototype.widgetEventPrefix||b,widgetBaseClass:f},d),a.widget.bridge(b,a[e][b])},a.widget.bridge=function(c,d){a.fn[c]=function(e){var f=typeof e=="string",g=Array.prototype.slice.call(arguments,1),h=this;e=!f&&g.length?a.extend.apply(null,[!0,e].concat(g)):e;if(f&&e.charAt(0)==="_")return h;f?this.each(function(){var d=a.data(this,c),f=d&&a.isFunction(d[e])?d[e].apply(d,g):d;if(f!==d&&f!==b){h=f;return!1}}):this.each(function(){var b=a.data(this,c);b?b.option(e||{})._init():a.data(this,c,new d(e,this))});return h}},a.Widget=function(a,b){arguments.length&&this._createWidget(a,b)},a.Widget.prototype={widgetName:"widget",widgetEventPrefix:"",options:{disabled:!1},_createWidget:function(b,c){a.data(c,this.widgetName,this),this.element=a(c),this.options=a.extend(!0,{},this.options,this._getCreateOptions(),b);var d=this;this.element.bind("remove."+this.widgetName,function(){d.destroy()}),this._create(),this._trigger("create"),this._init()},_getCreateOptions:function(){return a.metadata&&a.metadata.get(this.element[0])[this.widgetName]},_create:function(){},_init:function(){},destroy:function(){this.element.unbind("."+this.widgetName).removeData(this.widgetName),this.widget().unbind("."+this.widgetName).removeAttr("aria-disabled").removeClass(this.widgetBaseClass+"-disabled "+"ui-state-disabled")},widget:function(){return this.element},option:function(c,d){var e=c;if(arguments.length===0)return a.extend({},this.options);if(typeof c=="string"){if(d===b)return this.options[c];e={},e[c]=d}this._setOptions(e);return this},_setOptions:function(b){var c=this;a.each(b,function(a,b){c._setOption(a,b)});return this},_setOption:function(a,b){this.options[a]=b,a==="disabled"&&this.widget()[b?"addClass":"removeClass"](this.widgetBaseClass+"-disabled"+" "+"ui-state-disabled").attr("aria-disabled",b);return this},enable:function(){return this._setOption("disabled",!1)},disable:function(){return this._setOption("disabled",!0)},_trigger:function(b,c,d){var e,f,g=this.options[b];d=d||{},c=a.Event(c),c.type=(b===this.widgetEventPrefix?b:this.widgetEventPrefix+b).toLowerCase(),c.target=this.element[0],f=c.originalEvent;if(f)for(e in f)e in c||(c[e]=f[e]);this.element.trigger(c,d);return!(a.isFunction(g)&&g.call(this.element[0],c,d)===!1||c.isDefaultPrevented())}}})(jQuery);/*!
+ * jQuery UI Mouse 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Mouse
+ *
+ * Depends:
+ * jquery.ui.widget.js
+ */(function(a,b){var c=!1;a(document).mouseup(function(a){c=!1}),a.widget("ui.mouse",{options:{cancel:":input,option",distance:1,delay:0},_mouseInit:function(){var b=this;this.element.bind("mousedown."+this.widgetName,function(a){return b._mouseDown(a)}).bind("click."+this.widgetName,function(c){if(!0===a.data(c.target,b.widgetName+".preventClickEvent")){a.removeData(c.target,b.widgetName+".preventClickEvent"),c.stopImmediatePropagation();return!1}}),this.started=!1},_mouseDestroy:function(){this.element.unbind("."+this.widgetName)},_mouseDown:function(b){if(!c){this._mouseStarted&&this._mouseUp(b),this._mouseDownEvent=b;var d=this,e=b.which==1,f=typeof this.options.cancel=="string"&&b.target.nodeName?a(b.target).closest(this.options.cancel).length:!1;if(!e||f||!this._mouseCapture(b))return!0;this.mouseDelayMet=!this.options.delay,this.mouseDelayMet||(this._mouseDelayTimer=setTimeout(function(){d.mouseDelayMet=!0},this.options.delay));if(this._mouseDistanceMet(b)&&this._mouseDelayMet(b)){this._mouseStarted=this._mouseStart(b)!==!1;if(!this._mouseStarted){b.preventDefault();return!0}}!0===a.data(b.target,this.widgetName+".preventClickEvent")&&a.removeData(b.target,this.widgetName+".preventClickEvent"),this._mouseMoveDelegate=function(a){return d._mouseMove(a)},this._mouseUpDelegate=function(a){return d._mouseUp(a)},a(document).bind("mousemove."+this.widgetName,this._mouseMoveDelegate).bind("mouseup."+this.widgetName,this._mouseUpDelegate),b.preventDefault(),c=!0;return!0}},_mouseMove:function(b){if(a.browser.msie&&!(document.documentMode>=9)&&!b.button)return this._mouseUp(b);if(this._mouseStarted){this._mouseDrag(b);return b.preventDefault()}this._mouseDistanceMet(b)&&this._mouseDelayMet(b)&&(this._mouseStarted=this._mouseStart(this._mouseDownEvent,b)!==!1,this._mouseStarted?this._mouseDrag(b):this._mouseUp(b));return!this._mouseStarted},_mouseUp:function(b){a(document).unbind("mousemove."+this.widgetName,this._mouseMoveDelegate).unbind("mouseup."+this.widgetName,this._mouseUpDelegate),this._mouseStarted&&(this._mouseStarted=!1,b.target==this._mouseDownEvent.target&&a.data(b.target,this.widgetName+".preventClickEvent",!0),this._mouseStop(b));return!1},_mouseDistanceMet:function(a){return Math.max(Math.abs(this._mouseDownEvent.pageX-a.pageX),Math.abs(this._mouseDownEvent.pageY-a.pageY))>=this.options.distance},_mouseDelayMet:function(a){return this.mouseDelayMet},_mouseStart:function(a){},_mouseDrag:function(a){},_mouseStop:function(a){},_mouseCapture:function(a){return!0}})})(jQuery);/*
+ * jQuery UI Position 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Position
+ */(function(a,b){a.ui=a.ui||{};var c=/left|center|right/,d=/top|center|bottom/,e="center",f={},g=a.fn.position,h=a.fn.offset;a.fn.position=function(b){if(!b||!b.of)return g.apply(this,arguments);b=a.extend({},b);var h=a(b.of),i=h[0],j=(b.collision||"flip").split(" "),k=b.offset?b.offset.split(" "):[0,0],l,m,n;i.nodeType===9?(l=h.width(),m=h.height(),n={top:0,left:0}):i.setTimeout?(l=h.width(),m=h.height(),n={top:h.scrollTop(),left:h.scrollLeft()}):i.preventDefault?(b.at="left top",l=m=0,n={top:b.of.pageY,left:b.of.pageX}):(l=h.outerWidth(),m=h.outerHeight(),n=h.offset()),a.each(["my","at"],function(){var a=(b[this]||"").split(" ");a.length===1&&(a=c.test(a[0])?a.concat([e]):d.test(a[0])?[e].concat(a):[e,e]),a[0]=c.test(a[0])?a[0]:e,a[1]=d.test(a[1])?a[1]:e,b[this]=a}),j.length===1&&(j[1]=j[0]),k[0]=parseInt(k[0],10)||0,k.length===1&&(k[1]=k[0]),k[1]=parseInt(k[1],10)||0,b.at[0]==="right"?n.left+=l:b.at[0]===e&&(n.left+=l/2),b.at[1]==="bottom"?n.top+=m:b.at[1]===e&&(n.top+=m/2),n.left+=k[0],n.top+=k[1];return this.each(function(){var c=a(this),d=c.outerWidth(),g=c.outerHeight(),h=parseInt(a.curCSS(this,"marginLeft",!0))||0,i=parseInt(a.curCSS(this,"marginTop",!0))||0,o=d+h+(parseInt(a.curCSS(this,"marginRight",!0))||0),p=g+i+(parseInt(a.curCSS(this,"marginBottom",!0))||0),q=a.extend({},n),r;b.my[0]==="right"?q.left-=d:b.my[0]===e&&(q.left-=d/2),b.my[1]==="bottom"?q.top-=g:b.my[1]===e&&(q.top-=g/2),f.fractions||(q.left=Math.round(q.left),q.top=Math.round(q.top)),r={left:q.left-h,top:q.top-i},a.each(["left","top"],function(c,e){a.ui.position[j[c]]&&a.ui.position[j[c]][e](q,{targetWidth:l,targetHeight:m,elemWidth:d,elemHeight:g,collisionPosition:r,collisionWidth:o,collisionHeight:p,offset:k,my:b.my,at:b.at})}),a.fn.bgiframe&&c.bgiframe(),c.offset(a.extend(q,{using:b.using}))})},a.ui.position={fit:{left:function(b,c){var d=a(window),e=c.collisionPosition.left+c.collisionWidth-d.width()-d.scrollLeft();b.left=e>0?b.left-e:Math.max(b.left-c.collisionPosition.left,b.left)},top:function(b,c){var d=a(window),e=c.collisionPosition.top+c.collisionHeight-d.height()-d.scrollTop();b.top=e>0?b.top-e:Math.max(b.top-c.collisionPosition.top,b.top)}},flip:{left:function(b,c){if(c.at[0]!==e){var d=a(window),f=c.collisionPosition.left+c.collisionWidth-d.width()-d.scrollLeft(),g=c.my[0]==="left"?-c.elemWidth:c.my[0]==="right"?c.elemWidth:0,h=c.at[0]==="left"?c.targetWidth:-c.targetWidth,i=-2*c.offset[0];b.left+=c.collisionPosition.left<0?g+h+i:f>0?g+h+i:0}},top:function(b,c){if(c.at[1]!==e){var d=a(window),f=c.collisionPosition.top+c.collisionHeight-d.height()-d.scrollTop(),g=c.my[1]==="top"?-c.elemHeight:c.my[1]==="bottom"?c.elemHeight:0,h=c.at[1]==="top"?c.targetHeight:-c.targetHeight,i=-2*c.offset[1];b.top+=c.collisionPosition.top<0?g+h+i:f>0?g+h+i:0}}}},a.offset.setOffset||(a.offset.setOffset=function(b,c){/static/.test(a.curCSS(b,"position"))&&(b.style.position="relative");var d=a(b),e=d.offset(),f=parseInt(a.curCSS(b,"top",!0),10)||0,g=parseInt(a.curCSS(b,"left",!0),10)||0,h={top:c.top-e.top+f,left:c.left-e.left+g};"using"in c?c.using.call(b,h):d.css(h)},a.fn.offset=function(b){var c=this[0];if(!c||!c.ownerDocument)return null;if(b)return this.each(function(){a.offset.setOffset(this,b)});return h.call(this)}),function(){var b=document.getElementsByTagName("body")[0],c=document.createElement("div"),d,e,g,h,i;d=document.createElement(b?"div":"body"),g={visibility:"hidden",width:0,height:0,border:0,margin:0,background:"none"},b&&jQuery.extend(g,{position:"absolute",left:"-1000px",top:"-1000px"});for(var j in g)d.style[j]=g[j];d.appendChild(c),e=b||document.documentElement,e.insertBefore(d,e.firstChild),c.style.cssText="position: absolute; left: 10.7432222px; top: 10.432325px; height: 30px; width: 201px;",h=a(c).offset(function(a,b){return b}).offset(),d.innerHTML="",e.removeChild(d),i=h.top+h.left+(b?2e3:0),f.fractions=i>21&&i<22}()})(jQuery);/*
+ * jQuery UI Draggable 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Draggables
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.mouse.js
+ * jquery.ui.widget.js
+ */(function(a,b){a.widget("ui.draggable",a.ui.mouse,{widgetEventPrefix:"drag",options:{addClasses:!0,appendTo:"parent",axis:!1,connectToSortable:!1,containment:!1,cursor:"auto",cursorAt:!1,grid:!1,handle:!1,helper:"original",iframeFix:!1,opacity:!1,refreshPositions:!1,revert:!1,revertDuration:500,scope:"default",scroll:!0,scrollSensitivity:20,scrollSpeed:20,snap:!1,snapMode:"both",snapTolerance:20,stack:!1,zIndex:!1},_create:function(){this.options.helper=="original"&&!/^(?:r|a|f)/.test(this.element.css("position"))&&(this.element[0].style.position="relative"),this.options.addClasses&&this.element.addClass("ui-draggable"),this.options.disabled&&this.element.addClass("ui-draggable-disabled"),this._mouseInit()},destroy:function(){if(!!this.element.data("draggable")){this.element.removeData("draggable").unbind(".draggable").removeClass("ui-draggable ui-draggable-dragging ui-draggable-disabled"),this._mouseDestroy();return this}},_mouseCapture:function(b){var c=this.options;if(this.helper||c.disabled||a(b.target).is(".ui-resizable-handle"))return!1;this.handle=this._getHandle(b);if(!this.handle)return!1;c.iframeFix&&a(c.iframeFix===!0?"iframe":c.iframeFix).each(function(){a('<div class="ui-draggable-iframeFix" style="background: #fff;"></div>').css({width:this.offsetWidth+"px",height:this.offsetHeight+"px",position:"absolute",opacity:"0.001",zIndex:1e3}).css(a(this).offset()).appendTo("body")});return!0},_mouseStart:function(b){var c=this.options;this.helper=this._createHelper(b),this._cacheHelperProportions(),a.ui.ddmanager&&(a.ui.ddmanager.current=this),this._cacheMargins(),this.cssPosition=this.helper.css("position"),this.scrollParent=this.helper.scrollParent(),this.offset=this.positionAbs=this.element.offset(),this.offset={top:this.offset.top-this.margins.top,left:this.offset.left-this.margins.left},a.extend(this.offset,{click:{left:b.pageX-this.offset.left,top:b.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()}),this.originalPosition=this.position=this._generatePosition(b),this.originalPageX=b.pageX,this.originalPageY=b.pageY,c.cursorAt&&this._adjustOffsetFromHelper(c.cursorAt),c.containment&&this._setContainment();if(this._trigger("start",b)===!1){this._clear();return!1}this._cacheHelperProportions(),a.ui.ddmanager&&!c.dropBehaviour&&a.ui.ddmanager.prepareOffsets(this,b),this.helper.addClass("ui-draggable-dragging"),this._mouseDrag(b,!0),a.ui.ddmanager&&a.ui.ddmanager.dragStart(this,b);return!0},_mouseDrag:function(b,c){this.position=this._generatePosition(b),this.positionAbs=this._convertPositionTo("absolute");if(!c){var d=this._uiHash();if(this._trigger("drag",b,d)===!1){this._mouseUp({});return!1}this.position=d.position}if(!this.options.axis||this.options.axis!="y")this.helper[0].style.left=this.position.left+"px";if(!this.options.axis||this.options.axis!="x")this.helper[0].style.top=this.position.top+"px";a.ui.ddmanager&&a.ui.ddmanager.drag(this,b);return!1},_mouseStop:function(b){var c=!1;a.ui.ddmanager&&!this.options.dropBehaviour&&(c=a.ui.ddmanager.drop(this,b)),this.dropped&&(c=this.dropped,this.dropped=!1);if((!this.element[0]||!this.element[0].parentNode)&&this.options.helper=="original")return!1;if(this.options.revert=="invalid"&&!c||this.options.revert=="valid"&&c||this.options.revert===!0||a.isFunction(this.options.revert)&&this.options.revert.call(this.element,c)){var d=this;a(this.helper).animate(this.originalPosition,parseInt(this.options.revertDuration,10),function(){d._trigger("stop",b)!==!1&&d._clear()})}else this._trigger("stop",b)!==!1&&this._clear();return!1},_mouseUp:function(b){this.options.iframeFix===!0&&a("div.ui-draggable-iframeFix").each(function(){this.parentNode.removeChild(this)}),a.ui.ddmanager&&a.ui.ddmanager.dragStop(this,b);return a.ui.mouse.prototype._mouseUp.call(this,b)},cancel:function(){this.helper.is(".ui-draggable-dragging")?this._mouseUp({}):this._clear();return this},_getHandle:function(b){var c=!this.options.handle||!a(this.options.handle,this.element).length?!0:!1;a(this.options.handle,this.element).find("*").andSelf().each(function(){this==b.target&&(c=!0)});return c},_createHelper:function(b){var c=this.options,d=a.isFunction(c.helper)?a(c.helper.apply(this.element[0],[b])):c.helper=="clone"?this.element.clone().removeAttr("id"):this.element;d.parents("body").length||d.appendTo(c.appendTo=="parent"?this.element[0].parentNode:c.appendTo),d[0]!=this.element[0]&&!/(fixed|absolute)/.test(d.css("position"))&&d.css("position","absolute");return d},_adjustOffsetFromHelper:function(b){typeof b=="string"&&(b=b.split(" ")),a.isArray(b)&&(b={left:+b[0],top:+b[1]||0}),"left"in b&&(this.offset.click.left=b.left+this.margins.left),"right"in b&&(this.offset.click.left=this.helperProportions.width-b.right+this.margins.left),"top"in b&&(this.offset.click.top=b.top+this.margins.top),"bottom"in b&&(this.offset.click.top=this.helperProportions.height-b.bottom+this.margins.top)},_getParentOffset:function(){this.offsetParent=this.helper.offsetParent();var b=this.offsetParent.offset();this.cssPosition=="absolute"&&this.scrollParent[0]!=document&&a.ui.contains(this.scrollParent[0],this.offsetParent[0])&&(b.left+=this.scrollParent.scrollLeft(),b.top+=this.scrollParent.scrollTop());if(this.offsetParent[0]==document.body||this.offsetParent[0].tagName&&this.offsetParent[0].tagName.toLowerCase()=="html"&&a.browser.msie)b={top:0,left:0};return{top:b.top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:b.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if(this.cssPosition=="relative"){var a=this.element.position();return{top:a.top-(parseInt(this.helper.css("top"),10)||0)+this.scrollParent.scrollTop(),left:a.left-(parseInt(this.helper.css("left"),10)||0)+this.scrollParent.scrollLeft()}}return{top:0,left:0}},_cacheMargins:function(){this.margins={left:parseInt(this.element.css("marginLeft"),10)||0,top:parseInt(this.element.css("marginTop"),10)||0,right:parseInt(this.element.css("marginRight"),10)||0,bottom:parseInt(this.element.css("marginBottom"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var b=this.options;b.containment=="parent"&&(b.containment=this.helper[0].parentNode);if(b.containment=="document"||b.containment=="window")this.containment=[b.containment=="document"?0:a(window).scrollLeft()-this.offset.relative.left-this.offset.parent.left,b.containment=="document"?0:a(window).scrollTop()-this.offset.relative.top-this.offset.parent.top,(b.containment=="document"?0:a(window).scrollLeft())+a(b.containment=="document"?document:window).width()-this.helperProportions.width-this.margins.left,(b.containment=="document"?0:a(window).scrollTop())+(a(b.containment=="document"?document:window).height()||document.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top];if(!/^(document|window|parent)$/.test(b.containment)&&b.containment.constructor!=Array){var c=a(b.containment),d=c[0];if(!d)return;var e=c.offset(),f=a(d).css("overflow")!="hidden";this.containment=[(parseInt(a(d).css("borderLeftWidth"),10)||0)+(parseInt(a(d).css("paddingLeft"),10)||0),(parseInt(a(d).css("borderTopWidth"),10)||0)+(parseInt(a(d).css("paddingTop"),10)||0),(f?Math.max(d.scrollWidth,d.offsetWidth):d.offsetWidth)-(parseInt(a(d).css("borderLeftWidth"),10)||0)-(parseInt(a(d).css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left-this.margins.right,(f?Math.max(d.scrollHeight,d.offsetHeight):d.offsetHeight)-(parseInt(a(d).css("borderTopWidth"),10)||0)-(parseInt(a(d).css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top-this.margins.bottom],this.relative_container=c}else b.containment.constructor==Array&&(this.containment=b.containment)},_convertPositionTo:function(b,c){c||(c=this.position);var d=b=="absolute"?1:-1,e=this.options,f=this.cssPosition=="absolute"&&(this.scrollParent[0]==document||!a.ui.contains(this.scrollParent[0],this.offsetParent[0]))?this.offsetParent:this.scrollParent,g=/(html|body)/i.test(f[0].tagName);return{top:c.top+this.offset.relative.top*d+this.offset.parent.top*d-(a.browser.safari&&a.browser.version<526&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollTop():g?0:f.scrollTop())*d),left:c.left+this.offset.relative.left*d+this.offset.parent.left*d-(a.browser.safari&&a.browser.version<526&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():g?0:f.scrollLeft())*d)}},_generatePosition:function(b){var c=this.options,d=this.cssPosition=="absolute"&&(this.scrollParent[0]==document||!a.ui.contains(this.scrollParent[0],this.offsetParent[0]))?this.offsetParent:this.scrollParent,e=/(html|body)/i.test(d[0].tagName),f=b.pageX,g=b.pageY;if(this.originalPosition){var h;if(this.containment){if(this.relative_container){var i=this.relative_container.offset();h=[this.containment[0]+i.left,this.containment[1]+i.top,this.containment[2]+i.left,this.containment[3]+i.top]}else h=this.containment;b.pageX-this.offset.click.left<h[0]&&(f=h[0]+this.offset.click.left),b.pageY-this.offset.click.top<h[1]&&(g=h[1]+this.offset.click.top),b.pageX-this.offset.click.left>h[2]&&(f=h[2]+this.offset.click.left),b.pageY-this.offset.click.top>h[3]&&(g=h[3]+this.offset.click.top)}if(c.grid){var j=c.grid[1]?this.originalPageY+Math.round((g-this.originalPageY)/c.grid[1])*c.grid[1]:this.originalPageY;g=h?j-this.offset.click.top<h[1]||j-this.offset.click.top>h[3]?j-this.offset.click.top<h[1]?j+c.grid[1]:j-c.grid[1]:j:j;var k=c.grid[0]?this.originalPageX+Math.round((f-this.originalPageX)/c.grid[0])*c.grid[0]:this.originalPageX;f=h?k-this.offset.click.left<h[0]||k-this.offset.click.left>h[2]?k-this.offset.click.left<h[0]?k+c.grid[0]:k-c.grid[0]:k:k}}return{top:g-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+(a.browser.safari&&a.browser.version<526&&this.cssPosition=="fixed"?0:this.cssPosition=="fixed"?-this.scrollParent.scrollTop():e?0:d.scrollTop()),left:f-this.offset.click.left-this.offset.relative.left-this.offset.parent.left+(a.browser.safari&&a.browser.version<526&&this.cssPosition=="fixed"?0:this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():e?0:d.scrollLeft())}},_clear:function(){this.helper.removeClass("ui-draggable-dragging"),this.helper[0]!=this.element[0]&&!this.cancelHelperRemoval&&this.helper.remove(),this.helper=null,this.cancelHelperRemoval=!1},_trigger:function(b,c,d){d=d||this._uiHash(),a.ui.plugin.call(this,b,[c,d]),b=="drag"&&(this.positionAbs=this._convertPositionTo("absolute"));return a.Widget.prototype._trigger.call(this,b,c,d)},plugins:{},_uiHash:function(a){return{helper:this.helper,position:this.position,originalPosition:this.originalPosition,offset:this.positionAbs}}}),a.extend(a.ui.draggable,{version:"1.8.17"}),a.ui.plugin.add("draggable","connectToSortable",{start:function(b,c){var d=a(this).data("draggable"),e=d.options,f=a.extend({},c,{item:d.element});d.sortables=[],a(e.connectToSortable).each(function(){var c=a.data(this,"sortable");c&&!c.options.disabled&&(d.sortables.push({instance:c,shouldRevert:c.options.revert}),c.refreshPositions(),c._trigger("activate",b,f))})},stop:function(b,c){var d=a(this).data("draggable"),e=a.extend({},c,{item:d.element});a.each(d.sortables,function(){this.instance.isOver?(this.instance.isOver=0,d.cancelHelperRemoval=!0,this.instance.cancelHelperRemoval=!1,this.shouldRevert&&(this.instance.options.revert=!0),this.instance._mouseStop(b),this.instance.options.helper=this.instance.options._helper,d.options.helper=="original"&&this.instance.currentItem.css({top:"auto",left:"auto"})):(this.instance.cancelHelperRemoval=!1,this.instance._trigger("deactivate",b,e))})},drag:function(b,c){var d=a(this).data("draggable"),e=this,f=function(b){var c=this.offset.click.top,d=this.offset.click.left,e=this.positionAbs.top,f=this.positionAbs.left,g=b.height,h=b.width,i=b.top,j=b.left;return a.ui.isOver(e+c,f+d,i,j,g,h)};a.each(d.sortables,function(f){this.instance.positionAbs=d.positionAbs,this.instance.helperProportions=d.helperProportions,this.instance.offset.click=d.offset.click,this.instance._intersectsWith(this.instance.containerCache)?(this.instance.isOver||(this.instance.isOver=1,this.instance.currentItem=a(e).clone().removeAttr("id").appendTo(this.instance.element).data("sortable-item",!0),this.instance.options._helper=this.instance.options.helper,this.instance.options.helper=function(){return c.helper[0]},b.target=this.instance.currentItem[0],this.instance._mouseCapture(b,!0),this.instance._mouseStart(b,!0,!0),this.instance.offset.click.top=d.offset.click.top,this.instance.offset.click.left=d.offset.click.left,this.instance.offset.parent.left-=d.offset.parent.left-this.instance.offset.parent.left,this.instance.offset.parent.top-=d.offset.parent.top-this.instance.offset.parent.top,d._trigger("toSortable",b),d.dropped=this.instance.element,d.currentItem=d.element,this.instance.fromOutside=d),this.instance.currentItem&&this.instance._mouseDrag(b)):this.instance.isOver&&(this.instance.isOver=0,this.instance.cancelHelperRemoval=!0,this.instance.options.revert=!1,this.instance._trigger("out",b,this.instance._uiHash(this.instance)),this.instance._mouseStop(b,!0),this.instance.options.helper=this.instance.options._helper,this.instance.currentItem.remove(),this.instance.placeholder&&this.instance.placeholder.remove(),d._trigger("fromSortable",b),d.dropped=!1)})}}),a.ui.plugin.add("draggable","cursor",{start:function(b,c){var d=a("body"),e=a(this).data("draggable").options;d.css("cursor")&&(e._cursor=d.css("cursor")),d.css("cursor",e.cursor)},stop:function(b,c){var d=a(this).data("draggable").options;d._cursor&&a("body").css("cursor",d._cursor)}}),a.ui.plugin.add("draggable","opacity",{start:function(b,c){var d=a(c.helper),e=a(this).data("draggable").options;d.css("opacity")&&(e._opacity=d.css("opacity")),d.css("opacity",e.opacity)},stop:function(b,c){var d=a(this).data("draggable").options;d._opacity&&a(c.helper).css("opacity",d._opacity)}}),a.ui.plugin.add("draggable","scroll",{start:function(b,c){var d=a(this).data("draggable");d.scrollParent[0]!=document&&d.scrollParent[0].tagName!="HTML"&&(d.overflowOffset=d.scrollParent.offset())},drag:function(b,c){var d=a(this).data("draggable"),e=d.options,f=!1;if(d.scrollParent[0]!=document&&d.scrollParent[0].tagName!="HTML"){if(!e.axis||e.axis!="x")d.overflowOffset.top+d.scrollParent[0].offsetHeight-b.pageY<e.scrollSensitivity?d.scrollParent[0].scrollTop=f=d.scrollParent[0].scrollTop+e.scrollSpeed:b.pageY-d.overflowOffset.top<e.scrollSensitivity&&(d.scrollParent[0].scrollTop=f=d.scrollParent[0].scrollTop-e.scrollSpeed);if(!e.axis||e.axis!="y")d.overflowOffset.left+d.scrollParent[0].offsetWidth-b.pageX<e.scrollSensitivity?d.scrollParent[0].scrollLeft=f=d.scrollParent[0].scrollLeft+e.scrollSpeed:b.pageX-d.overflowOffset.left<e.scrollSensitivity&&(d.scrollParent[0].scrollLeft=f=d.scrollParent[0].scrollLeft-e.scrollSpeed)}else{if(!e.axis||e.axis!="x")b.pageY-a(document).scrollTop()<e.scrollSensitivity?f=a(document).scrollTop(a(document).scrollTop()-e.scrollSpeed):a(window).height()-(b.pageY-a(document).scrollTop())<e.scrollSensitivity&&(f=a(document).scrollTop(a(document).scrollTop()+e.scrollSpeed));if(!e.axis||e.axis!="y")b.pageX-a(document).scrollLeft()<e.scrollSensitivity?f=a(document).scrollLeft(a(document).scrollLeft()-e.scrollSpeed):a(window).width()-(b.pageX-a(document).scrollLeft())<e.scrollSensitivity&&(f=a(document).scrollLeft(a(document).scrollLeft()+e.scrollSpeed))}f!==!1&&a.ui.ddmanager&&!e.dropBehaviour&&a.ui.ddmanager.prepareOffsets(d,b)}}),a.ui.plugin.add("draggable","snap",{start:function(b,c){var d=a(this).data("draggable"),e=d.options;d.snapElements=[],a(e.snap.constructor!=String?e.snap.items||":data(draggable)":e.snap).each(function(){var b=a(this),c=b.offset();this!=d.element[0]&&d.snapElements.push({item:this,width:b.outerWidth(),height:b.outerHeight(),top:c.top,left:c.left})})},drag:function(b,c){var d=a(this).data("draggable"),e=d.options,f=e.snapTolerance,g=c.offset.left,h=g+d.helperProportions.width,i=c.offset.top,j=i+d.helperProportions.height;for(var k=d.snapElements.length-1;k>=0;k--){var l=d.snapElements[k].left,m=l+d.snapElements[k].width,n=d.snapElements[k].top,o=n+d.snapElements[k].height;if(!(l-f<g&&g<m+f&&n-f<i&&i<o+f||l-f<g&&g<m+f&&n-f<j&&j<o+f||l-f<h&&h<m+f&&n-f<i&&i<o+f||l-f<h&&h<m+f&&n-f<j&&j<o+f)){d.snapElements[k].snapping&&d.options.snap.release&&d.options.snap.release.call(d.element,b,a.extend(d._uiHash(),{snapItem:d.snapElements[k].item})),d.snapElements[k].snapping=!1;continue}if(e.snapMode!="inner"){var p=Math.abs(n-j)<=f,q=Math.abs(o-i)<=f,r=Math.abs(l-h)<=f,s=Math.abs(m-g)<=f;p&&(c.position.top=d._convertPositionTo("relative",{top:n-d.helperProportions.height,left:0}).top-d.margins.top),q&&(c.position.top=d._convertPositionTo("relative",{top:o,left:0}).top-d.margins.top),r&&(c.position.left=d._convertPositionTo("relative",{top:0,left:l-d.helperProportions.width}).left-d.margins.left),s&&(c.position.left=d._convertPositionTo("relative",{top:0,left:m}).left-d.margins.left)}var t=p||q||r||s;if(e.snapMode!="outer"){var p=Math.abs(n-i)<=f,q=Math.abs(o-j)<=f,r=Math.abs(l-g)<=f,s=Math.abs(m-h)<=f;p&&(c.position.top=d._convertPositionTo("relative",{top:n,left:0}).top-d.margins.top),q&&(c.position.top=d._convertPositionTo("relative",{top:o-d.helperProportions.height,left:0}).top-d.margins.top),r&&(c.position.left=d._convertPositionTo("relative",{top:0,left:l}).left-d.margins.left),s&&(c.position.left=d._convertPositionTo("relative",{top:0,left:m-d.helperProportions.width}).left-d.margins.left)}!d.snapElements[k].snapping&&(p||q||r||s||t)&&d.options.snap.snap&&d.options.snap.snap.call(d.element,b,a.extend(d._uiHash(),{snapItem:d.snapElements[k].item})),d.snapElements[k].snapping=p||q||r||s||t}}}),a.ui.plugin.add("draggable","stack",{start:function(b,c){var d=a(this).data("draggable").options,e=a.makeArray(a(d.stack)).sort(function(b,c){return(parseInt(a(b).css("zIndex"),10)||0)-(parseInt(a(c).css("zIndex"),10)||0)});if(!!e.length){var f=parseInt(e[0].style.zIndex)||0;a(e).each(function(a){this.style.zIndex=f+a}),this[0].style.zIndex=f+e.length}}}),a.ui.plugin.add("draggable","zIndex",{start:function(b,c){var d=a(c.helper),e=a(this).data("draggable").options;d.css("zIndex")&&(e._zIndex=d.css("zIndex")),d.css("zIndex",e.zIndex)},stop:function(b,c){var d=a(this).data("draggable").options;d._zIndex&&a(c.helper).css("zIndex",d._zIndex)}})})(jQuery);/*
+ * jQuery UI Droppable 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Droppables
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.widget.js
+ * jquery.ui.mouse.js
+ * jquery.ui.draggable.js
+ */(function(a,b){a.widget("ui.droppable",{widgetEventPrefix:"drop",options:{accept:"*",activeClass:!1,addClasses:!0,greedy:!1,hoverClass:!1,scope:"default",tolerance:"intersect"},_create:function(){var b=this.options,c=b.accept;this.isover=0,this.isout=1,this.accept=a.isFunction(c)?c:function(a){return a.is(c)},this.proportions={width:this.element[0].offsetWidth,height:this.element[0].offsetHeight},a.ui.ddmanager.droppables[b.scope]=a.ui.ddmanager.droppables[b.scope]||[],a.ui.ddmanager.droppables[b.scope].push(this),b.addClasses&&this.element.addClass("ui-droppable")},destroy:function(){var b=a.ui.ddmanager.droppables[this.options.scope];for(var c=0;c<b.length;c++)b[c]==this&&b.splice(c,1);this.element.removeClass("ui-droppable ui-droppable-disabled").removeData("droppable").unbind(".droppable");return this},_setOption:function(b,c){b=="accept"&&(this.accept=a.isFunction(c)?c:function(a){return a.is(c)}),a.Widget.prototype._setOption.apply(this,arguments)},_activate:function(b){var c=a.ui.ddmanager.current;this.options.activeClass&&this.element.addClass(this.options.activeClass),c&&this._trigger("activate",b,this.ui(c))},_deactivate:function(b){var c=a.ui.ddmanager.current;this.options.activeClass&&this.element.removeClass(this.options.activeClass),c&&this._trigger("deactivate",b,this.ui(c))},_over:function(b){var c=a.ui.ddmanager.current;!!c&&(c.currentItem||c.element)[0]!=this.element[0]&&this.accept.call(this.element[0],c.currentItem||c.element)&&(this.options.hoverClass&&this.element.addClass(this.options.hoverClass),this._trigger("over",b,this.ui(c)))},_out:function(b){var c=a.ui.ddmanager.current;!!c&&(c.currentItem||c.element)[0]!=this.element[0]&&this.accept.call(this.element[0],c.currentItem||c.element)&&(this.options.hoverClass&&this.element.removeClass(this.options.hoverClass),this._trigger("out",b,this.ui(c)))},_drop:function(b,c){var d=c||a.ui.ddmanager.current;if(!d||(d.currentItem||d.element)[0]==this.element[0])return!1;var e=!1;this.element.find(":data(droppable)").not(".ui-draggable-dragging").each(function(){var b=a.data(this,"droppable");if(b.options.greedy&&!b.options.disabled&&b.options.scope==d.options.scope&&b.accept.call(b.element[0],d.currentItem||d.element)&&a.ui.intersect(d,a.extend(b,{offset:b.element.offset()}),b.options.tolerance)){e=!0;return!1}});if(e)return!1;if(this.accept.call(this.element[0],d.currentItem||d.element)){this.options.activeClass&&this.element.removeClass(this.options.activeClass),this.options.hoverClass&&this.element.removeClass(this.options.hoverClass),this._trigger("drop",b,this.ui(d));return this.element}return!1},ui:function(a){return{draggable:a.currentItem||a.element,helper:a.helper,position:a.position,offset:a.positionAbs}}}),a.extend(a.ui.droppable,{version:"1.8.17"}),a.ui.intersect=function(b,c,d){if(!c.offset)return!1;var e=(b.positionAbs||b.position.absolute).left,f=e+b.helperProportions.width,g=(b.positionAbs||b.position.absolute).top,h=g+b.helperProportions.height,i=c.offset.left,j=i+c.proportions.width,k=c.offset.top,l=k+c.proportions.height;switch(d){case"fit":return i<=e&&f<=j&&k<=g&&h<=l;case"intersect":return i<e+b.helperProportions.width/2&&f-b.helperProportions.width/2<j&&k<g+b.helperProportions.height/2&&h-b.helperProportions.height/2<l;case"pointer":var m=(b.positionAbs||b.position.absolute).left+(b.clickOffset||b.offset.click).left,n=(b.positionAbs||b.position.absolute).top+(b.clickOffset||b.offset.click).top,o=a.ui.isOver(n,m,k,i,c.proportions.height,c.proportions.width);return o;case"touch":return(g>=k&&g<=l||h>=k&&h<=l||g<k&&h>l)&&(e>=i&&e<=j||f>=i&&f<=j||e<i&&f>j);default:return!1}},a.ui.ddmanager={current:null,droppables:{"default":[]},prepareOffsets:function(b,c){var d=a.ui.ddmanager.droppables[b.options.scope]||[],e=c?c.type:null,f=(b.currentItem||b.element).find(":data(droppable)").andSelf();droppablesLoop:for(var g=0;g<d.length;g++){if(d[g].options.disabled||b&&!d[g].accept.call(d[g].element[0],b.currentItem||b.element))continue;for(var h=0;h<f.length;h++)if(f[h]==d[g].element[0]){d[g].proportions.height=0;continue droppablesLoop}d[g].visible=d[g].element.css("display")!="none";if(!d[g].visible)continue;e=="mousedown"&&d[g]._activate.call(d[g],c),d[g].offset=d[g].element.offset(),d[g].proportions={width:d[g].element[0].offsetWidth,height:d[g].element[0].offsetHeight}}},drop:function(b,c){var d=!1;a.each(a.ui.ddmanager.droppables[b.options.scope]||[],function(){!this.options||(!this.options.disabled&&this.visible&&a.ui.intersect(b,this,this.options.tolerance)&&(d=this._drop.call(this,c)||d),!this.options.disabled&&this.visible&&this.accept.call(this.element[0],b.currentItem||b.element)&&(this.isout=1,this.isover=0,this._deactivate.call(this,c)))});return d},dragStart:function(b,c){b.element.parents(":not(body,html)").bind("scroll.droppable",function(){b.options.refreshPositions||a.ui.ddmanager.prepareOffsets(b,c)})},drag:function(b,c){b.options.refreshPositions&&a.ui.ddmanager.prepareOffsets(b,c),a.each(a.ui.ddmanager.droppables[b.options.scope]||[],function(){if(!(this.options.disabled||this.greedyChild||!this.visible)){var d=a.ui.intersect(b,this,this.options.tolerance),e=!d&&this.isover==1?"isout":d&&this.isover==0?"isover":null;if(!e)return;var f;if(this.options.greedy){var g=this.element.parents(":data(droppable):eq(0)");g.length&&(f=a.data(g[0],"droppable"),f.greedyChild=e=="isover"?1:0)}f&&e=="isover"&&(f.isover=0,f.isout=1,f._out.call(f,c)),this[e]=1,this[e=="isout"?"isover":"isout"]=0,this[e=="isover"?"_over":"_out"].call(this,c),f&&e=="isout"&&(f.isout=0,f.isover=1,f._over.call(f,c))}})},dragStop:function(b,c){b.element.parents(":not(body,html)").unbind("scroll.droppable"),b.options.refreshPositions||a.ui.ddmanager.prepareOffsets(b,c)}}})(jQuery);/*
+ * jQuery UI Resizable 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Resizables
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.mouse.js
+ * jquery.ui.widget.js
+ */(function(a,b){a.widget("ui.resizable",a.ui.mouse,{widgetEventPrefix:"resize",options:{alsoResize:!1,animate:!1,animateDuration:"slow",animateEasing:"swing",aspectRatio:!1,autoHide:!1,containment:!1,ghost:!1,grid:!1,handles:"e,s,se",helper:!1,maxHeight:null,maxWidth:null,minHeight:10,minWidth:10,zIndex:1e3},_create:function(){var b=this,c=this.options;this.element.addClass("ui-resizable"),a.extend(this,{_aspectRatio:!!c.aspectRatio,aspectRatio:c.aspectRatio,originalElement:this.element,_proportionallyResizeElements:[],_helper:c.helper||c.ghost||c.animate?c.helper||"ui-resizable-helper":null}),this.element[0].nodeName.match(/canvas|textarea|input|select|button|img/i)&&(/relative/.test(this.element.css("position"))&&a.browser.opera&&this.element.css({position:"relative",top:"auto",left:"auto"}),this.element.wrap(a('<div class="ui-wrapper" style="overflow: hidden;"></div>').css({position:this.element.css("position"),width:this.element.outerWidth(),height:this.element.outerHeight(),top:this.element.css("top"),left:this.element.css("left")})),this.element=this.element.parent().data("resizable",this.element.data("resizable")),this.elementIsWrapper=!0,this.element.css({marginLeft:this.originalElement.css("marginLeft"),marginTop:this.originalElement.css("marginTop"),marginRight:this.originalElement.css("marginRight"),marginBottom:this.originalElement.css("marginBottom")}),this.originalElement.css({marginLeft:0,marginTop:0,marginRight:0,marginBottom:0}),this.originalResizeStyle=this.originalElement.css("resize"),this.originalElement.css("resize","none"),this._proportionallyResizeElements.push(this.originalElement.css({position:"static",zoom:1,display:"block"})),this.originalElement.css({margin:this.originalElement.css("margin")}),this._proportionallyResize()),this.handles=c.handles||(a(".ui-resizable-handle",this.element).length?{n:".ui-resizable-n",e:".ui-resizable-e",s:".ui-resizable-s",w:".ui-resizable-w",se:".ui-resizable-se",sw:".ui-resizable-sw",ne:".ui-resizable-ne",nw:".ui-resizable-nw"}:"e,s,se");if(this.handles.constructor==String){this.handles=="all"&&(this.handles="n,e,s,w,se,sw,ne,nw");var d=this.handles.split(",");this.handles={};for(var e=0;e<d.length;e++){var f=a.trim(d[e]),g="ui-resizable-"+f,h=a('<div class="ui-resizable-handle '+g+'"></div>');/sw|se|ne|nw/.test(f)&&h.css({zIndex:++c.zIndex}),"se"==f&&h.addClass("ui-icon ui-icon-gripsmall-diagonal-se"),this.handles[f]=".ui-resizable-"+f,this.element.append(h)}}this._renderAxis=function(b){b=b||this.element;for(var c in this.handles){this.handles[c].constructor==String&&(this.handles[c]=a(this.handles[c],this.element).show());if(this.elementIsWrapper&&this.originalElement[0].nodeName.match(/textarea|input|select|button/i)){var d=a(this.handles[c],this.element),e=0;e=/sw|ne|nw|se|n|s/.test(c)?d.outerHeight():d.outerWidth();var f=["padding",/ne|nw|n/.test(c)?"Top":/se|sw|s/.test(c)?"Bottom":/^e$/.test(c)?"Right":"Left"].join("");b.css(f,e),this._proportionallyResize()}if(!a(this.handles[c]).length)continue}},this._renderAxis(this.element),this._handles=a(".ui-resizable-handle",this.element).disableSelection(),this._handles.mouseover(function(){if(!b.resizing){if(this.className)var a=this.className.match(/ui-resizable-(se|sw|ne|nw|n|e|s|w)/i);b.axis=a&&a[1]?a[1]:"se"}}),c.autoHide&&(this._handles.hide(),a(this.element).addClass("ui-resizable-autohide").hover(function(){c.disabled||(a(this).removeClass("ui-resizable-autohide"),b._handles.show())},function(){c.disabled||b.resizing||(a(this).addClass("ui-resizable-autohide"),b._handles.hide())})),this._mouseInit()},destroy:function(){this._mouseDestroy();var b=function(b){a(b).removeClass("ui-resizable ui-resizable-disabled ui-resizable-resizing").removeData("resizable").unbind(".resizable").find(".ui-resizable-handle").remove()};if(this.elementIsWrapper){b(this.element);var c=this.element;c.after(this.originalElement.css({position:c.css("position"),width:c.outerWidth(),height:c.outerHeight(),top:c.css("top"),left:c.css("left")})).remove()}this.originalElement.css("resize",this.originalResizeStyle),b(this.originalElement);return this},_mouseCapture:function(b){var c=!1;for(var d in this.handles)a(this.handles[d])[0]==b.target&&(c=!0);return!this.options.disabled&&c},_mouseStart:function(b){var d=this.options,e=this.element.position(),f=this.element;this.resizing=!0,this.documentScroll={top:a(document).scrollTop(),left:a(document).scrollLeft()},(f.is(".ui-draggable")||/absolute/.test(f.css("position")))&&f.css({position:"absolute",top:e.top,left:e.left}),a.browser.opera&&/relative/.test(f.css("position"))&&f.css({position:"relative",top:"auto",left:"auto"}),this._renderProxy();var g=c(this.helper.css("left")),h=c(this.helper.css("top"));d.containment&&(g+=a(d.containment).scrollLeft()||0,h+=a(d.containment).scrollTop()||0),this.offset=this.helper.offset(),this.position={left:g,top:h},this.size=this._helper?{width:f.outerWidth(),height:f.outerHeight()}:{width:f.width(),height:f.height()},this.originalSize=this._helper?{width:f.outerWidth(),height:f.outerHeight()}:{width:f.width(),height:f.height()},this.originalPosition={left:g,top:h},this.sizeDiff={width:f.outerWidth()-f.width(),height:f.outerHeight()-f.height()},this.originalMousePosition={left:b.pageX,top:b.pageY},this.aspectRatio=typeof d.aspectRatio=="number"?d.aspectRatio:this.originalSize.width/this.originalSize.height||1;var i=a(".ui-resizable-"+this.axis).css("cursor");a("body").css("cursor",i=="auto"?this.axis+"-resize":i),f.addClass("ui-resizable-resizing"),this._propagate("start",b);return!0},_mouseDrag:function(b){var c=this.helper,d=this.options,e={},f=this,g=this.originalMousePosition,h=this.axis,i=b.pageX-g.left||0,j=b.pageY-g.top||0,k=this._change[h];if(!k)return!1;var l=k.apply(this,[b,i,j]),m=a.browser.msie&&a.browser.version<7,n=this.sizeDiff;this._updateVirtualBoundaries(b.shiftKey);if(this._aspectRatio||b.shiftKey)l=this._updateRatio(l,b);l=this._respectSize(l,b),this._propagate("resize",b),c.css({top:this.position.top+"px",left:this.position.left+"px",width:this.size.width+"px",height:this.size.height+"px"}),!this._helper&&this._proportionallyResizeElements.length&&this._proportionallyResize(),this._updateCache(l),this._trigger("resize",b,this.ui());return!1},_mouseStop:function(b){this.resizing=!1;var c=this.options,d=this;if(this._helper){var e=this._proportionallyResizeElements,f=e.length&&/textarea/i.test(e[0].nodeName),g=f&&a.ui.hasScroll(e[0],"left")?0:d.sizeDiff.height,h=f?0:d.sizeDiff.width,i={width:d.helper.width()-h,height:d.helper.height()-g},j=parseInt(d.element.css("left"),10)+(d.position.left-d.originalPosition.left)||null,k=parseInt(d.element.css("top"),10)+(d.position.top-d.originalPosition.top)||null;c.animate||this.element.css(a.extend(i,{top:k,left:j})),d.helper.height(d.size.height),d.helper.width(d.size.width),this._helper&&!c.animate&&this._proportionallyResize()}a("body").css("cursor","auto"),this.element.removeClass("ui-resizable-resizing"),this._propagate("stop",b),this._helper&&this.helper.remove();return!1},_updateVirtualBoundaries:function(a){var b=this.options,c,e,f,g,h;h={minWidth:d(b.minWidth)?b.minWidth:0,maxWidth:d(b.maxWidth)?b.maxWidth:Infinity,minHeight:d(b.minHeight)?b.minHeight:0,maxHeight:d(b.maxHeight)?b.maxHeight:Infinity};if(this._aspectRatio||a)c=h.minHeight*this.aspectRatio,f=h.minWidth/this.aspectRatio,e=h.maxHeight*this.aspectRatio,g=h.maxWidth/this.aspectRatio,c>h.minWidth&&(h.minWidth=c),f>h.minHeight&&(h.minHeight=f),e<h.maxWidth&&(h.maxWidth=e),g<h.maxHeight&&(h.maxHeight=g);this._vBoundaries=h},_updateCache:function(a){var b=this.options;this.offset=this.helper.offset(),d(a.left)&&(this.position.left=a.left),d(a.top)&&(this.position.top=a.top),d(a.height)&&(this.size.height=a.height),d(a.width)&&(this.size.width=a.width)},_updateRatio:function(a,b){var c=this.options,e=this.position,f=this.size,g=this.axis;d(a.height)?a.width=a.height*this.aspectRatio:d(a.width)&&(a.height=a.width/this.aspectRatio),g=="sw"&&(a.left=e.left+(f.width-a.width),a.top=null),g=="nw"&&(a.top=e.top+(f.height-a.height),a.left=e.left+(f.width-a.width));return a},_respectSize:function(a,b){var c=this.helper,e=this._vBoundaries,f=this._aspectRatio||b.shiftKey,g=this.axis,h=d(a.width)&&e.maxWidth&&e.maxWidth<a.width,i=d(a.height)&&e.maxHeight&&e.maxHeight<a.height,j=d(a.width)&&e.minWidth&&e.minWidth>a.width,k=d(a.height)&&e.minHeight&&e.minHeight>a.height;j&&(a.width=e.minWidth),k&&(a.height=e.minHeight),h&&(a.width=e.maxWidth),i&&(a.height=e.maxHeight);var l=this.originalPosition.left+this.originalSize.width,m=this.position.top+this.size.height,n=/sw|nw|w/.test(g),o=/nw|ne|n/.test(g);j&&n&&(a.left=l-e.minWidth),h&&n&&(a.left=l-e.maxWidth),k&&o&&(a.top=m-e.minHeight),i&&o&&(a.top=m-e.maxHeight);var p=!a.width&&!a.height;p&&!a.left&&a.top?a.top=null:p&&!a.top&&a.left&&(a.left=null);return a},_proportionallyResize:function(){var b=this.options;if(!!this._proportionallyResizeElements.length){var c=this.helper||this.element;for(var d=0;d<this._proportionallyResizeElements.length;d++){var e=this._proportionallyResizeElements[d];if(!this.borderDif){var f=[e.css("borderTopWidth"),e.css("borderRightWidth"),e.css("borderBottomWidth"),e.css("borderLeftWidth")],g=[e.css("paddingTop"),e.css("paddingRight"),e.css("paddingBottom"),e.css("paddingLeft")];this.borderDif=a.map(f,function(a,b){var c=parseInt(a,10)||0,d=parseInt(g[b],10)||0;return c+d})}if(a.browser.msie&&(!!a(c).is(":hidden")||!!a(c).parents(":hidden").length))continue;e.css({height:c.height()-this.borderDif[0]-this.borderDif[2]||0,width:c.width()-this.borderDif[1]-this.borderDif[3]||0})}}},_renderProxy:function(){var b=this.element,c=this.options;this.elementOffset=b.offset();if(this._helper){this.helper=this.helper||a('<div style="overflow:hidden;"></div>');var d=a.browser.msie&&a.browser.version<7,e=d?1:0,f=d?2:-1;this.helper.addClass(this._helper).css({width:this.element.outerWidth()+f,height:this.element.outerHeight()+f,position:"absolute",left:this.elementOffset.left-e+"px",top:this.elementOffset.top-e+"px",zIndex:++c.zIndex}),this.helper.appendTo("body").disableSelection()}else this.helper=this.element},_change:{e:function(a,b,c){return{width:this.originalSize.width+b}},w:function(a,b,c){var d=this.options,e=this.originalSize,f=this.originalPosition;return{left:f.left+b,width:e.width-b}},n:function(a,b,c){var d=this.options,e=this.originalSize,f=this.originalPosition;return{top:f.top+c,height:e.height-c}},s:function(a,b,c){return{height:this.originalSize.height+c}},se:function(b,c,d){return a.extend(this._change.s.apply(this,arguments),this._change.e.apply(this,[b,c,d]))},sw:function(b,c,d){return a.extend(this._change.s.apply(this,arguments),this._change.w.apply(this,[b,c,d]))},ne:function(b,c,d){return a.extend(this._change.n.apply(this,arguments),this._change.e.apply(this,[b,c,d]))},nw:function(b,c,d){return a.extend(this._change.n.apply(this,arguments),this._change.w.apply(this,[b,c,d]))}},_propagate:function(b,c){a.ui.plugin.call(this,b,[c,this.ui()]),b!="resize"&&this._trigger(b,c,this.ui())},plugins:{},ui:function(){return{originalElement:this.originalElement,element:this.element,helper:this.helper,position:this.position,size:this.size,originalSize:this.originalSize,originalPosition:this.originalPosition}}}),a.extend(a.ui.resizable,{version:"1.8.17"}),a.ui.plugin.add("resizable","alsoResize",{start:function(b,c){var d=a(this).data("resizable"),e=d.options,f=function(b){a(b).each(function(){var b=a(this);b.data("resizable-alsoresize",{width:parseInt(b.width(),10),height:parseInt(b.height(),10),left:parseInt(b.css("left"),10),top:parseInt(b.css("top"),10),position:b.css("position")})})};typeof e.alsoResize=="object"&&!e.alsoResize.parentNode?e.alsoResize.length?(e.alsoResize=e.alsoResize[0],f(e.alsoResize)):a.each(e.alsoResize,function(a){f(a)}):f(e.alsoResize)},resize:function(b,c){var d=a(this).data("resizable"),e=d.options,f=d.originalSize,g=d.originalPosition,h={height:d.size.height-f.height||0,width:d.size.width-f.width||0,top:d.position.top-g.top||0,left:d.position.left-g.left||0},i=function(b,e){a(b).each(function(){var b=a(this),f=a(this).data("resizable-alsoresize"),g={},i=e&&e.length?e:b.parents(c.originalElement[0]).length?["width","height"]:["width","height","top","left"];a.each(i,function(a,b){var c=(f[b]||0)+(h[b]||0);c&&c>=0&&(g[b]=c||null)}),a.browser.opera&&/relative/.test(b.css("position"))&&(d._revertToRelativePosition=!0,b.css({position:"absolute",top:"auto",left:"auto"})),b.css(g)})};typeof e.alsoResize=="object"&&!e.alsoResize.nodeType?a.each(e.alsoResize,function(a,b){i(a,b)}):i(e.alsoResize)},stop:function(b,c){var d=a(this).data("resizable"),e=d.options,f=function(b){a(b).each(function(){var b=a(this);b.css({position:b.data("resizable-alsoresize").position})})};d._revertToRelativePosition&&(d._revertToRelativePosition=!1,typeof e.alsoResize=="object"&&!e.alsoResize.nodeType?a.each(e.alsoResize,function(a){f(a)}):f(e.alsoResize)),a(this).removeData("resizable-alsoresize")}}),a.ui.plugin.add("resizable","animate",{stop:function(b,c){var d=a(this).data("resizable"),e=d.options,f=d._proportionallyResizeElements,g=f.length&&/textarea/i.test(f[0].nodeName),h=g&&a.ui.hasScroll(f[0],"left")?0:d.sizeDiff.height,i=g?0:d.sizeDiff.width,j={width:d.size.width-i,height:d.size.height-h},k=parseInt(d.element.css("left"),10)+(d.position.left-d.originalPosition.left)||null,l=parseInt(d.element.css("top"),10)+(d.position.top-d.originalPosition.top)||null;d.element.animate(a.extend(j,l&&k?{top:l,left:k}:{}),{duration:e.animateDuration,easing:e.animateEasing,step:function(){var c={width:parseInt(d.element.css("width"),10),height:parseInt(d.element.css("height"),10),top:parseInt(d.element.css("top"),10),left:parseInt(d.element.css("left"),10)};f&&f.length&&a(f[0]).css({width:c.width,height:c.height}),d._updateCache(c),d._propagate("resize",b)}})}}),a.ui.plugin.add("resizable","containment",{start:function(b,d){var e=a(this).data("resizable"),f=e.options,g=e.element,h=f.containment,i=h instanceof a?h.get(0):/parent/.test(h)?g.parent().get(0):h;if(!!i){e.containerElement=a(i);if(/document/.test(h)||h==document)e.containerOffset={left:0,top:0},e.containerPosition={left:0,top:0},e.parentData={element:a(document),left:0,top:0,width:a(document).width(),height:a(document).height()||document.body.parentNode.scrollHeight};else{var j=a(i),k=[];a(["Top","Right","Left","Bottom"]).each(function(a,b){k[a]=c(j.css("padding"+b))}),e.containerOffset=j.offset(),e.containerPosition=j.position(),e.containerSize={height:j.innerHeight()-k[3],width:j.innerWidth()-k[1]};var l=e.containerOffset,m=e.containerSize.height,n=e.containerSize.width,o=a.ui.hasScroll(i,"left")?i.scrollWidth:n,p=a.ui.hasScroll(i)?i.scrollHeight:m;e.parentData={element:i,left:l.left,top:l.top,width:o,height:p}}}},resize:function(b,c){var d=a(this).data("resizable"),e=d.options,f=d.containerSize,g=d.containerOffset,h=d.size,i=d.position,j=d._aspectRatio||b.shiftKey,k={top:0,left:0},l=d.containerElement;l[0]!=document&&/static/.test(l.css("position"))&&(k=g),i.left<(d._helper?g.left:0)&&(d.size.width=d.size.width+(d._helper?d.position.left-g.left:d.position.left-k.left),j&&(d.size.height=d.size.width/e.aspectRatio),d.position.left=e.helper?g.left:0),i.top<(d._helper?g.top:0)&&(d.size.height=d.size.height+(d._helper?d.position.top-g.top:d.position.top),j&&(d.size.width=d.size.height*e.aspectRatio),d.position.top=d._helper?g.top:0),d.offset.left=d.parentData.left+d.position.left,d.offset.top=d.parentData.top+d.position.top;var m=Math.abs((d._helper?d.offset.left-k.left:d.offset.left-k.left)+d.sizeDiff.width),n=Math.abs((d._helper?d.offset.top-k.top:d.offset.top-g.top)+d.sizeDiff.height),o=d.containerElement.get(0)==d.element.parent().get(0),p=/relative|absolute/.test(d.containerElement.css("position"));o&&p&&(m-=d.parentData.left),m+d.size.width>=d.parentData.width&&(d.size.width=d.parentData.width-m,j&&(d.size.height=d.size.width/d.aspectRatio)),n+d.size.height>=d.parentData.height&&(d.size.height=d.parentData.height-n,j&&(d.size.width=d.size.height*d.aspectRatio))},stop:function(b,c){var d=a(this).data("resizable"),e=d.options,f=d.position,g=d.containerOffset,h=d.containerPosition,i=d.containerElement,j=a(d.helper),k=j.offset(),l=j.outerWidth()-d.sizeDiff.width,m=j.outerHeight()-d.sizeDiff.height;d._helper&&!e.animate&&/relative/.test(i.css("position"))&&a(this).css({left:k.left-h.left-g.left,width:l,height:m}),d._helper&&!e.animate&&/static/.test(i.css("position"))&&a(this).css({left:k.left-h.left-g.left,width:l,height:m})}}),a.ui.plugin.add("resizable","ghost",{start:function(b,c){var d=a(this).data("resizable"),e=d.options,f=d.size;d.ghost=d.originalElement.clone(),d.ghost.css({opacity:.25,display:"block",position:"relative",height:f.height,width:f.width,margin:0,left:0,top:0}).addClass("ui-resizable-ghost").addClass(typeof e.ghost=="string"?e.ghost:""),d.ghost.appendTo(d.helper)},resize:function(b,c){var d=a(this).data("resizable"),e=d.options;d.ghost&&d.ghost.css({position:"relative",height:d.size.height,width:d.size.width})},stop:function(b,c){var d=a(this).data("resizable"),e=d.options;d.ghost&&d.helper&&d.helper.get(0).removeChild(d.ghost.get(0))}}),a.ui.plugin.add("resizable","grid",{resize:function(b,c){var d=a(this).data("resizable"),e=d.options,f=d.size,g=d.originalSize,h=d.originalPosition,i=d.axis,j=e._aspectRatio||b.shiftKey;e.grid=typeof e.grid=="number"?[e.grid,e.grid]:e.grid;var k=Math.round((f.width-g.width)/(e.grid[0]||1))*(e.grid[0]||1),l=Math.round((f.height-g.height)/(e.grid[1]||1))*(e.grid[1]||1);/^(se|s|e)$/.test(i)?(d.size.width=g.width+k,d.size.height=g.height+l):/^(ne)$/.test(i)?(d.size.width=g.width+k,d.size.height=g.height+l,d.position.top=h.top-l):/^(sw)$/.test(i)?(d.size.width=g.width+k,d.size.height=g.height+l,d.position.left=h.left-k):(d.size.width=g.width+k,d.size.height=g.height+l,d.position.top=h.top-l,d.position.left=h.left-k)}});var c=function(a){return parseInt(a,10)||0},d=function(a){return!isNaN(parseInt(a,10))}})(jQuery);/*
+ * jQuery UI Selectable 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Selectables
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.mouse.js
+ * jquery.ui.widget.js
+ */(function(a,b){a.widget("ui.selectable",a.ui.mouse,{options:{appendTo:"body",autoRefresh:!0,distance:0,filter:"*",tolerance:"touch"},_create:function(){var b=this;this.element.addClass("ui-selectable"),this.dragged=!1;var c;this.refresh=function(){c=a(b.options.filter,b.element[0]),c.addClass("ui-selectee"),c.each(function(){var b=a(this),c=b.offset();a.data(this,"selectable-item",{element:this,$element:b,left:c.left,top:c.top,right:c.left+b.outerWidth(),bottom:c.top+b.outerHeight(),startselected:!1,selected:b.hasClass("ui-selected"),selecting:b.hasClass("ui-selecting"),unselecting:b.hasClass("ui-unselecting")})})},this.refresh(),this.selectees=c.addClass("ui-selectee"),this._mouseInit(),this.helper=a("<div class='ui-selectable-helper'></div>")},destroy:function(){this.selectees.removeClass("ui-selectee").removeData("selectable-item"),this.element.removeClass("ui-selectable ui-selectable-disabled").removeData("selectable").unbind(".selectable"),this._mouseDestroy();return this},_mouseStart:function(b){var c=this;this.opos=[b.pageX,b.pageY];if(!this.options.disabled){var d=this.options;this.selectees=a(d.filter,this.element[0]),this._trigger("start",b),a(d.appendTo).append(this.helper),this.helper.css({left:b.clientX,top:b.clientY,width:0,height:0}),d.autoRefresh&&this.refresh(),this.selectees.filter(".ui-selected").each(function(){var d=a.data(this,"selectable-item");d.startselected=!0,!b.metaKey&&!b.ctrlKey&&(d.$element.removeClass("ui-selected"),d.selected=!1,d.$element.addClass("ui-unselecting"),d.unselecting=!0,c._trigger("unselecting",b,{unselecting:d.element}))}),a(b.target).parents().andSelf().each(function(){var d=a.data(this,"selectable-item");if(d){var e=!b.metaKey&&!b.ctrlKey||!d.$element.hasClass("ui-selected");d.$element.removeClass(e?"ui-unselecting":"ui-selected").addClass(e?"ui-selecting":"ui-unselecting"),d.unselecting=!e,d.selecting=e,d.selected=e,e?c._trigger("selecting",b,{selecting:d.element}):c._trigger("unselecting",b,{unselecting:d.element});return!1}})}},_mouseDrag:function(b){var c=this;this.dragged=!0;if(!this.options.disabled){var d=this.options,e=this.opos[0],f=this.opos[1],g=b.pageX,h=b.pageY;if(e>g){var i=g;g=e,e=i}if(f>h){var i=h;h=f,f=i}this.helper.css({left:e,top:f,width:g-e,height:h-f}),this.selectees.each(function(){var i=a.data(this,"selectable-item");if(!!i&&i.element!=c.element[0]){var j=!1;d.tolerance=="touch"?j=!(i.left>g||i.right<e||i.top>h||i.bottom<f):d.tolerance=="fit"&&(j=i.left>e&&i.right<g&&i.top>f&&i.bottom<h),j?(i.selected&&(i.$element.removeClass("ui-selected"),i.selected=!1),i.unselecting&&(i.$element.removeClass("ui-unselecting"),i.unselecting=!1),i.selecting||(i.$element.addClass("ui-selecting"),i.selecting=!0,c._trigger("selecting",b,{selecting:i.element}))):(i.selecting&&((b.metaKey||b.ctrlKey)&&i.startselected?(i.$element.removeClass("ui-selecting"),i.selecting=!1,i.$element.addClass("ui-selected"),i.selected=!0):(i.$element.removeClass("ui-selecting"),i.selecting=!1,i.startselected&&(i.$element.addClass("ui-unselecting"),i.unselecting=!0),c._trigger("unselecting",b,{unselecting:i.element}))),i.selected&&!b.metaKey&&!b.ctrlKey&&!i.startselected&&(i.$element.removeClass("ui-selected"),i.selected=!1,i.$element.addClass("ui-unselecting"),i.unselecting=!0,c._trigger("unselecting",b,{unselecting:i.element})))}});return!1}},_mouseStop:function(b){var c=this;this.dragged=!1;var d=this.options;a(".ui-unselecting",this.element[0]).each(function(){var d=a.data(this,"selectable-item");d.$element.removeClass("ui-unselecting"),d.unselecting=!1,d.startselected=!1,c._trigger("unselected",b,{unselected:d.element})}),a(".ui-selecting",this.element[0]).each(function(){var d=a.data(this,"selectable-item");d.$element.removeClass("ui-selecting").addClass("ui-selected"),d.selecting=!1,d.selected=!0,d.startselected=!0,c._trigger("selected",b,{selected:d.element})}),this._trigger("stop",b),this.helper.remove();return!1}}),a.extend(a.ui.selectable,{version:"1.8.17"})})(jQuery);/*
+ * jQuery UI Sortable 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Sortables
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.mouse.js
+ * jquery.ui.widget.js
+ */(function(a,b){a.widget("ui.sortable",a.ui.mouse,{widgetEventPrefix:"sort",options:{appendTo:"parent",axis:!1,connectWith:!1,containment:!1,cursor:"auto",cursorAt:!1,dropOnEmpty:!0,forcePlaceholderSize:!1,forceHelperSize:!1,grid:!1,handle:!1,helper:"original",items:"> *",opacity:!1,placeholder:!1,revert:!1,scroll:!0,scrollSensitivity:20,scrollSpeed:20,scope:"default",tolerance:"intersect",zIndex:1e3},_create:function(){var a=this.options;this.containerCache={},this.element.addClass("ui-sortable"),this.refresh(),this.floating=this.items.length?a.axis==="x"||/left|right/.test(this.items[0].item.css("float"))||/inline|table-cell/.test(this.items[0].item.css("display")):!1,this.offset=this.element.offset(),this._mouseInit()},destroy:function(){this.element.removeClass("ui-sortable ui-sortable-disabled"),this._mouseDestroy();for(var a=this.items.length-1;a>=0;a--)this.items[a].item.removeData(this.widgetName+"-item");return this},_setOption:function(b,c){b==="disabled"?(this.options[b]=c,this.widget()[c?"addClass":"removeClass"]("ui-sortable-disabled")):a.Widget.prototype._setOption.apply(this,arguments)},_mouseCapture:function(b,c){var d=this;if(this.reverting)return!1;if(this.options.disabled||this.options.type=="static")return!1;this._refreshItems(b);var e=null,f=this,g=a(b.target).parents().each(function(){if(a.data(this,d.widgetName+"-item")==f){e=a(this);return!1}});a.data(b.target,d.widgetName+"-item")==f&&(e=a(b.target));if(!e)return!1;if(this.options.handle&&!c){var h=!1;a(this.options.handle,e).find("*").andSelf().each(function(){this==b.target&&(h=!0)});if(!h)return!1}this.currentItem=e,this._removeCurrentsFromItems();return!0},_mouseStart:function(b,c,d){var e=this.options,f=this;this.currentContainer=this,this.refreshPositions(),this.helper=this._createHelper(b),this._cacheHelperProportions(),this._cacheMargins(),this.scrollParent=this.helper.scrollParent(),this.offset=this.currentItem.offset(),this.offset={top:this.offset.top-this.margins.top,left:this.offset.left-this.margins.left},this.helper.css("position","absolute"),this.cssPosition=this.helper.css("position"),a.extend(this.offset,{click:{left:b.pageX-this.offset.left,top:b.pageY-this.offset.top},parent:this._getParentOffset(),relative:this._getRelativeOffset()}),this.originalPosition=this._generatePosition(b),this.originalPageX=b.pageX,this.originalPageY=b.pageY,e.cursorAt&&this._adjustOffsetFromHelper(e.cursorAt),this.domPosition={prev:this.currentItem.prev()[0],parent:this.currentItem.parent()[0]},this.helper[0]!=this.currentItem[0]&&this.currentItem.hide(),this._createPlaceholder(),e.containment&&this._setContainment(),e.cursor&&(a("body").css("cursor")&&(this._storedCursor=a("body").css("cursor")),a("body").css("cursor",e.cursor)),e.opacity&&(this.helper.css("opacity")&&(this._storedOpacity=this.helper.css("opacity")),this.helper.css("opacity",e.opacity)),e.zIndex&&(this.helper.css("zIndex")&&(this._storedZIndex=this.helper.css("zIndex")),this.helper.css("zIndex",e.zIndex)),this.scrollParent[0]!=document&&this.scrollParent[0].tagName!="HTML"&&(this.overflowOffset=this.scrollParent.offset()),this._trigger("start",b,this._uiHash()),this._preserveHelperProportions||this._cacheHelperProportions();if(!d)for(var g=this.containers.length-1;g>=0;g--)this.containers[g]._trigger("activate",b,f._uiHash(this));a.ui.ddmanager&&(a.ui.ddmanager.current=this),a.ui.ddmanager&&!e.dropBehaviour&&a.ui.ddmanager.prepareOffsets(this,b),this.dragging=!0,this.helper.addClass("ui-sortable-helper"),this._mouseDrag(b);return!0},_mouseDrag:function(b){this.position=this._generatePosition(b),this.positionAbs=this._convertPositionTo("absolute"),this.lastPositionAbs||(this.lastPositionAbs=this.positionAbs);if(this.options.scroll){var c=this.options,d=!1;this.scrollParent[0]!=document&&this.scrollParent[0].tagName!="HTML"?(this.overflowOffset.top+this.scrollParent[0].offsetHeight-b.pageY<c.scrollSensitivity?this.scrollParent[0].scrollTop=d=this.scrollParent[0].scrollTop+c.scrollSpeed:b.pageY-this.overflowOffset.top<c.scrollSensitivity&&(this.scrollParent[0].scrollTop=d=this.scrollParent[0].scrollTop-c.scrollSpeed),this.overflowOffset.left+this.scrollParent[0].offsetWidth-b.pageX<c.scrollSensitivity?this.scrollParent[0].scrollLeft=d=this.scrollParent[0].scrollLeft+c.scrollSpeed:b.pageX-this.overflowOffset.left<c.scrollSensitivity&&(this.scrollParent[0].scrollLeft=d=this.scrollParent[0].scrollLeft-c.scrollSpeed)):(b.pageY-a(document).scrollTop()<c.scrollSensitivity?d=a(document).scrollTop(a(document).scrollTop()-c.scrollSpeed):a(window).height()-(b.pageY-a(document).scrollTop())<c.scrollSensitivity&&(d=a(document).scrollTop(a(document).scrollTop()+c.scrollSpeed)),b.pageX-a(document).scrollLeft()<c.scrollSensitivity?d=a(document).scrollLeft(a(document).scrollLeft()-c.scrollSpeed):a(window).width()-(b.pageX-a(document).scrollLeft())<c.scrollSensitivity&&(d=a(document).scrollLeft(a(document).scrollLeft()+c.scrollSpeed))),d!==!1&&a.ui.ddmanager&&!c.dropBehaviour&&a.ui.ddmanager.prepareOffsets(this,b)}this.positionAbs=this._convertPositionTo("absolute");if(!this.options.axis||this.options.axis!="y")this.helper[0].style.left=this.position.left+"px";if(!this.options.axis||this.options.axis!="x")this.helper[0].style.top=this.position.top+"px";for(var e=this.items.length-1;e>=0;e--){var f=this.items[e],g=f.item[0],h=this._intersectsWithPointer(f);if(!h)continue;if(g!=this.currentItem[0]&&this.placeholder[h==1?"next":"prev"]()[0]!=g&&!a.ui.contains(this.placeholder[0],g)&&(this.options.type=="semi-dynamic"?!a.ui.contains(this.element[0],g):!0)){this.direction=h==1?"down":"up";if(this.options.tolerance=="pointer"||this._intersectsWithSides(f))this._rearrange(b,f);else break;this._trigger("change",b,this._uiHash());break}}this._contactContainers(b),a.ui.ddmanager&&a.ui.ddmanager.drag(this,b),this._trigger("sort",b,this._uiHash()),this.lastPositionAbs=this.positionAbs;return!1},_mouseStop:function(b,c){if(!!b){a.ui.ddmanager&&!this.options.dropBehaviour&&a.ui.ddmanager.drop(this,b);if(this.options.revert){var d=this,e=d.placeholder.offset();d.reverting=!0,a(this.helper).animate({left:e.left-this.offset.parent.left-d.margins.left+(this.offsetParent[0]==document.body?0:this.offsetParent[0].scrollLeft),top:e.top-this.offset.parent.top-d.margins.top+(this.offsetParent[0]==document.body?0:this.offsetParent[0].scrollTop)},parseInt(this.options.revert,10)||500,function(){d._clear(b)})}else this._clear(b,c);return!1}},cancel:function(){var b=this;if(this.dragging){this._mouseUp({target:null}),this.options.helper=="original"?this.currentItem.css(this._storedCSS).removeClass("ui-sortable-helper"):this.currentItem.show();for(var c=this.containers.length-1;c>=0;c--)this.containers[c]._trigger("deactivate",null,b._uiHash(this)),this.containers[c].containerCache.over&&(this.containers[c]._trigger("out",null,b._uiHash(this)),this.containers[c].containerCache.over=0)}this.placeholder&&(this.placeholder[0].parentNode&&this.placeholder[0].parentNode.removeChild(this.placeholder[0]),this.options.helper!="original"&&this.helper&&this.helper[0].parentNode&&this.helper.remove(),a.extend(this,{helper:null,dragging:!1,reverting:!1,_noFinalSort:null}),this.domPosition.prev?a(this.domPosition.prev).after(this.currentItem):a(this.domPosition.parent).prepend(this.currentItem));return this},serialize:function(b){var c=this._getItemsAsjQuery(b&&b.connected),d=[];b=b||{},a(c).each(function(){var c=(a(b.item||this).attr(b.attribute||"id")||"").match(b.expression||/(.+)[-=_](.+)/);c&&d.push((b.key||c[1]+"[]")+"="+(b.key&&b.expression?c[1]:c[2]))}),!d.length&&b.key&&d.push(b.key+"=");return d.join("&")},toArray:function(b){var c=this._getItemsAsjQuery(b&&b.connected),d=[];b=b||{},c.each(function(){d.push(a(b.item||this).attr(b.attribute||"id")||"")});return d},_intersectsWith:function(a){var b=this.positionAbs.left,c=b+this.helperProportions.width,d=this.positionAbs.top,e=d+this.helperProportions.height,f=a.left,g=f+a.width,h=a.top,i=h+a.height,j=this.offset.click.top,k=this.offset.click.left,l=d+j>h&&d+j<i&&b+k>f&&b+k<g;return this.options.tolerance=="pointer"||this.options.forcePointerForContainers||this.options.tolerance!="pointer"&&this.helperProportions[this.floating?"width":"height"]>a[this.floating?"width":"height"]?l:f<b+this.helperProportions.width/2&&c-this.helperProportions.width/2<g&&h<d+this.helperProportions.height/2&&e-this.helperProportions.height/2<i},_intersectsWithPointer:function(b){var c=a.ui.isOverAxis(this.positionAbs.top+this.offset.click.top,b.top,b.height),d=a.ui.isOverAxis(this.positionAbs.left+this.offset.click.left,b.left,b.width),e=c&&d,f=this._getDragVerticalDirection(),g=this._getDragHorizontalDirection();if(!e)return!1;return this.floating?g&&g=="right"||f=="down"?2:1:f&&(f=="down"?2:1)},_intersectsWithSides:function(b){var c=a.ui.isOverAxis(this.positionAbs.top+this.offset.click.top,b.top+b.height/2,b.height),d=a.ui.isOverAxis(this.positionAbs.left+this.offset.click.left,b.left+b.width/2,b.width),e=this._getDragVerticalDirection(),f=this._getDragHorizontalDirection();return this.floating&&f?f=="right"&&d||f=="left"&&!d:e&&(e=="down"&&c||e=="up"&&!c)},_getDragVerticalDirection:function(){var a=this.positionAbs.top-this.lastPositionAbs.top;return a!=0&&(a>0?"down":"up")},_getDragHorizontalDirection:function(){var a=this.positionAbs.left-this.lastPositionAbs.left;return a!=0&&(a>0?"right":"left")},refresh:function(a){this._refreshItems(a),this.refreshPositions();return this},_connectWith:function(){var a=this.options;return a.connectWith.constructor==String?[a.connectWith]:a.connectWith},_getItemsAsjQuery:function(b){var c=this,d=[],e=[],f=this._connectWith();if(f&&b)for(var g=f.length-1;g>=0;g--){var h=a(f[g]);for(var i=h.length-1;i>=0;i--){var j=a.data(h[i],this.widgetName);j&&j!=this&&!j.options.disabled&&e.push([a.isFunction(j.options.items)?j.options.items.call(j.element):a(j.options.items,j.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),j])}}e.push([a.isFunction(this.options.items)?this.options.items.call(this.element,null,{options:this.options,item:this.currentItem}):a(this.options.items,this.element).not(".ui-sortable-helper").not(".ui-sortable-placeholder"),this]);for(var g=e.length-1;g>=0;g--)e[g][0].each(function(){d.push(this)});return a(d)},_removeCurrentsFromItems:function(){var a=this.currentItem.find(":data("+this.widgetName+"-item)");for(var b=0;b<this.items.length;b++)for(var c=0;c<a.length;c++)a[c]==this.items[b].item[0]&&this.items.splice(b,1)},_refreshItems:function(b){this.items=[],this.containers=[this];var c=this.items,d=this,e=[[a.isFunction(this.options.items)?this.options.items.call(this.element[0],b,{item:this.currentItem}):a(this.options.items,this.element),this]],f=this._connectWith();if(f)for(var g=f.length-1;g>=0;g--){var h=a(f[g]);for(var i=h.length-1;i>=0;i--){var j=a.data(h[i],this.widgetName);j&&j!=this&&!j.options.disabled&&(e.push([a.isFunction(j.options.items)?j.options.items.call(j.element[0],b,{item:this.currentItem}):a(j.options.items,j.element),j]),this.containers.push(j))}}for(var g=e.length-1;g>=0;g--){var k=e[g][1],l=e[g][0];for(var i=0,m=l.length;i<m;i++){var n=a(l[i]);n.data(this.widgetName+"-item",k),c.push({item:n,instance:k,width:0,height:0,left:0,top:0})}}},refreshPositions:function(b){this.offsetParent&&this.helper&&(this.offset.parent=this._getParentOffset());for(var c=this.items.length-1;c>=0;c--){var d=this.items[c];if(d.instance!=this.currentContainer&&this.currentContainer&&d.item[0]!=this.currentItem[0])continue;var e=this.options.toleranceElement?a(this.options.toleranceElement,d.item):d.item;b||(d.width=e.outerWidth(),d.height=e.outerHeight());var f=e.offset();d.left=f.left,d.top=f.top}if(this.options.custom&&this.options.custom.refreshContainers)this.options.custom.refreshContainers.call(this);else for(var c=this.containers.length-1;c>=0;c--){var f=this.containers[c].element.offset();this.containers[c].containerCache.left=f.left,this.containers[c].containerCache.top=f.top,this.containers[c].containerCache.width=this.containers[c].element.outerWidth(),this.containers[c].containerCache.height=this.containers[c].element.outerHeight()}return this},_createPlaceholder:function(b){var c=b||this,d=c.options;if(!d.placeholder||d.placeholder.constructor==String){var e=d.placeholder;d.placeholder={element:function(){var b=a(document.createElement(c.currentItem[0].nodeName)).addClass(e||c.currentItem[0].className+" ui-sortable-placeholder").removeClass("ui-sortable-helper")[0];e||(b.style.visibility="hidden");return b},update:function(a,b){if(!e||!!d.forcePlaceholderSize)b.height()||b.height(c.currentItem.innerHeight()-parseInt(c.currentItem.css("paddingTop")||0,10)-parseInt(c.currentItem.css("paddingBottom")||0,10)),b.width()||b.width(c.currentItem.innerWidth()-parseInt(c.currentItem.css("paddingLeft")||0,10)-parseInt(c.currentItem.css("paddingRight")||0,10))}}}c.placeholder=a(d.placeholder.element.call(c.element,c.currentItem)),c.currentItem.after(c.placeholder),d.placeholder.update(c,c.placeholder)},_contactContainers:function(b){var c=null,d=null;for(var e=this.containers.length-1;e>=0;e--){if(a.ui.contains(this.currentItem[0],this.containers[e].element[0]))continue;if(this._intersectsWith(this.containers[e].containerCache)){if(c&&a.ui.contains(this.containers[e].element[0],c.element[0]))continue;c=this.containers[e],d=e}else this.containers[e].containerCache.over&&(this.containers[e]._trigger("out",b,this._uiHash(this)),this.containers[e].containerCache.over=0)}if(!!c)if(this.containers.length===1)this.containers[d]._trigger("over",b,this._uiHash(this)),this.containers[d].containerCache.over=1;else if(this.currentContainer!=this.containers[d]){var f=1e4,g=null,h=this.positionAbs[this.containers[d].floating?"left":"top"];for(var i=this.items.length-1;i>=0;i--){if(!a.ui.contains(this.containers[d].element[0],this.items[i].item[0]))continue;var j=this.items[i][this.containers[d].floating?"left":"top"];Math.abs(j-h)<f&&(f=Math.abs(j-h),g=this.items[i])}if(!g&&!this.options.dropOnEmpty)return;this.currentContainer=this.containers[d],g?this._rearrange(b,g,null,!0):this._rearrange(b,null,this.containers[d].element,!0),this._trigger("change",b,this._uiHash()),this.containers[d]._trigger("change",b,this._uiHash(this)),this.options.placeholder.update(this.currentContainer,this.placeholder),this.containers[d]._trigger("over",b,this._uiHash(this)),this.containers[d].containerCache.over=1}},_createHelper:function(b){var c=this.options,d=a.isFunction(c.helper)?a(c.helper.apply(this.element[0],[b,this.currentItem])):c.helper=="clone"?this.currentItem.clone():this.currentItem;d.parents("body").length||a(c.appendTo!="parent"?c.appendTo:this.currentItem[0].parentNode)[0].appendChild(d[0]),d[0]==this.currentItem[0]&&(this._storedCSS={width:this.currentItem[0].style.width,height:this.currentItem[0].style.height,position:this.currentItem.css("position"),top:this.currentItem.css("top"),left:this.currentItem.css("left")}),(d[0].style.width==""||c.forceHelperSize)&&d.width(this.currentItem.width()),(d[0].style.height==""||c.forceHelperSize)&&d.height(this.currentItem.height());return d},_adjustOffsetFromHelper:function(b){typeof b=="string"&&(b=b.split(" ")),a.isArray(b)&&(b={left:+b[0],top:+b[1]||0}),"left"in b&&(this.offset.click.left=b.left+this.margins.left),"right"in b&&(this.offset.click.left=this.helperProportions.width-b.right+this.margins.left),"top"in b&&(this.offset.click.top=b.top+this.margins.top),"bottom"in b&&(this.offset.click.top=this.helperProportions.height-b.bottom+this.margins.top)},_getParentOffset:function(){this.offsetParent=this.helper.offsetParent();var b=this.offsetParent.offset();this.cssPosition=="absolute"&&this.scrollParent[0]!=document&&a.ui.contains(this.scrollParent[0],this.offsetParent[0])&&(b.left+=this.scrollParent.scrollLeft(),b.top+=this.scrollParent.scrollTop());if(this.offsetParent[0]==document.body||this.offsetParent[0].tagName&&this.offsetParent[0].tagName.toLowerCase()=="html"&&a.browser.msie)b={top:0,left:0};return{top:b.top+(parseInt(this.offsetParent.css("borderTopWidth"),10)||0),left:b.left+(parseInt(this.offsetParent.css("borderLeftWidth"),10)||0)}},_getRelativeOffset:function(){if(this.cssPosition=="relative"){var a=this.currentItem.position();return{top:a.top-(parseInt(this.helper.css("top"),10)||0)+this.scrollParent.scrollTop(),left:a.left-(parseInt(this.helper.css("left"),10)||0)+this.scrollParent.scrollLeft()}}return{top:0,left:0}},_cacheMargins:function(){this.margins={left:parseInt(this.currentItem.css("marginLeft"),10)||0,top:parseInt(this.currentItem.css("marginTop"),10)||0}},_cacheHelperProportions:function(){this.helperProportions={width:this.helper.outerWidth(),height:this.helper.outerHeight()}},_setContainment:function(){var b=this.options;b.containment=="parent"&&(b.containment=this.helper[0].parentNode);if(b.containment=="document"||b.containment=="window")this.containment=[0-this.offset.relative.left-this.offset.parent.left,0-this.offset.relative.top-this.offset.parent.top,a(b.containment=="document"?document:window).width()-this.helperProportions.width-this.margins.left,(a(b.containment=="document"?document:window).height()||document.body.parentNode.scrollHeight)-this.helperProportions.height-this.margins.top];if(!/^(document|window|parent)$/.test(b.containment)){var c=a(b.containment)[0],d=a(b.containment).offset(),e=a(c).css("overflow")!="hidden";this.containment=[d.left+(parseInt(a(c).css("borderLeftWidth"),10)||0)+(parseInt(a(c).css("paddingLeft"),10)||0)-this.margins.left,d.top+(parseInt(a(c).css("borderTopWidth"),10)||0)+(parseInt(a(c).css("paddingTop"),10)||0)-this.margins.top,d.left+(e?Math.max(c.scrollWidth,c.offsetWidth):c.offsetWidth)-(parseInt(a(c).css("borderLeftWidth"),10)||0)-(parseInt(a(c).css("paddingRight"),10)||0)-this.helperProportions.width-this.margins.left,d.top+(e?Math.max(c.scrollHeight,c.offsetHeight):c.offsetHeight)-(parseInt(a(c).css("borderTopWidth"),10)||0)-(parseInt(a(c).css("paddingBottom"),10)||0)-this.helperProportions.height-this.margins.top]}},_convertPositionTo:function(b,c){c||(c=this.position);var d=b=="absolute"?1:-1,e=this.options,f=this.cssPosition=="absolute"&&(this.scrollParent[0]==document||!a.ui.contains(this.scrollParent[0],this.offsetParent[0]))?this.offsetParent:this.scrollParent,g=/(html|body)/i.test(f[0].tagName);return{top:c.top+this.offset.relative.top*d+this.offset.parent.top*d-(a.browser.safari&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollTop():g?0:f.scrollTop())*d),left:c.left+this.offset.relative.left*d+this.offset.parent.left*d-(a.browser.safari&&this.cssPosition=="fixed"?0:(this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():g?0:f.scrollLeft())*d)}},_generatePosition:function(b){var c=this.options,d=this.cssPosition=="absolute"&&(this.scrollParent[0]==document||!a.ui.contains(this.scrollParent[0],this.offsetParent[0]))?this.offsetParent:this.scrollParent,e=/(html|body)/i.test(d[0].tagName);this.cssPosition=="relative"&&(this.scrollParent[0]==document||this.scrollParent[0]==this.offsetParent[0])&&(this.offset.relative=this._getRelativeOffset());var f=b.pageX,g=b.pageY;if(this.originalPosition){this.containment&&(b.pageX-this.offset.click.left<this.containment[0]&&(f=this.containment[0]+this.offset.click.left),b.pageY-this.offset.click.top<this.containment[1]&&(g=this.containment[1]+this.offset.click.top),b.pageX-this.offset.click.left>this.containment[2]&&(f=this.containment[2]+this.offset.click.left),b.pageY-this.offset.click.top>this.containment[3]&&(g=this.containment[3]+this.offset.click.top));if(c.grid){var h=this.originalPageY+Math.round((g-this.originalPageY)/c.grid[1])*c.grid[1];g=this.containment?h-this.offset.click.top<this.containment[1]||h-this.offset.click.top>this.containment[3]?h-this.offset.click.top<this.containment[1]?h+c.grid[1]:h-c.grid[1]:h:h;var i=this.originalPageX+Math.round((f-this.originalPageX)/c.grid[0])*c.grid[0];f=this.containment?i-this.offset.click.left<this.containment[0]||i-this.offset.click.left>this.containment[2]?i-this.offset.click.left<this.containment[0]?i+c.grid[0]:i-c.grid[0]:i:i}}return{top:g-this.offset.click.top-this.offset.relative.top-this.offset.parent.top+(a.browser.safari&&this.cssPosition=="fixed"?0:this.cssPosition=="fixed"?-this.scrollParent.scrollTop():e?0:d.scrollTop()),left:f-this.offset.click.left-this.offset.relative.left-this.offset.parent.left+(a.browser.safari&&this.cssPosition=="fixed"?0:this.cssPosition=="fixed"?-this.scrollParent.scrollLeft():e?0:d.scrollLeft())}},_rearrange:function(a,b,c,d){c?c[0].appendChild(this.placeholder[0]):b.item[0].parentNode.insertBefore(this.placeholder[0],this.direction=="down"?b.item[0]:b.item[0].nextSibling),this.counter=this.counter?++this.counter:1;var e=this,f=this.counter;window.setTimeout(function(){f==e.counter&&e.refreshPositions(!d)},0)},_clear:function(b,c){this.reverting=!1;var d=[],e=this;!this._noFinalSort&&this.currentItem.parent().length&&this.placeholder.before(this.currentItem),this._noFinalSort=null;if(this.helper[0]==this.currentItem[0]){for(var f in this._storedCSS)if(this._storedCSS[f]=="auto"||this._storedCSS[f]=="static")this._storedCSS[f]="";this.currentItem.css(this._storedCSS).removeClass("ui-sortable-helper")}else this.currentItem.show();this.fromOutside&&!c&&d.push(function(a){this._trigger("receive",a,this._uiHash(this.fromOutside))}),(this.fromOutside||this.domPosition.prev!=this.currentItem.prev().not(".ui-sortable-helper")[0]||this.domPosition.parent!=this.currentItem.parent()[0])&&!c&&d.push(function(a){this._trigger("update",a,this._uiHash())});if(!a.ui.contains(this.element[0],this.currentItem[0])){c||d.push(function(a){this._trigger("remove",a,this._uiHash())});for(var f=this.containers.length-1;f>=0;f--)a.ui.contains(this.containers[f].element[0],this.currentItem[0])&&!c&&(d.push(function(a){return function(b){a._trigger("receive",b,this._uiHash(this))}}.call(this,this.containers[f])),d.push(function(a){return function(b){a._trigger("update",b,this._uiHash(this))}}.call(this,this.containers[f])))}for(var f=this.containers.length-1;f>=0;f--)c||d.push(function(a){return function(b){a._trigger("deactivate",b,this._uiHash(this))}}.call(this,this.containers[f])),this.containers[f].containerCache.over&&(d.push(function(a){return function(b){a._trigger("out",b,this._uiHash(this))}}.call(this,this.containers[f])),this.containers[f].containerCache.over=0);this._storedCursor&&a("body").css("cursor",this._storedCursor),this._storedOpacity&&this.helper.css("opacity",this._storedOpacity),this._storedZIndex&&this.helper.css("zIndex",this._storedZIndex=="auto"?"":this._storedZIndex),this.dragging=!1;if(this.cancelHelperRemoval){if(!c){this._trigger("beforeStop",b,this._uiHash());for(var f=0;f<d.length;f++)d[f].call(this,b);this._trigger("stop",b,this._uiHash())}return!1}c||this._trigger("beforeStop",b,this._uiHash()),this.placeholder[0].parentNode.removeChild(this.placeholder[0]),this.helper[0]!=this.currentItem[0]&&this.helper.remove(),this.helper=null;if(!c){for(var f=0;f<d.length;f++)d[f].call(this,b);this._trigger("stop",b,this._uiHash())}this.fromOutside=!1;return!0},_trigger:function(){a.Widget.prototype._trigger.apply(this,arguments)===!1&&this.cancel()},_uiHash:function(b){var c=b||this;return{helper:c.helper,placeholder:c.placeholder||a([]),position:c.position,originalPosition:c.originalPosition,offset:c.positionAbs,item:c.currentItem,sender:b?b.element:null}}}),a.extend(a.ui.sortable,{version:"1.8.17"})})(jQuery);/*
+ * jQuery UI Accordion 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Accordion
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.widget.js
+ */(function(a,b){a.widget("ui.accordion",{options:{active:0,animated:"slide",autoHeight:!0,clearStyle:!1,collapsible:!1,event:"click",fillSpace:!1,header:"> li > :first-child,> :not(li):even",icons:{header:"ui-icon-triangle-1-e",headerSelected:"ui-icon-triangle-1-s"},navigation:!1,navigationFilter:function(){return this.href.toLowerCase()===location.href.toLowerCase()}},_create:function(){var b=this,c=b.options;b.running=0,b.element.addClass("ui-accordion ui-widget ui-helper-reset").children("li").addClass("ui-accordion-li-fix"),b.headers=b.element.find(c.header).addClass("ui-accordion-header ui-helper-reset ui-state-default ui-corner-all").bind("mouseenter.accordion",function(){c.disabled||a(this).addClass("ui-state-hover")}).bind("mouseleave.accordion",function(){c.disabled||a(this).removeClass("ui-state-hover")}).bind("focus.accordion",function(){c.disabled||a(this).addClass("ui-state-focus")}).bind("blur.accordion",function(){c.disabled||a(this).removeClass("ui-state-focus")}),b.headers.next().addClass("ui-accordion-content ui-helper-reset ui-widget-content ui-corner-bottom");if(c.navigation){var d=b.element.find("a").filter(c.navigationFilter).eq(0);if(d.length){var e=d.closest(".ui-accordion-header");e.length?b.active=e:b.active=d.closest(".ui-accordion-content").prev()}}b.active=b._findActive(b.active||c.active).addClass("ui-state-default ui-state-active").toggleClass("ui-corner-all").toggleClass("ui-corner-top"),b.active.next().addClass("ui-accordion-content-active"),b._createIcons(),b.resize(),b.element.attr("role","tablist"),b.headers.attr("role","tab").bind("keydown.accordion",function(a){return b._keydown(a)}).next().attr("role","tabpanel"),b.headers.not(b.active||"").attr({"aria-expanded":"false","aria-selected":"false",tabIndex:-1}).next().hide(),b.active.length?b.active.attr({"aria-expanded":"true","aria-selected":"true",tabIndex:0}):b.headers.eq(0).attr("tabIndex",0),a.browser.safari||b.headers.find("a").attr("tabIndex",-1),c.event&&b.headers.bind(c.event.split(" ").join(".accordion ")+".accordion",function(a){b._clickHandler.call(b,a,this),a.preventDefault()})},_createIcons:function(){var b=this.options;b.icons&&(a("<span></span>").addClass("ui-icon "+b.icons.header).prependTo(this.headers),this.active.children(".ui-icon").toggleClass(b.icons.header).toggleClass(b.icons.headerSelected),this.element.addClass("ui-accordion-icons"))},_destroyIcons:function(){this.headers.children(".ui-icon").remove(),this.element.removeClass("ui-accordion-icons")},destroy:function(){var b=this.options;this.element.removeClass("ui-accordion ui-widget ui-helper-reset").removeAttr("role"),this.headers.unbind(".accordion").removeClass("ui-accordion-header ui-accordion-disabled ui-helper-reset ui-state-default ui-corner-all ui-state-active ui-state-disabled ui-corner-top").removeAttr("role").removeAttr("aria-expanded").removeAttr("aria-selected").removeAttr("tabIndex"),this.headers.find("a").removeAttr("tabIndex"),this._destroyIcons();var c=this.headers.next().css("display","").removeAttr("role").removeClass("ui-helper-reset ui-widget-content ui-corner-bottom ui-accordion-content ui-accordion-content-active ui-accordion-disabled ui-state-disabled");(b.autoHeight||b.fillHeight)&&c.css("height","");return a.Widget.prototype.destroy.call(this)},_setOption:function(b,c){a.Widget.prototype._setOption.apply(this,arguments),b=="active"&&this.activate(c),b=="icons"&&(this._destroyIcons(),c&&this._createIcons()),b=="disabled"&&this.headers.add(this.headers.next())[c?"addClass":"removeClass"]("ui-accordion-disabled ui-state-disabled")},_keydown:function(b){if(!(this.options.disabled||b.altKey||b.ctrlKey)){var c=a.ui.keyCode,d=this.headers.length,e=this.headers.index(b.target),f=!1;switch(b.keyCode){case c.RIGHT:case c.DOWN:f=this.headers[(e+1)%d];break;case c.LEFT:case c.UP:f=this.headers[(e-1+d)%d];break;case c.SPACE:case c.ENTER:this._clickHandler({target:b.target},b.target),b.preventDefault()}if(f){a(b.target).attr("tabIndex",-1),a(f).attr("tabIndex",0),f.focus();return!1}return!0}},resize:function(){var b=this.options,c;if(b.fillSpace){if(a.browser.msie){var d=this.element.parent().css("overflow");this.element.parent().css("overflow","hidden")}c=this.element.parent().height(),a.browser.msie&&this.element.parent().css("overflow",d),this.headers.each(function(){c-=a(this).outerHeight(!0)}),this.headers.next().each(function(){a(this).height(Math.max(0,c-a(this).innerHeight()+a(this).height()))}).css("overflow","auto")}else b.autoHeight&&(c=0,this.headers.next().each(function(){c=Math.max(c,a(this).height("").height())}).height(c));return this},activate:function(a){this.options.active=a;var b=this._findActive(a)[0];this._clickHandler({target:b},b);return this},_findActive:function(b){return b?typeof b=="number"?this.headers.filter(":eq("+b+")"):this.headers.not(this.headers.not(b)):b===!1?a([]):this.headers.filter(":eq(0)")},_clickHandler:function(b,c){var d=this.options;if(!d.disabled){if(!b.target){if(!d.collapsible)return;this.active.removeClass("ui-state-active ui-corner-top").addClass("ui-state-default ui-corner-all").children(".ui-icon").removeClass(d.icons.headerSelected).addClass(d.icons.header),this.active.next().addClass("ui-accordion-content-active");var e=this.active.next(),f={options:d,newHeader:a([]),oldHeader:d.active,newContent:a([]),oldContent:e},g=this.active=a([]);this._toggle(g,e,f);return}var h=a(b.currentTarget||c),i=h[0]===this.active[0];d.active=d.collapsible&&i?!1:this.headers.index(h);if(this.running||!d.collapsible&&i)return;var j=this.active,g=h.next(),e=this.active.next(),f={options:d,newHeader:i&&d.collapsible?a([]):h,oldHeader:this.active,newContent:i&&d.collapsible?a([]):g,oldContent:e},k=this.headers.index(this.active[0])>this.headers.index(h[0]);this.active=i?a([]):h,this._toggle(g,e,f,i,k),j.removeClass("ui-state-active ui-corner-top").addClass("ui-state-default ui-corner-all").children(".ui-icon").removeClass(d.icons.headerSelected).addClass(d.icons.header),i||(h.removeClass("ui-state-default ui-corner-all").addClass("ui-state-active ui-corner-top").children(".ui-icon").removeClass(d.icons.header).addClass(d.icons.headerSelected),h.next().addClass("ui-accordion-content-active"));return}},_toggle:function(b,c,d,e,f){var g=this,h=g.options;g.toShow=b,g.toHide=c,g.data=d;var i=function(){if(!!g)return g._completed.apply(g,arguments)};g._trigger("changestart",null,g.data),g.running=c.size()===0?b.size():c.size();if(h.animated){var j={};h.collapsible&&e?j={toShow:a([]),toHide:c,complete:i,down:f,autoHeight:h.autoHeight||h.fillSpace}:j={toShow:b,toHide:c,complete:i,down:f,autoHeight:h.autoHeight||h.fillSpace},h.proxied||(h.proxied=h.animated),h.proxiedDuration||(h.proxiedDuration=h.duration),h.animated=a.isFunction(h.proxied)?h.proxied(j):h.proxied,h.duration=a.isFunction(h.proxiedDuration)?h.proxiedDuration(j):h.proxiedDuration;var k=a.ui.accordion.animations,l=h.duration,m=h.animated;m&&!k[m]&&!a.easing[m]&&(m="slide"),k[m]||(k[m]=function(a){this.slide(a,{easing:m,duration:l||700})}),k[m](j)}else h.collapsible&&e?b.toggle():(c.hide(),b.show()),i(!0);c.prev().attr({"aria-expanded":"false","aria-selected":"false",tabIndex:-1}).blur(),b.prev().attr({"aria-expanded":"true","aria-selected":"true",tabIndex:0}).focus()},_completed:function(a){this.running=a?0:--this.running;this.running||(this.options.clearStyle&&this.toShow.add(this.toHide).css({height:"",overflow:""}),this.toHide.removeClass("ui-accordion-content-active"),this.toHide.length&&(this.toHide.parent()[0].className=this.toHide.parent()[0].className),this._trigger("change",null,this.data))}}),a.extend(a.ui.accordion,{version:"1.8.17",animations:{slide:function(b,c){b=a.extend({easing:"swing",duration:300},b,c);if(!b.toHide.size())b.toShow.animate({height:"show",paddingTop:"show",paddingBottom:"show"},b);else{if(!b.toShow.size()){b.toHide.animate({height:"hide",paddingTop:"hide",paddingBottom:"hide"},b);return}var d=b.toShow.css("overflow"),e=0,f={},g={},h=["height","paddingTop","paddingBottom"],i,j=b.toShow;i=j[0].style.width,j.width(j.parent().width()-parseFloat(j.css("paddingLeft"))-parseFloat(j.css("paddingRight"))-(parseFloat(j.css("borderLeftWidth"))||0)-(parseFloat(j.css("borderRightWidth"))||0)),a.each(h,function(c,d){g[d]="hide";var e=(""+a.css(b.toShow[0],d)).match(/^([\d+-.]+)(.*)$/);f[d]={value:e[1],unit:e[2]||"px"}}),b.toShow.css({height:0,overflow:"hidden"}).show(),b.toHide.filter(":hidden").each(b.complete).end().filter(":visible").animate(g,{step:function(a,c){c.prop=="height"&&(e=c.end-c.start===0?0:(c.now-c.start)/(c.end-c.start)),b.toShow[0].style[c.prop]=e*f[c.prop].value+f[c.prop].unit},duration:b.duration,easing:b.easing,complete:function(){b.autoHeight||b.toShow.css("height",""),b.toShow.css({width:i,overflow:d}),b.complete()}})}},bounceslide:function(a){this.slide(a,{easing:a.down?"easeOutBounce":"swing",duration:a.down?1e3:200})}}})})(jQuery);/*
+ * jQuery UI Autocomplete 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Autocomplete
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.widget.js
+ * jquery.ui.position.js
+ */(function(a,b){var c=0;a.widget("ui.autocomplete",{options:{appendTo:"body",autoFocus:!1,delay:300,minLength:1,position:{my:"left top",at:"left bottom",collision:"none"},source:null},pending:0,_create:function(){var b=this,c=this.element[0].ownerDocument,d;this.element.addClass("ui-autocomplete-input").attr("autocomplete","off").attr({role:"textbox","aria-autocomplete":"list","aria-haspopup":"true"}).bind("keydown.autocomplete",function(c){if(!b.options.disabled&&!b.element.propAttr("readOnly")){d=!1;var e=a.ui.keyCode;switch(c.keyCode){case e.PAGE_UP:b._move("previousPage",c);break;case e.PAGE_DOWN:b._move("nextPage",c);break;case e.UP:b._move("previous",c),c.preventDefault();break;case e.DOWN:b._move("next",c),c.preventDefault();break;case e.ENTER:case e.NUMPAD_ENTER:b.menu.active&&(d=!0,c.preventDefault());case e.TAB:if(!b.menu.active)return;b.menu.select(c);break;case e.ESCAPE:b.element.val(b.term),b.close(c);break;default:clearTimeout(b.searching),b.searching=setTimeout(function(){b.term!=b.element.val()&&(b.selectedItem=null,b.search(null,c))},b.options.delay)}}}).bind("keypress.autocomplete",function(a){d&&(d=!1,a.preventDefault())}).bind("focus.autocomplete",function(){b.options.disabled||(b.selectedItem=null,b.previous=b.element.val())}).bind("blur.autocomplete",function(a){b.options.disabled||(clearTimeout(b.searching),b.closing=setTimeout(function(){b.close(a),b._change(a)},150))}),this._initSource(),this.response=function(){return b._response.apply(b,arguments)},this.menu=a("<ul></ul>").addClass("ui-autocomplete").appendTo(a(this.options.appendTo||"body",c)[0]).mousedown(function(c){var d=b.menu.element[0];a(c.target).closest(".ui-menu-item").length||setTimeout(function(){a(document).one("mousedown",function(c){c.target!==b.element[0]&&c.target!==d&&!a.ui.contains(d,c.target)&&b.close()})},1),setTimeout(function(){clearTimeout(b.closing)},13)}).menu({focus:function(a,c){var d=c.item.data("item.autocomplete");!1!==b._trigger("focus",a,{item:d})&&/^key/.test(a.originalEvent.type)&&b.element.val(d.value)},selected:function(a,d){var e=d.item.data("item.autocomplete"),f=b.previous;b.element[0]!==c.activeElement&&(b.element.focus(),b.previous=f,setTimeout(function(){b.previous=f,b.selectedItem=e},1)),!1!==b._trigger("select",a,{item:e})&&b.element.val(e.value),b.term=b.element.val(),b.close(a),b.selectedItem=e},blur:function(a,c){b.menu.element.is(":visible")&&b.element.val()!==b.term&&b.element.val(b.term)}}).zIndex(this.element.zIndex()+1).css({top:0,left:0}).hide().data("menu"),a.fn.bgiframe&&this.menu.element.bgiframe(),b.beforeunloadHandler=function(){b.element.removeAttr("autocomplete")},a(window).bind("beforeunload",b.beforeunloadHandler)},destroy:function(){this.element.removeClass("ui-autocomplete-input").removeAttr("autocomplete").removeAttr("role").removeAttr("aria-autocomplete").removeAttr("aria-haspopup"),this.menu.element.remove(),a(window).unbind("beforeunload",this.beforeunloadHandler),a.Widget.prototype.destroy.call(this)},_setOption:function(b,c){a.Widget.prototype._setOption.apply(this,arguments),b==="source"&&this._initSource(),b==="appendTo"&&this.menu.element.appendTo(a(c||"body",this.element[0].ownerDocument)[0]),b==="disabled"&&c&&this.xhr&&this.xhr.abort()},_initSource:function(){var b=this,d,e;a.isArray(this.options.source)?(d=this.options.source,this.source=function(b,c){c(a.ui.autocomplete.filter(d,b.term))}):typeof this.options.source=="string"?(e=this.options.source,this.source=function(d,f){b.xhr&&b.xhr.abort(),b.xhr=a.ajax({url:e,data:d,dataType:"json",autocompleteRequest:++c,success:function(a,b){this.autocompleteRequest===c&&f(a)},error:function(){this.autocompleteRequest===c&&f([])}})}):this.source=this.options.source},search:function(a,b){a=a!=null?a:this.element.val(),this.term=this.element.val();if(a.length<this.options.minLength)return this.close(b);clearTimeout(this.closing);if(this._trigger("search",b)!==!1)return this._search(a)},_search:function(a){this.pending++,this.element.addClass("ui-autocomplete-loading"),this.source({term:a},this.response)},_response:function(a){!this.options.disabled&&a&&a.length?(a=this._normalize(a),this._suggest(a),this._trigger("open")):this.close(),this.pending--,this.pending||this.element.removeClass("ui-autocomplete-loading")},close:function(a){clearTimeout(this.closing),this.menu.element.is(":visible")&&(this.menu.element.hide(),this.menu.deactivate(),this._trigger("close",a))},_change:function(a){this.previous!==this.element.val()&&this._trigger("change",a,{item:this.selectedItem})},_normalize:function(b){if(b.length&&b[0].label&&b[0].value)return b;return a.map(b,function(b){if(typeof b=="string")return{label:b,value:b};return a.extend({label:b.label||b.value,value:b.value||b.label},b)})},_suggest:function(b){var c=this.menu.element.empty().zIndex(this.element.zIndex()+1);this._renderMenu(c,b),this.menu.deactivate(),this.menu.refresh(),c.show(),this._resizeMenu(),c.position(a.extend({of:this.element},this.options.position)),this.options.autoFocus&&this.menu.next(new a.Event("mouseover"))},_resizeMenu:function(){var a=this.menu.element;a.outerWidth(Math.max(a.width("").outerWidth()+1,this.element.outerWidth()))},_renderMenu:function(b,c){var d=this;a.each(c,function(a,c){d._renderItem(b,c)})},_renderItem:function(b,c){return a("<li></li>").data("item.autocomplete",c).append(a("<a></a>").text(c.label)).appendTo(b)},_move:function(a,b){if(!this.menu.element.is(":visible"))this.search(null,b);else{if(this.menu.first()&&/^previous/.test(a)||this.menu.last()&&/^next/.test(a)){this.element.val(this.term),this.menu.deactivate();return}this.menu[a](b)}},widget:function(){return this.menu.element}}),a.extend(a.ui.autocomplete,{escapeRegex:function(a){return a.replace(/[-[\]{}()*+?.,\\^$|#\s]/g,"\\$&")},filter:function(b,c){var d=new RegExp(a.ui.autocomplete.escapeRegex(c),"i");return a.grep(b,function(a){return d.test(a.label||a.value||a)})}})})(jQuery),function(a){a.widget("ui.menu",{_create:function(){var b=this;this.element.addClass("ui-menu ui-widget ui-widget-content ui-corner-all").attr({role:"listbox","aria-activedescendant":"ui-active-menuitem"}).click(function(c){!a(c.target).closest(".ui-menu-item a").length||(c.preventDefault(),b.select(c))}),this.refresh()},refresh:function(){var b=this,c=this.element.children("li:not(.ui-menu-item):has(a)").addClass("ui-menu-item").attr("role","menuitem");c.children("a").addClass("ui-corner-all").attr("tabindex",-1).mouseenter(function(c){b.activate(c,a(this).parent())}).mouseleave(function(){b.deactivate()})},activate:function(a,b){this.deactivate();if(this.hasScroll()){var c=b.offset().top-this.element.offset().top,d=this.element.scrollTop(),e=this.element.height();c<0?this.element.scrollTop(d+c):c>=e&&this.element.scrollTop(d+c-e+b.height())}this.active=b.eq(0).children("a").addClass("ui-state-hover").attr("id","ui-active-menuitem").end(),this._trigger("focus",a,{item:b})},deactivate:function(){!this.active||(this.active.children("a").removeClass("ui-state-hover").removeAttr("id"),this._trigger("blur"),this.active=null)},next:function(a){this.move("next",".ui-menu-item:first",a)},previous:function(a){this.move("prev",".ui-menu-item:last",a)},first:function(){return this.active&&!this.active.prevAll(".ui-menu-item").length},last:function(){return this.active&&!this.active.nextAll(".ui-menu-item").length},move:function(a,b,c){if(!this.active)this.activate(c,this.element.children(b));else{var d=this.active[a+"All"](".ui-menu-item").eq(0);d.length?this.activate(c,d):this.activate(c,this.element.children(b))}},nextPage:function(b){if(this.hasScroll()){if(!this.active||this.last()){this.activate(b,this.element.children(".ui-menu-item:first"));return}var c=this.active.offset().top,d=this.element.height(),e=this.element.children(".ui-menu-item").filter(function(){var b=a(this).offset().top-c-d+a(this).height();return b<10&&b>-10});e.length||(e=this.element.children(".ui-menu-item:last")),this.activate(b,e)}else this.activate(b,this.element.children(".ui-menu-item").filter(!this.active||this.last()?":first":":last"))},previousPage:function(b){if(this.hasScroll()){if(!this.active||this.first()){this.activate(b,this.element.children(".ui-menu-item:last"));return}var c=this.active.offset().top,d=this.element.height();result=this.element.children(".ui-menu-item").filter(function(){var b=a(this).offset().top-c+d-a(this).height();return b<10&&b>-10}),result.length||(result=this.element.children(".ui-menu-item:first")),this.activate(b,result)}else this.activate(b,this.element.children(".ui-menu-item").filter(!this.active||this.first()?":last":":first"))},hasScroll:function(){return this.element.height()<this.element[a.fn.prop?"prop":"attr"]("scrollHeight")},select:function(a){this._trigger("selected",a,{item:this.active})}})}(jQuery);/*
+ * jQuery UI Button 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Button
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.widget.js
+ */(function(a,b){var c,d,e,f,g="ui-button ui-widget ui-state-default ui-corner-all",h="ui-state-hover ui-state-active ",i="ui-button-icons-only ui-button-icon-only ui-button-text-icons ui-button-text-icon-primary ui-button-text-icon-secondary ui-button-text-only",j=function(){var b=a(this).find(":ui-button");setTimeout(function(){b.button("refresh")},1)},k=function(b){var c=b.name,d=b.form,e=a([]);c&&(d?e=a(d).find("[name='"+c+"']"):e=a("[name='"+c+"']",b.ownerDocument).filter(function(){return!this.form}));return e};a.widget("ui.button",{options:{disabled:null,text:!0,label:null,icons:{primary:null,secondary:null}},_create:function(){this.element.closest("form").unbind("reset.button").bind("reset.button",j),typeof this.options.disabled!="boolean"&&(this.options.disabled=this.element.propAttr("disabled")),this._determineButtonType(),this.hasTitle=!!this.buttonElement.attr("title");var b=this,h=this.options,i=this.type==="checkbox"||this.type==="radio",l="ui-state-hover"+(i?"":" ui-state-active"),m="ui-state-focus";h.label===null&&(h.label=this.buttonElement.html()),this.element.is(":disabled")&&(h.disabled=!0),this.buttonElement.addClass(g).attr("role","button").bind("mouseenter.button",function(){h.disabled||(a(this).addClass("ui-state-hover"),this===c&&a(this).addClass("ui-state-active"))}).bind("mouseleave.button",function(){h.disabled||a(this).removeClass(l)}).bind("click.button",function(a){h.disabled&&(a.preventDefault(),a.stopImmediatePropagation())}),this.element.bind("focus.button",function(){b.buttonElement.addClass(m)}).bind("blur.button",function(){b.buttonElement.removeClass(m)}),i&&(this.element.bind("change.button",function(){f||b.refresh()}),this.buttonElement.bind("mousedown.button",function(a){h.disabled||(f=!1,d=a.pageX,e=a.pageY)}).bind("mouseup.button",function(a){!h.disabled&&(d!==a.pageX||e!==a.pageY)&&(f=!0)})),this.type==="checkbox"?this.buttonElement.bind("click.button",function(){if(h.disabled||f)return!1;a(this).toggleClass("ui-state-active"),b.buttonElement.attr("aria-pressed",b.element[0].checked)}):this.type==="radio"?this.buttonElement.bind("click.button",function(){if(h.disabled||f)return!1;a(this).addClass("ui-state-active"),b.buttonElement.attr("aria-pressed","true");var c=b.element[0];k(c).not(c).map(function(){return a(this).button("widget")[0]}).removeClass("ui-state-active").attr("aria-pressed","false")}):(this.buttonElement.bind("mousedown.button",function(){if(h.disabled)return!1;a(this).addClass("ui-state-active"),c=this,a(document).one("mouseup",function(){c=null})}).bind("mouseup.button",function(){if(h.disabled)return!1;a(this).removeClass("ui-state-active")}).bind("keydown.button",function(b){if(h.disabled)return!1;(b.keyCode==a.ui.keyCode.SPACE||b.keyCode==a.ui.keyCode.ENTER)&&a(this).addClass("ui-state-active")}).bind("keyup.button",function(){a(this).removeClass("ui-state-active")}),this.buttonElement.is("a")&&this.buttonElement.keyup(function(b){b.keyCode===a.ui.keyCode.SPACE&&a(this).click()})),this._setOption("disabled",h.disabled),this._resetButton()},_determineButtonType:function(){this.element.is(":checkbox")?this.type="checkbox":this.element.is(":radio")?this.type="radio":this.element.is("input")?this.type="input":this.type="button";if(this.type==="checkbox"||this.type==="radio"){var a=this.element.parents().filter(":last"),b="label[for='"+this.element.attr("id")+"']";this.buttonElement=a.find(b),this.buttonElement.length||(a=a.length?a.siblings():this.element.siblings(),this.buttonElement=a.filter(b),this.buttonElement.length||(this.buttonElement=a.find(b))),this.element.addClass("ui-helper-hidden-accessible");var c=this.element.is(":checked");c&&this.buttonElement.addClass("ui-state-active"),this.buttonElement.attr("aria-pressed",c)}else this.buttonElement=this.element},widget:function(){return this.buttonElement},destroy:function(){this.element.removeClass("ui-helper-hidden-accessible"),this.buttonElement.removeClass(g+" "+h+" "+i).removeAttr("role").removeAttr("aria-pressed").html(this.buttonElement.find(".ui-button-text").html()),this.hasTitle||this.buttonElement.removeAttr("title"),a.Widget.prototype.destroy.call(this)},_setOption:function(b,c){a.Widget.prototype._setOption.apply(this,arguments);b==="disabled"?c?this.element.propAttr("disabled",!0):this.element.propAttr("disabled",!1):this._resetButton()},refresh:function(){var b=this.element.is(":disabled");b!==this.options.disabled&&this._setOption("disabled",b),this.type==="radio"?k(this.element[0]).each(function(){a(this).is(":checked")?a(this).button("widget").addClass("ui-state-active").attr("aria-pressed","true"):a(this).button("widget").removeClass("ui-state-active").attr("aria-pressed","false")}):this.type==="checkbox"&&(this.element.is(":checked")?this.buttonElement.addClass("ui-state-active").attr("aria-pressed","true"):this.buttonElement.removeClass("ui-state-active").attr("aria-pressed","false"))},_resetButton:function(){if(this.type==="input")this.options.label&&this.element.val(this.options.label);else{var b=this.buttonElement.removeClass(i),c=a("<span></span>",this.element[0].ownerDocument).addClass("ui-button-text").html(this.options.label).appendTo(b.empty()).text(),d=this.options.icons,e=d.primary&&d.secondary,f=[];d.primary||d.secondary?(this.options.text&&f.push("ui-button-text-icon"+(e?"s":d.primary?"-primary":"-secondary")),d.primary&&b.prepend("<span class='ui-button-icon-primary ui-icon "+d.primary+"'></span>"),d.secondary&&b.append("<span class='ui-button-icon-secondary ui-icon "+d.secondary+"'></span>"),this.options.text||(f.push(e?"ui-button-icons-only":"ui-button-icon-only"),this.hasTitle||b.attr("title",c))):f.push("ui-button-text-only"),b.addClass(f.join(" "))}}}),a.widget("ui.buttonset",{options:{items:":button, :submit, :reset, :checkbox, :radio, a, :data(button)"},_create:function(){this.element.addClass("ui-buttonset")},_init:function(){this.refresh()},_setOption:function(b,c){b==="disabled"&&this.buttons.button("option",b,c),a.Widget.prototype._setOption.apply(this,arguments)},refresh:function(){var b=this.element.css("direction")==="rtl";this.buttons=this.element.find(this.options.items).filter(":ui-button").button("refresh").end().not(":ui-button").button().end().map(function(){return a(this).button("widget")[0]}).removeClass("ui-corner-all ui-corner-left ui-corner-right").filter(":first").addClass(b?"ui-corner-right":"ui-corner-left").end().filter(":last").addClass(b?"ui-corner-left":"ui-corner-right").end().end()},destroy:function(){this.element.removeClass("ui-buttonset"),this.buttons.map(function(){return a(this).button("widget")[0]}).removeClass("ui-corner-left ui-corner-right").end().button("destroy"),a.Widget.prototype.destroy.call(this)}})})(jQuery);/*
+ * jQuery UI Dialog 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Dialog
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.widget.js
+ * jquery.ui.button.js
+ * jquery.ui.draggable.js
+ * jquery.ui.mouse.js
+ * jquery.ui.position.js
+ * jquery.ui.resizable.js
+ */(function(a,b){var c="ui-dialog ui-widget ui-widget-content ui-corner-all ",d={buttons:!0,height:!0,maxHeight:!0,maxWidth:!0,minHeight:!0,minWidth:!0,width:!0},e={maxHeight:!0,maxWidth:!0,minHeight:!0,minWidth:!0},f=a.attrFn||{val:!0,css:!0,html:!0,text:!0,data:!0,width:!0,height:!0,offset:!0,click:!0};a.widget("ui.dialog",{options:{autoOpen:!0,buttons:{},closeOnEscape:!0,closeText:"close",dialogClass:"",draggable:!0,hide:null,height:"auto",maxHeight:!1,maxWidth:!1,minHeight:150,minWidth:150,modal:!1,position:{my:"center",at:"center",collision:"fit",using:function(b){var c=a(this).css(b).offset().top;c<0&&a(this).css("top",b.top-c)}},resizable:!0,show:null,stack:!0,title:"",width:300,zIndex:1e3},_create:function(){this.originalTitle=this.element.attr("title"),typeof this.originalTitle!="string"&&(this.originalTitle=""),this.options.title=this.options.title||this.originalTitle;var b=this,d=b.options,e=d.title||" ",f=a.ui.dialog.getTitleId(b.element),g=(b.uiDialog=a("<div></div>")).appendTo(document.body).hide().addClass(c+d.dialogClass).css({zIndex:d.zIndex}).attr("tabIndex",-1).css("outline",0).keydown(function(c){d.closeOnEscape&&!c.isDefaultPrevented()&&c.keyCode&&c.keyCode===a.ui.keyCode.ESCAPE&&(b.close(c),c.preventDefault())}).attr({role:"dialog","aria-labelledby":f}).mousedown(function(a){b.moveToTop(!1,a)}),h=b.element.show().removeAttr("title").addClass("ui-dialog-content ui-widget-content").appendTo(g),i=(b.uiDialogTitlebar=a("<div></div>")).addClass("ui-dialog-titlebar ui-widget-header ui-corner-all ui-helper-clearfix").prependTo(g),j=a('<a href="#"></a>').addClass("ui-dialog-titlebar-close ui-corner-all").attr("role","button").hover(function(){j.addClass("ui-state-hover")},function(){j.removeClass("ui-state-hover")}).focus(function(){j.addClass("ui-state-focus")}).blur(function(){j.removeClass("ui-state-focus")}).click(function(a){b.close(a);return!1}).appendTo(i),k=(b.uiDialogTitlebarCloseText=a("<span></span>")).addClass("ui-icon ui-icon-closethick").text(d.closeText).appendTo(j),l=a("<span></span>").addClass("ui-dialog-title").attr("id",f).html(e).prependTo(i);a.isFunction(d.beforeclose)&&!a.isFunction(d.beforeClose)&&(d.beforeClose=d.beforeclose),i.find("*").add(i).disableSelection(),d.draggable&&a.fn.draggable&&b._makeDraggable(),d.resizable&&a.fn.resizable&&b._makeResizable(),b._createButtons(d.buttons),b._isOpen=!1,a.fn.bgiframe&&g.bgiframe()},_init:function(){this.options.autoOpen&&this.open()},destroy:function(){var a=this;a.overlay&&a.overlay.destroy(),a.uiDialog.hide(),a.element.unbind(".dialog").removeData("dialog").removeClass("ui-dialog-content ui-widget-content").hide().appendTo("body"),a.uiDialog.remove(),a.originalTitle&&a.element.attr("title",a.originalTitle);return a},widget:function(){return this.uiDialog},close:function(b){var c=this,d,e;if(!1!==c._trigger("beforeClose",b)){c.overlay&&c.overlay.destroy(),c.uiDialog.unbind("keypress.ui-dialog"),c._isOpen=!1,c.options.hide?c.uiDialog.hide(c.options.hide,function(){c._trigger("close",b)}):(c.uiDialog.hide(),c._trigger("close",b)),a.ui.dialog.overlay.resize(),c.options.modal&&(d=0,a(".ui-dialog").each(function(){this!==c.uiDialog[0]&&(e=a(this).css("z-index"),isNaN(e)||(d=Math.max(d,e)))}),a.ui.dialog.maxZ=d);return c}},isOpen:function(){return this._isOpen},moveToTop:function(b,c){var d=this,e=d.options,f;if(e.modal&&!b||!e.stack&&!e.modal)return d._trigger("focus",c);e.zIndex>a.ui.dialog.maxZ&&(a.ui.dialog.maxZ=e.zIndex),d.overlay&&(a.ui.dialog.maxZ+=1,d.overlay.$el.css("z-index",a.ui.dialog.overlay.maxZ=a.ui.dialog.maxZ)),f={scrollTop:d.element.scrollTop(),scrollLeft:d.element.scrollLeft()},a.ui.dialog.maxZ+=1,d.uiDialog.css("z-index",a.ui.dialog.maxZ),d.element.attr(f),d._trigger("focus",c);return d},open:function(){if(!this._isOpen){var b=this,c=b.options,d=b.uiDialog;b.overlay=c.modal?new a.ui.dialog.overlay(b):null,b._size(),b._position(c.position),d.show(c.show),b.moveToTop(!0),c.modal&&d.bind("keydown.ui-dialog",function(b){if(b.keyCode===a.ui.keyCode.TAB){var c=a(":tabbable",this),d=c.filter(":first"),e=c.filter(":last");if(b.target===e[0]&&!b.shiftKey){d.focus(1);return!1}if(b.target===d[0]&&b.shiftKey){e.focus(1);return!1}}}),a(b.element.find(":tabbable").get().concat(d.find(".ui-dialog-buttonpane :tabbable").get().concat(d.get()))).eq(0).focus(),b._isOpen=!0,b._trigger("open");return b}},_createButtons:function(b){var c=this,d=!1,e=a("<div></div>").addClass("ui-dialog-buttonpane ui-widget-content ui-helper-clearfix"),g=a("<div></div>").addClass("ui-dialog-buttonset").appendTo(e);c.uiDialog.find(".ui-dialog-buttonpane").remove(),typeof b=="object"&&b!==null&&a.each(b,function(){return!(d=!0)}),d&&(a.each(b,function(b,d){d=a.isFunction(d)?{click:d,text:b}:d;var e=a('<button type="button"></button>').click(function(){d.click.apply(c.element[0],arguments)}).appendTo(g);a.each(d,function(a,b){a!=="click"&&(a in f?e[a](b):e.attr(a,b))}),a.fn.button&&e.button()}),e.appendTo(c.uiDialog))},_makeDraggable:function(){function f(a){return{position:a.position,offset:a.offset}}var b=this,c=b.options,d=a(document),e;b.uiDialog.draggable({cancel:".ui-dialog-content, .ui-dialog-titlebar-close",handle:".ui-dialog-titlebar",containment:"document",start:function(d,g){e=c.height==="auto"?"auto":a(this).height(),a(this).height(a(this).height()).addClass("ui-dialog-dragging"),b._trigger("dragStart",d,f(g))},drag:function(a,c){b._trigger("drag",a,f(c))},stop:function(g,h){c.position=[h.position.left-d.scrollLeft(),h.position.top-d.scrollTop()],a(this).removeClass("ui-dialog-dragging").height(e),b._trigger("dragStop",g,f(h)),a.ui.dialog.overlay.resize()}})},_makeResizable:function(c){function h(a){return{originalPosition:a.originalPosition,originalSize:a.originalSize,position:a.position,size:a.size}}c=c===b?this.options.resizable:c;var d=this,e=d.options,f=d.uiDialog.css("position"),g=typeof c=="string"?c:"n,e,s,w,se,sw,ne,nw";d.uiDialog.resizable({cancel:".ui-dialog-content",containment:"document",alsoResize:d.element,maxWidth:e.maxWidth,maxHeight:e.maxHeight,minWidth:e.minWidth,minHeight:d._minHeight(),handles:g,start:function(b,c){a(this).addClass("ui-dialog-resizing"),d._trigger("resizeStart",b,h(c))},resize:function(a,b){d._trigger("resize",a,h(b))},stop:function(b,c){a(this).removeClass("ui-dialog-resizing"),e.height=a(this).height(),e.width=a(this).width(),d._trigger("resizeStop",b,h(c)),a.ui.dialog.overlay.resize()}}).css("position",f).find(".ui-resizable-se").addClass("ui-icon ui-icon-grip-diagonal-se")},_minHeight:function(){var a=this.options;return a.height==="auto"?a.minHeight:Math.min(a.minHeight,a.height)},_position:function(b){var c=[],d=[0,0],e;if(b){if(typeof b=="string"||typeof b=="object"&&"0"in b)c=b.split?b.split(" "):[b[0],b[1]],c.length===1&&(c[1]=c[0]),a.each(["left","top"],function(a,b){+c[a]===c[a]&&(d[a]=c[a],c[a]=b)}),b={my:c.join(" "),at:c.join(" "),offset:d.join(" ")};b=a.extend({},a.ui.dialog.prototype.options.position,b)}else b=a.ui.dialog.prototype.options.position;e=this.uiDialog.is(":visible"),e||this.uiDialog.show(),this.uiDialog.css({top:0,left:0}).position(a.extend({of:window},b)),e||this.uiDialog.hide()},_setOptions:function(b){var c=this,f={},g=!1;a.each(b,function(a,b){c._setOption(a,b),a in d&&(g=!0),a in e&&(f[a]=b)}),g&&this._size(),this.uiDialog.is(":data(resizable)")&&this.uiDialog.resizable("option",f)},_setOption:function(b,d){var e=this,f=e.uiDialog;switch(b){case"beforeclose":b="beforeClose";break;case"buttons":e._createButtons(d);break;case"closeText":e.uiDialogTitlebarCloseText.text(""+d);break;case"dialogClass":f.removeClass(e.options.dialogClass).addClass(c+d);break;case"disabled":d?f.addClass("ui-dialog-disabled"):f.removeClass("ui-dialog-disabled");break;case"draggable":var g=f.is(":data(draggable)");g&&!d&&f.draggable("destroy"),!g&&d&&e._makeDraggable();break;case"position":e._position(d);break;case"resizable":var h=f.is(":data(resizable)");h&&!d&&f.resizable("destroy"),h&&typeof d=="string"&&f.resizable("option","handles",d),!h&&d!==!1&&e._makeResizable(d);break;case"title":a(".ui-dialog-title",e.uiDialogTitlebar).html(""+(d||" "))}a.Widget.prototype._setOption.apply(e,arguments)},_size:function(){var b=this.options,c,d,e=this.uiDialog.is(":visible");this.element.show().css({width:"auto",minHeight:0,height:0}),b.minWidth>b.width&&(b.width=b.minWidth),c=this.uiDialog.css({height:"auto",width:b.width}).height(),d=Math.max(0,b.minHeight-c);if(b.height==="auto")if(a.support.minHeight)this.element.css({minHeight:d,height:"auto"});else{this.uiDialog.show();var f=this.element.css("height","auto").height();e||this.uiDialog.hide(),this.element.height(Math.max(f,d))}else this.element.height(Math.max(b.height-c,0));this.uiDialog.is(":data(resizable)")&&this.uiDialog.resizable("option","minHeight",this._minHeight())}}),a.extend(a.ui.dialog,{version:"1.8.17",uuid:0,maxZ:0,getTitleId:function(a){var b=a.attr("id");b||(this.uuid+=1,b=this.uuid);return"ui-dialog-title-"+b},overlay:function(b){this.$el=a.ui.dialog.overlay.create(b)}}),a.extend(a.ui.dialog.overlay,{instances:[],oldInstances:[],maxZ:0,events:a.map("focus,mousedown,mouseup,keydown,keypress,click".split(","),function(a){return a+".dialog-overlay"}).join(" "),create:function(b){this.instances.length===0&&(setTimeout(function(){a.ui.dialog.overlay.instances.length&&a(document).bind(a.ui.dialog.overlay.events,function(b){if(a(b.target).zIndex()<a.ui.dialog.overlay.maxZ)return!1})},1),a(document).bind("keydown.dialog-overlay",function(c){b.options.closeOnEscape&&!c.isDefaultPrevented()&&c.keyCode&&c.keyCode===a.ui.keyCode.ESCAPE&&(b.close(c),c.preventDefault())}),a(window).bind("resize.dialog-overlay",a.ui.dialog.overlay.resize));var c=(this.oldInstances.pop()||a("<div></div>").addClass("ui-widget-overlay")).appendTo(document.body).css({width:this.width(),height:this.height()});a.fn.bgiframe&&c.bgiframe(),this.instances.push(c);return c},destroy:function(b){var c=a.inArray(b,this.instances);c!=-1&&this.oldInstances.push(this.instances.splice(c,1)[0]),this.instances.length===0&&a([document,window]).unbind(".dialog-overlay"),b.remove();var d=0;a.each(this.instances,function(){d=Math.max(d,this.css("z-index"))}),this.maxZ=d},height:function(){var b,c;if(a.browser.msie&&a.browser.version<7){b=Math.max(document.documentElement.scrollHeight,document.body.scrollHeight),c=Math.max(document.documentElement.offsetHeight,document.body.offsetHeight);return b<c?a(window).height()+"px":b+"px"}return a(document).height()+"px"},width:function(){var b,c;if(a.browser.msie){b=Math.max(document.documentElement.scrollWidth,document.body.scrollWidth),c=Math.max(document.documentElement.offsetWidth,document.body.offsetWidth);return b<c?a(window).width()+"px":b+"px"}return a(document).width()+"px"},resize:function(){var b=a([]);a.each(a.ui.dialog.overlay.instances,function(){b=b.add(this)}),b.css({width:0,height:0}).css({width:a.ui.dialog.overlay.width(),height:a.ui.dialog.overlay.height()})}}),a.extend(a.ui.dialog.overlay.prototype,{destroy:function(){a.ui.dialog.overlay.destroy(this.$el)}})})(jQuery);/*
+ * jQuery UI Slider 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Slider
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.mouse.js
+ * jquery.ui.widget.js
+ */(function(a,b){var c=5;a.widget("ui.slider",a.ui.mouse,{widgetEventPrefix:"slide",options:{animate:!1,distance:0,max:100,min:0,orientation:"horizontal",range:!1,step:1,value:0,values:null},_create:function(){var b=this,d=this.options,e=this.element.find(".ui-slider-handle").addClass("ui-state-default ui-corner-all"),f="<a class='ui-slider-handle ui-state-default ui-corner-all' href='#'></a>",g=d.values&&d.values.length||1,h=[];this._keySliding=!1,this._mouseSliding=!1,this._animateOff=!0,this._handleIndex=null,this._detectOrientation(),this._mouseInit(),this.element.addClass("ui-slider ui-slider-"+this.orientation+" ui-widget"+" ui-widget-content"+" ui-corner-all"+(d.disabled?" ui-slider-disabled ui-disabled":"")),this.range=a([]),d.range&&(d.range===!0&&(d.values||(d.values=[this._valueMin(),this._valueMin()]),d.values.length&&d.values.length!==2&&(d.values=[d.values[0],d.values[0]])),this.range=a("<div></div>").appendTo(this.element).addClass("ui-slider-range ui-widget-header"+(d.range==="min"||d.range==="max"?" ui-slider-range-"+d.range:"")));for(var i=e.length;i<g;i+=1)h.push(f);this.handles=e.add(a(h.join("")).appendTo(b.element)),this.handle=this.handles.eq(0),this.handles.add(this.range).filter("a").click(function(a){a.preventDefault()}).hover(function(){d.disabled||a(this).addClass("ui-state-hover")},function(){a(this).removeClass("ui-state-hover")}).focus(function(){d.disabled?a(this).blur():(a(".ui-slider .ui-state-focus").removeClass("ui-state-focus"),a(this).addClass("ui-state-focus"))}).blur(function(){a(this).removeClass("ui-state-focus")}),this.handles.each(function(b){a(this).data("index.ui-slider-handle",b)}),this.handles.keydown(function(d){var e=!0,f=a(this).data("index.ui-slider-handle"),g,h,i,j;if(!b.options.disabled){switch(d.keyCode){case a.ui.keyCode.HOME:case a.ui.keyCode.END:case a.ui.keyCode.PAGE_UP:case a.ui.keyCode.PAGE_DOWN:case a.ui.keyCode.UP:case a.ui.keyCode.RIGHT:case a.ui.keyCode.DOWN:case a.ui.keyCode.LEFT:e=!1;if(!b._keySliding){b._keySliding=!0,a(this).addClass("ui-state-active"),g=b._start(d,f);if(g===!1)return}}j=b.options.step,b.options.values&&b.options.values.length?h=i=b.values(f):h=i=b.value();switch(d.keyCode){case a.ui.keyCode.HOME:i=b._valueMin();break;case a.ui.keyCode.END:i=b._valueMax();break;case a.ui.keyCode.PAGE_UP:i=b._trimAlignValue(h+(b._valueMax()-b._valueMin())/c);break;case a.ui.keyCode.PAGE_DOWN:i=b._trimAlignValue(h-(b._valueMax()-b._valueMin())/c);break;case a.ui.keyCode.UP:case a.ui.keyCode.RIGHT:if(h===b._valueMax())return;i=b._trimAlignValue(h+j);break;case a.ui.keyCode.DOWN:case a.ui.keyCode.LEFT:if(h===b._valueMin())return;i=b._trimAlignValue(h-j)}b._slide(d,f,i);return e}}).keyup(function(c){var d=a(this).data("index.ui-slider-handle");b._keySliding&&(b._keySliding=!1,b._stop(c,d),b._change(c,d),a(this).removeClass("ui-state-active"))}),this._refreshValue(),this._animateOff=!1},destroy:function(){this.handles.remove(),this.range.remove(),this.element.removeClass("ui-slider ui-slider-horizontal ui-slider-vertical ui-slider-disabled ui-widget ui-widget-content ui-corner-all").removeData("slider").unbind(".slider"),this._mouseDestroy();return this},_mouseCapture:function(b){var c=this.options,d,e,f,g,h,i,j,k,l;if(c.disabled)return!1;this.elementSize={width:this.element.outerWidth(),height:this.element.outerHeight()},this.elementOffset=this.element.offset(),d={x:b.pageX,y:b.pageY},e=this._normValueFromMouse(d),f=this._valueMax()-this._valueMin()+1,h=this,this.handles.each(function(b){var c=Math.abs(e-h.values(b));f>c&&(f=c,g=a(this),i=b)}),c.range===!0&&this.values(1)===c.min&&(i+=1,g=a(this.handles[i])),j=this._start(b,i);if(j===!1)return!1;this._mouseSliding=!0,h._handleIndex=i,g.addClass("ui-state-active").focus(),k=g.offset(),l=!a(b.target).parents().andSelf().is(".ui-slider-handle"),this._clickOffset=l?{left:0,top:0}:{left:b.pageX-k.left-g.width()/2,top:b.pageY-k.top-g.height()/2-(parseInt(g.css("borderTopWidth"),10)||0)-(parseInt(g.css("borderBottomWidth"),10)||0)+(parseInt(g.css("marginTop"),10)||0)},this.handles.hasClass("ui-state-hover")||this._slide(b,i,e),this._animateOff=!0;return!0},_mouseStart:function(a){return!0},_mouseDrag:function(a){var b={x:a.pageX,y:a.pageY},c=this._normValueFromMouse(b);this._slide(a,this._handleIndex,c);return!1},_mouseStop:function(a){this.handles.removeClass("ui-state-active"),this._mouseSliding=!1,this._stop(a,this._handleIndex),this._change(a,this._handleIndex),this._handleIndex=null,this._clickOffset=null,this._animateOff=!1;return!1},_detectOrientation:function(){this.orientation=this.options.orientation==="vertical"?"vertical":"horizontal"},_normValueFromMouse:function(a){var b,c,d,e,f;this.orientation==="horizontal"?(b=this.elementSize.width,c=a.x-this.elementOffset.left-(this._clickOffset?this._clickOffset.left:0)):(b=this.elementSize.height,c=a.y-this.elementOffset.top-(this._clickOffset?this._clickOffset.top:0)),d=c/b,d>1&&(d=1),d<0&&(d=0),this.orientation==="vertical"&&(d=1-d),e=this._valueMax()-this._valueMin(),f=this._valueMin()+d*e;return this._trimAlignValue(f)},_start:function(a,b){var c={handle:this.handles[b],value:this.value()};this.options.values&&this.options.values.length&&(c.value=this.values(b),c.values=this.values());return this._trigger("start",a,c)},_slide:function(a,b,c){var d,e,f;this.options.values&&this.options.values.length?(d=this.values(b?0:1),this.options.values.length===2&&this.options.range===!0&&(b===0&&c>d||b===1&&c<d)&&(c=d),c!==this.values(b)&&(e=this.values(),e[b]=c,f=this._trigger("slide",a,{handle:this.handles[b],value:c,values:e}),d=this.values(b?0:1),f!==!1&&this.values(b,c,!0))):c!==this.value()&&(f=this._trigger("slide",a,{handle:this.handles[b],value:c}),f!==!1&&this.value(c))},_stop:function(a,b){var c={handle:this.handles[b],value:this.value()};this.options.values&&this.options.values.length&&(c.value=this.values(b),c.values=this.values()),this._trigger("stop",a,c)},_change:function(a,b){if(!this._keySliding&&!this._mouseSliding){var c={handle:this.handles[b],value:this.value()};this.options.values&&this.options.values.length&&(c.value=this.values(b),c.values=this.values()),this._trigger("change",a,c)}},value:function(a){if(arguments.length)this.options.value=this._trimAlignValue(a),this._refreshValue(),this._change(null,0);else return this._value()},values:function(b,c){var d,e,f;if(arguments.length>1)this.options.values[b]=this._trimAlignValue(c),this._refreshValue(),this._change(null,b);else{if(!arguments.length)return this._values();if(!a.isArray(arguments[0]))return this.options.values&&this.options.values.length?this._values(b):this.value();d=this.options.values,e=arguments[0];for(f=0;f<d.length;f+=1)d[f]=this._trimAlignValue(e[f]),this._change(null,f);this._refreshValue()}},_setOption:function(b,c){var d,e=0;a.isArray(this.options.values)&&(e=this.options.values.length),a.Widget.prototype._setOption.apply(this,arguments);switch(b){case"disabled":c?(this.handles.filter(".ui-state-focus").blur(),this.handles.removeClass("ui-state-hover"),this.handles.propAttr("disabled",!0),this.element.addClass("ui-disabled")):(this.handles.propAttr("disabled",!1),this.element.removeClass("ui-disabled"));break;case"orientation":this._detectOrientation(),this.element.removeClass("ui-slider-horizontal ui-slider-vertical").addClass("ui-slider-"+this.orientation),this._refreshValue();break;case"value":this._animateOff=!0,this._refreshValue(),this._change(null,0),this._animateOff=!1;break;case"values":this._animateOff=!0,this._refreshValue();for(d=0;d<e;d+=1)this._change(null,d);this._animateOff=!1}},_value:function(){var a=this.options.value;a=this._trimAlignValue(a);return a},_values:function(a){var b,c,d;if(arguments.length){b=this.options.values[a],b=this._trimAlignValue(b);return b}c=this.options.values.slice();for(d=0;d<c.length;d+=1)c[d]=this._trimAlignValue(c[d]);return c},_trimAlignValue:function(a){if(a<=this._valueMin())return this._valueMin();if(a>=this._valueMax())return this._valueMax();var b=this.options.step>0?this.options.step:1,c=(a-this._valueMin())%b,d=a-c;Math.abs(c)*2>=b&&(d+=c>0?b:-b);return parseFloat(d.toFixed(5))},_valueMin:function(){return this.options.min},_valueMax:function(){return this.options.max},_refreshValue:function(){var b=this.options.range,c=this.options,d=this,e=this._animateOff?!1:c.animate,f,g={},h,i,j,k;this.options.values&&this.options.values.length?this.handles.each(function(b,i){f=(d.values(b)-d._valueMin())/(d._valueMax()-d._valueMin())*100,g[d.orientation==="horizontal"?"left":"bottom"]=f+"%",a(this).stop(1,1)[e?"animate":"css"](g,c.animate),d.options.range===!0&&(d.orientation==="horizontal"?(b===0&&d.range.stop(1,1)[e?"animate":"css"]({left:f+"%"},c.animate),b===1&&d.range[e?"animate":"css"]({width:f-h+"%"},{queue:!1,duration:c.animate})):(b===0&&d.range.stop(1,1)[e?"animate":"css"]({bottom:f+"%"},c.animate),b===1&&d.range[e?"animate":"css"]({height:f-h+"%"},{queue:!1,duration:c.animate}))),h=f}):(i=this.value(),j=this._valueMin(),k=this._valueMax(),f=k!==j?(i-j)/(k-j)*100:0,g[d.orientation==="horizontal"?"left":"bottom"]=f+"%",this.handle.stop(1,1)[e?"animate":"css"](g,c.animate),b==="min"&&this.orientation==="horizontal"&&this.range.stop(1,1)[e?"animate":"css"]({width:f+"%"},c.animate),b==="max"&&this.orientation==="horizontal"&&this.range[e?"animate":"css"]({width:100-f+"%"},{queue:!1,duration:c.animate}),b==="min"&&this.orientation==="vertical"&&this.range.stop(1,1)[e?"animate":"css"]({height:f+"%"},c.animate),b==="max"&&this.orientation==="vertical"&&this.range[e?"animate":"css"]({height:100-f+"%"},{queue:!1,duration:c.animate}))}}),a.extend(a.ui.slider,{version:"1.8.17"})})(jQuery);/*
+ * jQuery UI Tabs 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Tabs
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.widget.js
+ */(function(a,b){function f(){return++d}function e(){return++c}var c=0,d=0;a.widget("ui.tabs",{options:{add:null,ajaxOptions:null,cache:!1,cookie:null,collapsible:!1,disable:null,disabled:[],enable:null,event:"click",fx:null,idPrefix:"ui-tabs-",load:null,panelTemplate:"<div></div>",remove:null,select:null,show:null,spinner:"<em>Loading…</em>",tabTemplate:"<li><a href='#{href}'><span>#{label}</span></a></li>"},_create:function(){this._tabify(!0)},_setOption:function(a,b){if(a=="selected"){if(this.options.collapsible&&b==this.options.selected)return;this.select(b)}else this.options[a]=b,this._tabify()},_tabId:function(a){return a.title&&a.title.replace(/\s/g,"_").replace(/[^\w\u00c0-\uFFFF-]/g,"")||this.options.idPrefix+e()},_sanitizeSelector:function(a){return a.replace(/:/g,"\\:")},_cookie:function(){var b=this.cookie||(this.cookie=this.options.cookie.name||"ui-tabs-"+f());return a.cookie.apply(null,[b].concat(a.makeArray(arguments)))},_ui:function(a,b){return{tab:a,panel:b,index:this.anchors.index(a)}},_cleanup:function(){this.lis.filter(".ui-state-processing").removeClass("ui-state-processing").find("span:data(label.tabs)").each(function(){var b=a(this);b.html(b.data("label.tabs")).removeData("label.tabs")})},_tabify:function(c){function m(b,c){b.css("display",""),!a.support.opacity&&c.opacity&&b[0].style.removeAttribute("filter")}var d=this,e=this.options,f=/^#.+/;this.list=this.element.find("ol,ul").eq(0),this.lis=a(" > li:has(a[href])",this.list),this.anchors=this.lis.map(function(){return a("a",this)[0]}),this.panels=a([]),this.anchors.each(function(b,c){var g=a(c).attr("href"),h=g.split("#")[0],i;h&&(h===location.toString().split("#")[0]||(i=a("base")[0])&&h===i.href)&&(g=c.hash,c.href=g);if(f.test(g))d.panels=d.panels.add(d.element.find(d._sanitizeSelector(g)));else if(g&&g!=="#"){a.data(c,"href.tabs",g),a.data(c,"load.tabs",g.replace(/#.*$/,""));var j=d._tabId(c);c.href="#"+j;var k=d.element.find("#"+j);k.length||(k=a(e.panelTemplate).attr("id",j).addClass("ui-tabs-panel ui-widget-content ui-corner-bottom").insertAfter(d.panels[b-1]||d.list),k.data("destroy.tabs",!0)),d.panels=d.panels.add(k)}else e.disabled.push(b)}),c?(this.element.addClass("ui-tabs ui-widget ui-widget-content ui-corner-all"),this.list.addClass("ui-tabs-nav ui-helper-reset ui-helper-clearfix ui-widget-header ui-corner-all"),this.lis.addClass("ui-state-default ui-corner-top"),this.panels.addClass("ui-tabs-panel ui-widget-content ui-corner-bottom"),e.selected===b?(location.hash&&this.anchors.each(function(a,b){if(b.hash==location.hash){e.selected=a;return!1}}),typeof e.selected!="number"&&e.cookie&&(e.selected=parseInt(d._cookie(),10)),typeof e.selected!="number"&&this.lis.filter(".ui-tabs-selected").length&&(e.selected=this.lis.index(this.lis.filter(".ui-tabs-selected"))),e.selected=e.selected||(this.lis.length?0:-1)):e.selected===null&&(e.selected=-1),e.selected=e.selected>=0&&this.anchors[e.selected]||e.selected<0?e.selected:0,e.disabled=a.unique(e.disabled.concat(a.map(this.lis.filter(".ui-state-disabled"),function(a,b){return d.lis.index(a)}))).sort(),a.inArray(e.selected,e.disabled)!=-1&&e.disabled.splice(a.inArray(e.selected,e.disabled),1),this.panels.addClass("ui-tabs-hide"),this.lis.removeClass("ui-tabs-selected ui-state-active"),e.selected>=0&&this.anchors.length&&(d.element.find(d._sanitizeSelector(d.anchors[e.selected].hash)).removeClass("ui-tabs-hide"),this.lis.eq(e.selected).addClass("ui-tabs-selected ui-state-active"),d.element.queue("tabs",function(){d._trigger("show",null,d._ui(d.anchors[e.selected],d.element.find(d._sanitizeSelector(d.anchors[e.selected].hash))[0]))}),this.load(e.selected)),a(window).bind("unload",function(){d.lis.add(d.anchors).unbind(".tabs"),d.lis=d.anchors=d.panels=null})):e.selected=this.lis.index(this.lis.filter(".ui-tabs-selected")),this.element[e.collapsible?"addClass":"removeClass"]("ui-tabs-collapsible"),e.cookie&&this._cookie(e.selected,e.cookie);for(var g=0,h;h=this.lis[g];g++)a(h)[a.inArray(g,e.disabled)!=-1&&!a(h).hasClass("ui-tabs-selected")?"addClass":"removeClass"]("ui-state-disabled");e.cache===!1&&this.anchors.removeData("cache.tabs"),this.lis.add(this.anchors).unbind(".tabs");if(e.event!=="mouseover"){var i=function(a,b){b.is(":not(.ui-state-disabled)")&&b.addClass("ui-state-"+a)},j=function(a,b){b.removeClass("ui-state-"+a)};this.lis.bind("mouseover.tabs",function(){i("hover",a(this))}),this.lis.bind("mouseout.tabs",function(){j("hover",a(this))}),this.anchors.bind("focus.tabs",function(){i("focus",a(this).closest("li"))}),this.anchors.bind("blur.tabs",function(){j("focus",a(this).closest("li"))})}var k,l;e.fx&&(a.isArray(e.fx)?(k=e.fx[0],l=e.fx[1]):k=l=e.fx);var n=l?function(b,c){a(b).closest("li").addClass("ui-tabs-selected ui-state-active"),c.hide().removeClass("ui-tabs-hide").animate(l,l.duration||"normal",function(){m(c,l),d._trigger("show",null,d._ui(b,c[0]))})}:function(b,c){a(b).closest("li").addClass("ui-tabs-selected ui-state-active"),c.removeClass("ui-tabs-hide"),d._trigger("show",null,d._ui(b,c[0]))},o=k?function(a,b){b.animate(k,k.duration||"normal",function(){d.lis.removeClass("ui-tabs-selected ui-state-active"),b.addClass("ui-tabs-hide"),m(b,k),d.element.dequeue("tabs")})}:function(a,b,c){d.lis.removeClass("ui-tabs-selected ui-state-active"),b.addClass("ui-tabs-hide"),d.element.dequeue("tabs")};this.anchors.bind(e.event+".tabs",function(){var b=this,c=a(b).closest("li"),f=d.panels.filter(":not(.ui-tabs-hide)"),g=d.element.find(d._sanitizeSelector(b.hash));if(c.hasClass("ui-tabs-selected")&&!e.collapsible||c.hasClass("ui-state-disabled")||c.hasClass("ui-state-processing")||d.panels.filter(":animated").length||d._trigger("select",null,d._ui(this,g[0]))===!1){this.blur();return!1}e.selected=d.anchors.index(this),d.abort();if(e.collapsible){if(c.hasClass("ui-tabs-selected")){e.selected=-1,e.cookie&&d._cookie(e.selected,e.cookie),d.element.queue("tabs",function(){o(b,f)}).dequeue("tabs"),this.blur();return!1}if(!f.length){e.cookie&&d._cookie(e.selected,e.cookie),d.element.queue("tabs",function(){n(b,g)}),d.load(d.anchors.index(this)),this.blur();return!1}}e.cookie&&d._cookie(e.selected,e.cookie);if(g.length)f.length&&d.element.queue("tabs",function(){o(b,f)}),d.element.queue("tabs",function(){n(b,g)}),d.load(d.anchors.index(this));else throw"jQuery UI Tabs: Mismatching fragment identifier.";a.browser.msie&&this.blur()}),this.anchors.bind("click.tabs",function(){return!1})},_getIndex:function(a){typeof a=="string"&&(a=this.anchors.index(this.anchors.filter("[href$="+a+"]")));return a},destroy:function(){var b=this.options;this.abort(),this.element.unbind(".tabs").removeClass("ui-tabs ui-widget ui-widget-content ui-corner-all ui-tabs-collapsible").removeData("tabs"),this.list.removeClass("ui-tabs-nav ui-helper-reset ui-helper-clearfix ui-widget-header ui-corner-all"),this.anchors.each(function(){var b=a.data(this,"href.tabs");b&&(this.href=b);var c=a(this).unbind(".tabs");a.each(["href","load","cache"],function(a,b){c.removeData(b+".tabs")})}),this.lis.unbind(".tabs").add(this.panels).each(function(){a.data(this,"destroy.tabs")?a(this).remove():a(this).removeClass(["ui-state-default","ui-corner-top","ui-tabs-selected","ui-state-active","ui-state-hover","ui-state-focus","ui-state-disabled","ui-tabs-panel","ui-widget-content","ui-corner-bottom","ui-tabs-hide"].join(" "))}),b.cookie&&this._cookie(null,b.cookie);return this},add:function(c,d,e){e===b&&(e=this.anchors.length);var f=this,g=this.options,h=a(g.tabTemplate.replace(/#\{href\}/g,c).replace(/#\{label\}/g,d)),i=c.indexOf("#")?this._tabId(a("a",h)[0]):c.replace("#","");h.addClass("ui-state-default ui-corner-top").data("destroy.tabs",!0);var j=f.element.find("#"+i);j.length||(j=a(g.panelTemplate).attr("id",i).data("destroy.tabs",!0)),j.addClass("ui-tabs-panel ui-widget-content ui-corner-bottom ui-tabs-hide"),e>=this.lis.length?(h.appendTo(this.list),j.appendTo(this.list[0].parentNode)):(h.insertBefore(this.lis[e]),j.insertBefore(this.panels[e])),g.disabled=a.map(g.disabled,function(a,b){return a>=e?++a:a}),this._tabify(),this.anchors.length==1&&(g.selected=0,h.addClass("ui-tabs-selected ui-state-active"),j.removeClass("ui-tabs-hide"),this.element.queue("tabs",function(){f._trigger("show",null,f._ui(f.anchors[0],f.panels[0]))}),this.load(0)),this._trigger("add",null,this._ui(this.anchors[e],this.panels[e]));return this},remove:function(b){b=this._getIndex(b);var c=this.options,d=this.lis.eq(b).remove(),e=this.panels.eq(b).remove();d.hasClass("ui-tabs-selected")&&this.anchors.length>1&&this.select(b+(b+1<this.anchors.length?1:-1)),c.disabled=a.map(a.grep(c.disabled,function(a,c){return a!=b}),function(a,c){return a>=b?--a:a}),this._tabify(),this._trigger("remove",null,this._ui(d.find("a")[0],e[0]));return this},enable:function(b){b=this._getIndex(b);var c=this.options;if(a.inArray(b,c.disabled)!=-1){this.lis.eq(b).removeClass("ui-state-disabled"),c.disabled=a.grep(c.disabled,function(a,c){return a!=b}),this._trigger("enable",null,this._ui(this.anchors[b],this.panels[b]));return this}},disable:function(a){a=this._getIndex(a);var b=this,c=this.options;a!=c.selected&&(this.lis.eq(a).addClass("ui-state-disabled"),c.disabled.push(a),c.disabled.sort(),this._trigger("disable",null,this._ui(this.anchors[a],this.panels[a])));return this},select:function(a){a=this._getIndex(a);if(a==-1)if(this.options.collapsible&&this.options.selected!=-1)a=this.options.selected;else return this;this.anchors.eq(a).trigger(this.options.event+".tabs");return this},load:function(b){b=this._getIndex(b);var c=this,d=this.options,e=this.anchors.eq(b)[0],f=a.data(e,"load.tabs");this.abort();if(!f||this.element.queue("tabs").length!==0&&a.data(e,"cache.tabs"))this.element.dequeue("tabs");else{this.lis.eq(b).addClass("ui-state-processing");if(d.spinner){var g=a("span",e);g.data("label.tabs",g.html()).html(d.spinner)}this.xhr=a.ajax(a.extend({},d.ajaxOptions,{url:f,success:function(f,g){c.element.find(c._sanitizeSelector(e.hash)).html(f),c._cleanup(),d.cache&&a.data(e,"cache.tabs",!0),c._trigger("load",null,c._ui(c.anchors[b],c.panels[b]));try{d.ajaxOptions.success(f,g)}catch(h){}},error:function(a,f,g){c._cleanup(),c._trigger("load",null,c._ui(c.anchors[b],c.panels[b]));try{d.ajaxOptions.error(a,f,b,e)}catch(g){}}})),c.element.dequeue("tabs");return this}},abort:function(){this.element.queue([]),this.panels.stop(!1,!0),this.element.queue("tabs",this.element.queue("tabs").splice(-2,2)),this.xhr&&(this.xhr.abort(),delete this.xhr),this._cleanup();return this},url:function(a,b){this.anchors.eq(a).removeData("cache.tabs").data("load.tabs",b);return this},length:function(){return this.anchors.length}}),a.extend(a.ui.tabs,{version:"1.8.17"}),a.extend(a.ui.tabs.prototype,{rotation:null,rotate:function(a,b){var c=this,d=this.options,e=c._rotate||(c._rotate=function(b){clearTimeout(c.rotation),c.rotation=setTimeout(function(){var a=d.selected;c.select(++a<c.anchors.length?a:0)},a),b&&b.stopPropagation()}),f=c._unrotate||(c._unrotate=b?function(a){t=d.selected,e()}:function(a){a.clientX&&c.rotate(null)});a?(this.element.bind("tabsshow",e),this.anchors.bind(d.event+".tabs",f),e()):(clearTimeout(c.rotation),this.element.unbind("tabsshow",e),this.anchors.unbind(d.event+".tabs",f),delete this._rotate,delete this._unrotate);return this}})})(jQuery);/*
+ * jQuery UI Datepicker 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Datepicker
+ *
+ * Depends:
+ * jquery.ui.core.js
+ */(function($,undefined){function isArray(a){return a&&($.browser.safari&&typeof a=="object"&&a.length||a.constructor&&a.constructor.toString().match(/\Array\(\)/))}function extendRemove(a,b){$.extend(a,b);for(var c in b)if(b[c]==null||b[c]==undefined)a[c]=b[c];return a}function bindHover(a){var b="button, .ui-datepicker-prev, .ui-datepicker-next, .ui-datepicker-calendar td a";return a.bind("mouseout",function(a){var c=$(a.target).closest(b);!c.length||c.removeClass("ui-state-hover ui-datepicker-prev-hover ui-datepicker-next-hover")}).bind("mouseover",function(c){var d=$(c.target).closest(b);!$.datepicker._isDisabledDatepicker(instActive.inline?a.parent()[0]:instActive.input[0])&&!!d.length&&(d.parents(".ui-datepicker-calendar").find("a").removeClass("ui-state-hover"),d.addClass("ui-state-hover"),d.hasClass("ui-datepicker-prev")&&d.addClass("ui-datepicker-prev-hover"),d.hasClass("ui-datepicker-next")&&d.addClass("ui-datepicker-next-hover"))})}function Datepicker(){this.debug=!1,this._curInst=null,this._keyEvent=!1,this._disabledInputs=[],this._datepickerShowing=!1,this._inDialog=!1,this._mainDivId="ui-datepicker-div",this._inlineClass="ui-datepicker-inline",this._appendClass="ui-datepicker-append",this._triggerClass="ui-datepicker-trigger",this._dialogClass="ui-datepicker-dialog",this._disableClass="ui-datepicker-disabled",this._unselectableClass="ui-datepicker-unselectable",this._currentClass="ui-datepicker-current-day",this._dayOverClass="ui-datepicker-days-cell-over",this.regional=[],this.regional[""]={closeText:"Done",prevText:"Prev",nextText:"Next",currentText:"Today",monthNames:["January","February","March","April","May","June","July","August","September","October","November","December"],monthNamesShort:["Jan","Feb","Mar","Apr","May","Jun","Jul","Aug","Sep","Oct","Nov","Dec"],dayNames:["Sunday","Monday","Tuesday","Wednesday","Thursday","Friday","Saturday"],dayNamesShort:["Sun","Mon","Tue","Wed","Thu","Fri","Sat"],dayNamesMin:["Su","Mo","Tu","We","Th","Fr","Sa"],weekHeader:"Wk",dateFormat:"mm/dd/yy",firstDay:0,isRTL:!1,showMonthAfterYear:!1,yearSuffix:""},this._defaults={showOn:"focus",showAnim:"fadeIn",showOptions:{},defaultDate:null,appendText:"",buttonText:"...",buttonImage:"",buttonImageOnly:!1,hideIfNoPrevNext:!1,navigationAsDateFormat:!1,gotoCurrent:!1,changeMonth:!1,changeYear:!1,yearRange:"c-10:c+10",showOtherMonths:!1,selectOtherMonths:!1,showWeek:!1,calculateWeek:this.iso8601Week,shortYearCutoff:"+10",minDate:null,maxDate:null,duration:"fast",beforeShowDay:null,beforeShow:null,onSelect:null,onChangeMonthYear:null,onClose:null,numberOfMonths:1,showCurrentAtPos:0,stepMonths:1,stepBigMonths:12,altField:"",altFormat:"",constrainInput:!0,showButtonPanel:!1,autoSize:!1,disabled:!1},$.extend(this._defaults,this.regional[""]),this.dpDiv=bindHover($('<div id="'+this._mainDivId+'" class="ui-datepicker ui-widget ui-widget-content ui-helper-clearfix ui-corner-all"></div>'))}$.extend($.ui,{datepicker:{version:"1.8.17"}});var PROP_NAME="datepicker",dpuuid=(new Date).getTime(),instActive;$.extend(Datepicker.prototype,{markerClassName:"hasDatepicker",maxRows:4,log:function(){this.debug&&console.log.apply("",arguments)},_widgetDatepicker:function(){return this.dpDiv},setDefaults:function(a){extendRemove(this._defaults,a||{});return this},_attachDatepicker:function(target,settings){var inlineSettings=null;for(var attrName in this._defaults){var attrValue=target.getAttribute("date:"+attrName);if(attrValue){inlineSettings=inlineSettings||{};try{inlineSettings[attrName]=eval(attrValue)}catch(err){inlineSettings[attrName]=attrValue}}}var nodeName=target.nodeName.toLowerCase(),inline=nodeName=="div"||nodeName=="span";target.id||(this.uuid+=1,target.id="dp"+this.uuid);var inst=this._newInst($(target),inline);inst.settings=$.extend({},settings||{},inlineSettings||{}),nodeName=="input"?this._connectDatepicker(target,inst):inline&&this._inlineDatepicker(target,inst)},_newInst:function(a,b){var c=a[0].id.replace(/([^A-Za-z0-9_-])/g,"\\\\$1");return{id:c,input:a,selectedDay:0,selectedMonth:0,selectedYear:0,drawMonth:0,drawYear:0,inline:b,dpDiv:b?bindHover($('<div class="'+this._inlineClass+' ui-datepicker ui-widget ui-widget-content ui-helper-clearfix ui-corner-all"></div>')):this.dpDiv}},_connectDatepicker:function(a,b){var c=$(a);b.append=$([]),b.trigger=$([]);c.hasClass(this.markerClassName)||(this._attachments(c,b),c.addClass(this.markerClassName).keydown(this._doKeyDown).keypress(this._doKeyPress).keyup(this._doKeyUp).bind("setData.datepicker",function(a,c,d){b.settings[c]=d}).bind("getData.datepicker",function(a,c){return this._get(b,c)}),this._autoSize(b),$.data(a,PROP_NAME,b),b.settings.disabled&&this._disableDatepicker(a))},_attachments:function(a,b){var c=this._get(b,"appendText"),d=this._get(b,"isRTL");b.append&&b.append.remove(),c&&(b.append=$('<span class="'+this._appendClass+'">'+c+"</span>"),a[d?"before":"after"](b.append)),a.unbind("focus",this._showDatepicker),b.trigger&&b.trigger.remove();var e=this._get(b,"showOn");(e=="focus"||e=="both")&&a.focus(this._showDatepicker);if(e=="button"||e=="both"){var f=this._get(b,"buttonText"),g=this._get(b,"buttonImage");b.trigger=$(this._get(b,"buttonImageOnly")?$("<img/>").addClass(this._triggerClass).attr({src:g,alt:f,title:f}):$('<button type="button"></button>').addClass(this._triggerClass).html(g==""?f:$("<img/>").attr({src:g,alt:f,title:f}))),a[d?"before":"after"](b.trigger),b.trigger.click(function(){$.datepicker._datepickerShowing&&$.datepicker._lastInput==a[0]?$.datepicker._hideDatepicker():$.datepicker._showDatepicker(a[0]);return!1})}},_autoSize:function(a){if(this._get(a,"autoSize")&&!a.inline){var b=new Date(2009,11,20),c=this._get(a,"dateFormat");if(c.match(/[DM]/)){var d=function(a){var b=0,c=0;for(var d=0;d<a.length;d++)a[d].length>b&&(b=a[d].length,c=d);return c};b.setMonth(d(this._get(a,c.match(/MM/)?"monthNames":"monthNamesShort"))),b.setDate(d(this._get(a,c.match(/DD/)?"dayNames":"dayNamesShort"))+20-b.getDay())}a.input.attr("size",this._formatDate(a,b).length)}},_inlineDatepicker:function(a,b){var c=$(a);c.hasClass(this.markerClassName)||(c.addClass(this.markerClassName).append(b.dpDiv).bind("setData.datepicker",function(a,c,d){b.settings[c]=d}).bind("getData.datepicker",function(a,c){return this._get(b,c)}),$.data(a,PROP_NAME,b),this._setDate(b,this._getDefaultDate(b),!0),this._updateDatepicker(b),this._updateAlternate(b),b.settings.disabled&&this._disableDatepicker(a),b.dpDiv.css("display","block"))},_dialogDatepicker:function(a,b,c,d,e){var f=this._dialogInst;if(!f){this.uuid+=1;var g="dp"+this.uuid;this._dialogInput=$('<input type="text" id="'+g+'" style="position: absolute; top: -100px; width: 0px; z-index: -10;"/>'),this._dialogInput.keydown(this._doKeyDown),$("body").append(this._dialogInput),f=this._dialogInst=this._newInst(this._dialogInput,!1),f.settings={},$.data(this._dialogInput[0],PROP_NAME,f)}extendRemove(f.settings,d||{}),b=b&&b.constructor==Date?this._formatDate(f,b):b,this._dialogInput.val(b),this._pos=e?e.length?e:[e.pageX,e.pageY]:null;if(!this._pos){var h=document.documentElement.clientWidth,i=document.documentElement.clientHeight,j=document.documentElement.scrollLeft||document.body.scrollLeft,k=document.documentElement.scrollTop||document.body.scrollTop;this._pos=[h/2-100+j,i/2-150+k]}this._dialogInput.css("left",this._pos[0]+20+"px").css("top",this._pos[1]+"px"),f.settings.onSelect=c,this._inDialog=!0,this.dpDiv.addClass(this._dialogClass),this._showDatepicker(this._dialogInput[0]),$.blockUI&&$.blockUI(this.dpDiv),$.data(this._dialogInput[0],PROP_NAME,f);return this},_destroyDatepicker:function(a){var b=$(a),c=$.data(a,PROP_NAME);if(!!b.hasClass(this.markerClassName)){var d=a.nodeName.toLowerCase();$.removeData(a,PROP_NAME),d=="input"?(c.append.remove(),c.trigger.remove(),b.removeClass(this.markerClassName).unbind("focus",this._showDatepicker).unbind("keydown",this._doKeyDown).unbind("keypress",this._doKeyPress).unbind("keyup",this._doKeyUp)):(d=="div"||d=="span")&&b.removeClass(this.markerClassName).empty()}},_enableDatepicker:function(a){var b=$(a),c=$.data(a,PROP_NAME);if(!!b.hasClass(this.markerClassName)){var d=a.nodeName.toLowerCase();if(d=="input")a.disabled=!1,c.trigger.filter("button").each(function(){this.disabled=!1}).end().filter("img").css({opacity:"1.0",cursor:""});else if(d=="div"||d=="span"){var e=b.children("."+this._inlineClass);e.children().removeClass("ui-state-disabled"),e.find("select.ui-datepicker-month, select.ui-datepicker-year").removeAttr("disabled")}this._disabledInputs=$.map(this._disabledInputs,function(b){return b==a?null:b})}},_disableDatepicker:function(a){var b=$(a),c=$.data(a,PROP_NAME);if(!!b.hasClass(this.markerClassName)){var d=a.nodeName.toLowerCase();if(d=="input")a.disabled=!0,c.trigger.filter("button").each(function(){this.disabled=!0}).end().filter("img").css({opacity:"0.5",cursor:"default"});else if(d=="div"||d=="span"){var e=b.children("."+this._inlineClass);e.children().addClass("ui-state-disabled"),e.find("select.ui-datepicker-month, select.ui-datepicker-year").attr("disabled","disabled")}this._disabledInputs=$.map(this._disabledInputs,function(b){return b==a?null:b}),this._disabledInputs[this._disabledInputs.length]=a}},_isDisabledDatepicker:function(a){if(!a)return!1;for(var b=0;b<this._disabledInputs.length;b++)if(this._disabledInputs[b]==a)return!0;return!1},_getInst:function(a){try{return $.data(a,PROP_NAME)}catch(b){throw"Missing instance data for this datepicker"}},_optionDatepicker:function(a,b,c){var d=this._getInst(a);if(arguments.length==2&&typeof b=="string")return b=="defaults"?$.extend({},$.datepicker._defaults):d?b=="all"?$.extend({},d.settings):this._get(d,b):null;var e=b||{};typeof b=="string"&&(e={},e[b]=c);if(d){this._curInst==d&&this._hideDatepicker();var f=this._getDateDatepicker(a,!0),g=this._getMinMaxDate(d,"min"),h=this._getMinMaxDate(d,"max");extendRemove(d.settings,e),g!==null&&e.dateFormat!==undefined&&e.minDate===undefined&&(d.settings.minDate=this._formatDate(d,g)),h!==null&&e.dateFormat!==undefined&&e.maxDate===undefined&&(d.settings.maxDate=this._formatDate(d,h)),this._attachments($(a),d),this._autoSize(d),this._setDate(d,f),this._updateAlternate(d),this._updateDatepicker(d)}},_changeDatepicker:function(a,b,c){this._optionDatepicker(a,b,c)},_refreshDatepicker:function(a){var b=this._getInst(a);b&&this._updateDatepicker(b)},_setDateDatepicker:function(a,b){var c=this._getInst(a);c&&(this._setDate(c,b),this._updateDatepicker(c),this._updateAlternate(c))},_getDateDatepicker:function(a,b){var c=this._getInst(a);c&&!c.inline&&this._setDateFromField(c,b);return c?this._getDate(c):null},_doKeyDown:function(a){var b=$.datepicker._getInst(a.target),c=!0,d=b.dpDiv.is(".ui-datepicker-rtl");b._keyEvent=!0;if($.datepicker._datepickerShowing)switch(a.keyCode){case 9:$.datepicker._hideDatepicker(),c=!1;break;case 13:var e=$("td."+$.datepicker._dayOverClass+":not(."+$.datepicker._currentClass+")",b.dpDiv);e[0]&&$.datepicker._selectDay(a.target,b.selectedMonth,b.selectedYear,e[0]);var f=$.datepicker._get(b,"onSelect");if(f){var g=$.datepicker._formatDate(b);f.apply(b.input?b.input[0]:null,[g,b])}else $.datepicker._hideDatepicker();return!1;case 27:$.datepicker._hideDatepicker();break;case 33:$.datepicker._adjustDate(a.target,a.ctrlKey?-$.datepicker._get(b,"stepBigMonths"):-$.datepicker._get(b,"stepMonths"),"M");break;case 34:$.datepicker._adjustDate(a.target,a.ctrlKey?+$.datepicker._get(b,"stepBigMonths"):+$.datepicker._get(b,"stepMonths"),"M");break;case 35:(a.ctrlKey||a.metaKey)&&$.datepicker._clearDate(a.target),c=a.ctrlKey||a.metaKey;break;case 36:(a.ctrlKey||a.metaKey)&&$.datepicker._gotoToday(a.target),c=a.ctrlKey||a.metaKey;break;case 37:(a.ctrlKey||a.metaKey)&&$.datepicker._adjustDate(a.target,d?1:-1,"D"),c=a.ctrlKey||a.metaKey,a.originalEvent.altKey&&$.datepicker._adjustDate(a.target,a.ctrlKey?-$.datepicker._get(b,"stepBigMonths"):-$.datepicker._get(b,"stepMonths"),"M");break;case 38:(a.ctrlKey||a.metaKey)&&$.datepicker._adjustDate(a.target,-7,"D"),c=a.ctrlKey||a.metaKey;break;case 39:(a.ctrlKey||a.metaKey)&&$.datepicker._adjustDate(a.target,d?-1:1,"D"),c=a.ctrlKey||a.metaKey,a.originalEvent.altKey&&$.datepicker._adjustDate(a.target,a.ctrlKey?+$.datepicker._get(b,"stepBigMonths"):+$.datepicker._get(b,"stepMonths"),"M");break;case 40:(a.ctrlKey||a.metaKey)&&$.datepicker._adjustDate(a.target,7,"D"),c=a.ctrlKey||a.metaKey;break;default:c=!1}else a.keyCode==36&&a.ctrlKey?$.datepicker._showDatepicker(this):c=!1;c&&(a.preventDefault(),a.stopPropagation())},_doKeyPress:function(a){var b=$.datepicker._getInst(a.target);if($.datepicker._get(b,"constrainInput")){var c=$.datepicker._possibleChars($.datepicker._get(b,"dateFormat")),d=String.fromCharCode(a.charCode==undefined?a.keyCode:a.charCode);return a.ctrlKey||a.metaKey||d<" "||!c||c.indexOf(d)>-1}},_doKeyUp:function(a){var b=$.datepicker._getInst(a.target);if(b.input.val()!=b.lastVal)try{var c=$.datepicker.parseDate($.datepicker._get(b,"dateFormat"),b.input?b.input.val():null,$.datepicker._getFormatConfig(b));c&&($.datepicker._setDateFromField(b),$.datepicker._updateAlternate(b),$.datepicker._updateDatepicker(b))}catch(a){$.datepicker.log(a)}return!0},_showDatepicker:function(a){a=a.target||a,a.nodeName.toLowerCase()!="input"&&(a=$("input",a.parentNode)[0]);if(!$.datepicker._isDisabledDatepicker(a)&&$.datepicker._lastInput!=a){var b=$.datepicker._getInst(a);$.datepicker._curInst&&$.datepicker._curInst!=b&&($.datepicker._curInst.dpDiv.stop(!0,!0),b&&$.datepicker._datepickerShowing&&$.datepicker._hideDatepicker($.datepicker._curInst.input[0]));var c=$.datepicker._get(b,"beforeShow"),d=c?c.apply(a,[a,b]):{};if(d===!1)return;extendRemove(b.settings,d),b.lastVal=null,$.datepicker._lastInput=a,$.datepicker._setDateFromField(b),$.datepicker._inDialog&&(a.value=""),$.datepicker._pos||($.datepicker._pos=$.datepicker._findPos(a),$.datepicker._pos[1]+=a.offsetHeight);var e=!1;$(a).parents().each(function(){e|=$(this).css("position")=="fixed";return!e}),e&&$.browser.opera&&($.datepicker._pos[0]-=document.documentElement.scrollLeft,$.datepicker._pos[1]-=document.documentElement.scrollTop);var f={left:$.datepicker._pos[0],top:$.datepicker._pos[1]};$.datepicker._pos=null,b.dpDiv.empty(),b.dpDiv.css({position:"absolute",display:"block",top:"-1000px"}),$.datepicker._updateDatepicker(b),f=$.datepicker._checkOffset(b,f,e),b.dpDiv.css({position:$.datepicker._inDialog&&$.blockUI?"static":e?"fixed":"absolute",display:"none",left:f.left+"px",top:f.top+"px"});if(!b.inline){var g=$.datepicker._get(b,"showAnim"),h=$.datepicker._get(b,"duration"),i=function(){var a=b.dpDiv.find("iframe.ui-datepicker-cover");if(!!a.length){var c=$.datepicker._getBorders(b.dpDiv);a.css({left:-c[0],top:-c[1],width:b.dpDiv.outerWidth(),height:b.dpDiv.outerHeight()})}};b.dpDiv.zIndex($(a).zIndex()+1),$.datepicker._datepickerShowing=!0,$.effects&&$.effects[g]?b.dpDiv.show(g,$.datepicker._get(b,"showOptions"),h,i):b.dpDiv[g||"show"](g?h:null,i),(!g||!h)&&i(),b.input.is(":visible")&&!b.input.is(":disabled")&&b.input.focus(),$.datepicker._curInst=b}}},_updateDatepicker:function(a){var b=this;b.maxRows=4;var c=$.datepicker._getBorders(a.dpDiv);instActive=a,a.dpDiv.empty().append(this._generateHTML(a));var d=a.dpDiv.find("iframe.ui-datepicker-cover");!d.length||d.css({left:-c[0],top:-c[1],width:a.dpDiv.outerWidth(),height:a.dpDiv.outerHeight()}),a.dpDiv.find("."+this._dayOverClass+" a").mouseover();var e=this._getNumberOfMonths(a),f=e[1],g=17;a.dpDiv.removeClass("ui-datepicker-multi-2 ui-datepicker-multi-3 ui-datepicker-multi-4").width(""),f>1&&a.dpDiv.addClass("ui-datepicker-multi-"+f).css("width",g*f+"em"),a.dpDiv[(e[0]!=1||e[1]!=1?"add":"remove")+"Class"]("ui-datepicker-multi"),a.dpDiv[(this._get(a,"isRTL")?"add":"remove")+"Class"]("ui-datepicker-rtl"),a==$.datepicker._curInst&&$.datepicker._datepickerShowing&&a.input&&a.input.is(":visible")&&!a.input.is(":disabled")&&a.input[0]!=document.activeElement&&a.input.focus();if(a.yearshtml){var h=a.yearshtml;setTimeout(function(){h===a.yearshtml&&a.yearshtml&&a.dpDiv.find("select.ui-datepicker-year:first").replaceWith(a.yearshtml),h=a.yearshtml=null},0)}},_getBorders:function(a){var b=function(a){return{thin:1,medium:2,thick:3}[a]||a};return[parseFloat(b(a.css("border-left-width"))),parseFloat(b(a.css("border-top-width")))]},_checkOffset:function(a,b,c){var d=a.dpDiv.outerWidth(),e=a.dpDiv.outerHeight(),f=a.input?a.input.outerWidth():0,g=a.input?a.input.outerHeight():0,h=document.documentElement.clientWidth+$(document).scrollLeft(),i=document.documentElement.clientHeight+$(document).scrollTop();b.left-=this._get(a,"isRTL")?d-f:0,b.left-=c&&b.left==a.input.offset().left?$(document).scrollLeft():0,b.top-=c&&b.top==a.input.offset().top+g?$(document).scrollTop():0,b.left-=Math.min(b.left,b.left+d>h&&h>d?Math.abs(b.left+d-h):0),b.top-=Math.min(b.top,b.top+e>i&&i>e?Math.abs(e+g):0);return b},_findPos:function(a){var b=this._getInst(a),c=this._get(b,"isRTL");while(a&&(a.type=="hidden"||a.nodeType!=1||$.expr.filters.hidden(a)))a=a[c?"previousSibling":"nextSibling"];var d=$(a).offset();return[d.left,d.top]},_hideDatepicker:function(a){var b=this._curInst;if(!(!b||a&&b!=$.data(a,PROP_NAME))&&this._datepickerShowing){var c=this._get(b,"showAnim"),d=this._get(b,"duration"),e=this,f=function(){$.datepicker._tidyDialog(b),e._curInst=null};$.effects&&$.effects[c]?b.dpDiv.hide(c,$.datepicker._get(b,"showOptions"),d,f):b.dpDiv[c=="slideDown"?"slideUp":c=="fadeIn"?"fadeOut":"hide"](c?d:null,f),c||f(),this._datepickerShowing=!1;var g=this._get(b,"onClose");g&&g.apply(b.input?b.input[0]:null,[b.input?b.input.val():"",b]),this._lastInput=null,this._inDialog&&(this._dialogInput.css({position:"absolute",left:"0",top:"-100px"}),$.blockUI&&($.unblockUI(),$("body").append(this.dpDiv))),this._inDialog=!1}},_tidyDialog:function(a){a.dpDiv.removeClass(this._dialogClass).unbind(".ui-datepicker-calendar")},_checkExternalClick:function(a){if(!!$.datepicker._curInst){var b=$(a.target),c=$.datepicker._getInst(b[0]);(b[0].id!=$.datepicker._mainDivId&&b.parents("#"+$.datepicker._mainDivId).length==0&&!b.hasClass($.datepicker.markerClassName)&&!b.hasClass($.datepicker._triggerClass)&&$.datepicker._datepickerShowing&&(!$.datepicker._inDialog||!$.blockUI)||b.hasClass($.datepicker.markerClassName)&&$.datepicker._curInst!=c)&&$.datepicker._hideDatepicker()}},_adjustDate:function(a,b,c){var d=$(a),e=this._getInst(d[0]);this._isDisabledDatepicker(d[0])||(this._adjustInstDate(e,b+(c=="M"?this._get(e,"showCurrentAtPos"):0),c),this._updateDatepicker(e))},_gotoToday:function(a){var b=$(a),c=this._getInst(b[0]);if(this._get(c,"gotoCurrent")&&c.currentDay)c.selectedDay=c.currentDay,c.drawMonth=c.selectedMonth=c.currentMonth,c.drawYear=c.selectedYear=c.currentYear;else{var d=new Date;c.selectedDay=d.getDate(),c.drawMonth=c.selectedMonth=d.getMonth(),c.drawYear=c.selectedYear=d.getFullYear()}this._notifyChange(c),this._adjustDate(b)},_selectMonthYear:function(a,b,c){var d=$(a),e=this._getInst(d[0]);e["selected"+(c=="M"?"Month":"Year")]=e["draw"+(c=="M"?"Month":"Year")]=parseInt(b.options[b.selectedIndex].value,10),this._notifyChange(e),this._adjustDate(d)},_selectDay:function(a,b,c,d){var e=$(a);if(!$(d).hasClass(this._unselectableClass)&&!this._isDisabledDatepicker(e[0])){var f=this._getInst(e[0]);f.selectedDay=f.currentDay=$("a",d).html(),f.selectedMonth=f.currentMonth=b,f.selectedYear=f.currentYear=c,this._selectDate(a,this._formatDate(f,f.currentDay,f.currentMonth,f.currentYear))}},_clearDate:function(a){var b=$(a),c=this._getInst(b[0]);this._selectDate(b,"")},_selectDate:function(a,b){var c=$(a),d=this._getInst(c[0]);b=b!=null?b:this._formatDate(d),d.input&&d.input.val(b),this._updateAlternate(d);var e=this._get(d,"onSelect");e?e.apply(d.input?d.input[0]:null,[b,d]):d.input&&d.input.trigger("change"),d.inline?this._updateDatepicker(d):(this._hideDatepicker(),this._lastInput=d.input[0],typeof d.input[0]!="object"&&d.input.focus(),this._lastInput=null)},_updateAlternate:function(a){var b=this._get(a,"altField");if(b){var c=this._get(a,"altFormat")||this._get(a,"dateFormat"),d=this._getDate(a),e=this.formatDate(c,d,this._getFormatConfig(a));$(b).each(function(){$(this).val(e)})}},noWeekends:function(a){var b=a.getDay();return[b>0&&b<6,""]},iso8601Week:function(a){var b=new Date(a.getTime());b.setDate(b.getDate()+4-(b.getDay()||7));var c=b.getTime();b.setMonth(0),b.setDate(1);return Math.floor(Math.round((c-b)/864e5)/7)+1},parseDate:function(a,b,c){if(a==null||b==null)throw"Invalid arguments";b=typeof b=="object"?b.toString():b+"";if(b=="")return null;var d=(c?c.shortYearCutoff:null)||this._defaults.shortYearCutoff;d=typeof d!="string"?d:(new Date).getFullYear()%100+parseInt(d,10);var e=(c?c.dayNamesShort:null)||this._defaults.dayNamesShort,f=(c?c.dayNames:null)||this._defaults.dayNames,g=(c?c.monthNamesShort:null)||this._defaults.monthNamesShort,h=(c?c.monthNames:null)||this._defaults.monthNames,i=-1,j=-1,k=-1,l=-1,m=!1,n=function(b){var c=s+1<a.length&&a.charAt(s+1)==b;c&&s++;return c},o=function(a){var c=n(a),d=a=="@"?14:a=="!"?20:a=="y"&&c?4:a=="o"?3:2,e=new RegExp("^\\d{1,"+d+"}"),f=b.substring(r).match(e);if(!f)throw"Missing number at position "+r;r+=f[0].length;return parseInt(f[0],10)},p=function(a,c,d){var e=$.map(n(a)?d:c,function(a,b){return[[b,a]]}).sort(function(a,b){return-(a[1].length-b[1].length)}),f=-1;$.each(e,function(a,c){var d=c[1];if(b.substr(r,d.length).toLowerCase()==d.toLowerCase()){f=c[0],r+=d.length;return!1}});if(f!=-1)return f+1;throw"Unknown name at position "+r},q=function(){if(b.charAt(r)!=a.charAt(s))throw"Unexpected literal at position "+r;r++},r=0;for(var s=0;s<a.length;s++)if(m)a.charAt(s)=="'"&&!n("'")?m=!1:q();else switch(a.charAt(s)){case"d":k=o("d");break;case"D":p("D",e,f);break;case"o":l=o("o");break;case"m":j=o("m");break;case"M":j=p("M",g,h);break;case"y":i=o("y");break;case"@":var t=new Date(o("@"));i=t.getFullYear(),j=t.getMonth()+1,k=t.getDate();break;case"!":var t=new Date((o("!")-this._ticksTo1970)/1e4);i=t.getFullYear(),j=t.getMonth()+1,k=t.getDate();break;case"'":n("'")?q():m=!0;break;default:q()}if(r<b.length)throw"Extra/unparsed characters found in date: "+b.substring(r);i==-1?i=(new Date).getFullYear():i<100&&(i+=(new Date).getFullYear()-(new Date).getFullYear()%100+(i<=d?0:-100));if(l>-1){j=1,k=l;for(;;){var u=this._getDaysInMonth(i,j-1);if(k<=u)break;j++,k-=u}}var t=this._daylightSavingAdjust(new Date(i,j-1,k));if(t.getFullYear()!=i||t.getMonth()+1!=j||t.getDate()!=k)throw"Invalid date";return t},ATOM:"yy-mm-dd",COOKIE:"D, dd M yy",ISO_8601:"yy-mm-dd",RFC_822:"D, d M y",RFC_850:"DD, dd-M-y",RFC_1036:"D, d M y",RFC_1123:"D, d M yy",RFC_2822:"D, d M yy",RSS:"D, d M y",TICKS:"!",TIMESTAMP:"@",W3C:"yy-mm-dd",_ticksTo1970:(718685+Math.floor(492.5)-Math.floor(19.7)+Math.floor(4.925))*24*60*60*1e7,formatDate:function(a,b,c){if(!b)return"";var d=(c?c.dayNamesShort:null)||this._defaults.dayNamesShort,e=(c?c.dayNames:null)||this._defaults.dayNames,f=(c?c.monthNamesShort:null)||this._defaults.monthNamesShort,g=(c?c.monthNames:null)||this._defaults.monthNames,h=function(b){var c=m+1<a.length&&a.charAt(m+1)==b;c&&m++;return c},i=function(a,b,c){var d=""+b;if(h(a))while(d.length<c)d="0"+d;return d},j=function(a,b,c,d){return h(a)?d[b]:c[b]},k="",l=!1;if(b)for(var m=0;m<a.length;m++)if(l)a.charAt(m)=="'"&&!h("'")?l=!1:k+=a.charAt(m);else switch(a.charAt(m)){case"d":k+=i("d",b.getDate(),2);break;case"D":k+=j("D",b.getDay(),d,e);break;case"o":k+=i("o",Math.round(((new Date(b.getFullYear(),b.getMonth(),b.getDate())).getTime()-(new Date(b.getFullYear(),0,0)).getTime())/864e5),3);break;case"m":k+=i("m",b.getMonth()+1,2);break;case"M":k+=j("M",b.getMonth(),f,g);break;case"y":k+=h("y")?b.getFullYear():(b.getYear()%100<10?"0":"")+b.getYear()%100;break;case"@":k+=b.getTime();break;case"!":k+=b.getTime()*1e4+this._ticksTo1970;break;case"'":h("'")?k+="'":l=!0;break;default:k+=a.charAt(m)}return k},_possibleChars:function(a){var b="",c=!1,d=function(b){var c=e+1<a.length&&a.charAt(e+1)==b;c&&e++;return c};for(var e=0;e<a.length;e++)if(c)a.charAt(e)=="'"&&!d("'")?c=!1:b+=a.charAt(e);else switch(a.charAt(e)){case"d":case"m":case"y":case"@":b+="0123456789";break;case"D":case"M":return null;case"'":d("'")?b+="'":c=!0;break;default:b+=a.charAt(e)}return b},_get:function(a,b){return a.settings[b]!==undefined?a.settings[b]:this._defaults[b]},_setDateFromField:function(a,b){if(a.input.val()!=a.lastVal){var c=this._get(a,"dateFormat"),d=a.lastVal=a.input?a.input.val():null,e,f;e=f=this._getDefaultDate(a);var g=this._getFormatConfig(a);try{e=this.parseDate(c,d,g)||f}catch(h){this.log(h),d=b?"":d}a.selectedDay=e.getDate(),a.drawMonth=a.selectedMonth=e.getMonth(),a.drawYear=a.selectedYear=e.getFullYear(),a.currentDay=d?e.getDate():0,a.currentMonth=d?e.getMonth():0,a.currentYear=d?e.getFullYear():0,this._adjustInstDate(a)}},_getDefaultDate:function(a){return this._restrictMinMax(a,this._determineDate(a,this._get(a,"defaultDate"),new Date))},_determineDate:function(a,b,c){var d=function(a){var b=new Date;b.setDate(b.getDate()+a);return b},e=function(b){try{return $.datepicker.parseDate($.datepicker._get(a,"dateFormat"),b,$.datepicker._getFormatConfig(a))}catch(c){}var d=(b.toLowerCase().match(/^c/)?$.datepicker._getDate(a):null)||new Date,e=d.getFullYear(),f=d.getMonth(),g=d.getDate(),h=/([+-]?[0-9]+)\s*(d|D|w|W|m|M|y|Y)?/g,i=h.exec(b);while(i){switch(i[2]||"d"){case"d":case"D":g+=parseInt(i[1],10);break;case"w":case"W":g+=parseInt(i[1],10)*7;break;case"m":case"M":f+=parseInt(i[1],10),g=Math.min(g,$.datepicker._getDaysInMonth(e,f));break;case"y":case"Y":e+=parseInt(i[1],10),g=Math.min(g,$.datepicker._getDaysInMonth(e,f))}i=h.exec(b)}return new Date(e,f,g)},f=b==null||b===""?c:typeof b=="string"?e(b):typeof b=="number"?isNaN(b)?c:d(b):new Date(b.getTime());f=f&&f.toString()=="Invalid Date"?c:f,f&&(f.setHours(0),f.setMinutes(0),f.setSeconds(0),f.setMilliseconds(0));return this._daylightSavingAdjust(f)},_daylightSavingAdjust:function(a){if(!a)return null;a.setHours(a.getHours()>12?a.getHours()+2:0);return a},_setDate:function(a,b,c){var d=!b,e=a.selectedMonth,f=a.selectedYear,g=this._restrictMinMax(a,this._determineDate(a,b,new Date));a.selectedDay=a.currentDay=g.getDate(),a.drawMonth=a.selectedMonth=a.currentMonth=g.getMonth(),a.drawYear=a.selectedYear=a.currentYear=g.getFullYear(),(e!=a.selectedMonth||f!=a.selectedYear)&&!c&&this._notifyChange(a),this._adjustInstDate(a),a.input&&a.input.val(d?"":this._formatDate(a))},_getDate:function(a){var b=!a.currentYear||a.input&&a.input.val()==""?null:this._daylightSavingAdjust(new Date(a.currentYear,a.currentMonth,a.currentDay));return b},_generateHTML:function(a){var b=new Date;b=this._daylightSavingAdjust(new Date(b.getFullYear(),b.getMonth(),b.getDate()));var c=this._get(a,"isRTL"),d=this._get(a,"showButtonPanel"),e=this._get(a,"hideIfNoPrevNext"),f=this._get(a,"navigationAsDateFormat"),g=this._getNumberOfMonths(a),h=this._get(a,"showCurrentAtPos"),i=this._get(a,"stepMonths"),j=g[0]!=1||g[1]!=1,k=this._daylightSavingAdjust(a.currentDay?new Date(a.currentYear,a.currentMonth,a.currentDay):new Date(9999,9,9)),l=this._getMinMaxDate(a,"min"),m=this._getMinMaxDate(a,"max"),n=a.drawMonth-h,o=a.drawYear;n<0&&(n+=12,o--);if(m){var p=this._daylightSavingAdjust(new Date(m.getFullYear(),m.getMonth()-g[0]*g[1]+1,m.getDate()));p=l&&p<l?l:p;while(this._daylightSavingAdjust(new Date(o,n,1))>p)n--,n<0&&(n=11,o--)}a.drawMonth=n,a.drawYear=o;var q=this._get(a,"prevText");q=f?this.formatDate(q,this._daylightSavingAdjust(new Date(o,n-i,1)),this._getFormatConfig(a)):q;var r=this._canAdjustMonth(a,-1,o,n)?'<a class="ui-datepicker-prev ui-corner-all" onclick="DP_jQuery_'+dpuuid+".datepicker._adjustDate('#"+a.id+"', -"+i+", 'M');\""+' title="'+q+'"><span class="ui-icon ui-icon-circle-triangle-'+(c?"e":"w")+'">'+q+"</span></a>":e?"":'<a class="ui-datepicker-prev ui-corner-all ui-state-disabled" title="'+q+'"><span class="ui-icon ui-icon-circle-triangle-'+(c?"e":"w")+'">'+q+"</span></a>",s=this._get(a,"nextText");s=f?this.formatDate(s,this._daylightSavingAdjust(new Date(o,n+i,1)),this._getFormatConfig(a)):s;var t=this._canAdjustMonth(a,1,o,n)?'<a class="ui-datepicker-next ui-corner-all" onclick="DP_jQuery_'+dpuuid+".datepicker._adjustDate('#"+a.id+"', +"+i+", 'M');\""+' title="'+s+'"><span class="ui-icon ui-icon-circle-triangle-'+(c?"w":"e")+'">'+s+"</span></a>":e?"":'<a class="ui-datepicker-next ui-corner-all ui-state-disabled" title="'+s+'"><span class="ui-icon ui-icon-circle-triangle-'+(c?"w":"e")+'">'+s+"</span></a>",u=this._get(a,"currentText"),v=this._get(a,"gotoCurrent")&&a.currentDay?k:b;u=f?this.formatDate(u,v,this._getFormatConfig(a)):u;var w=a.inline?"":'<button type="button" class="ui-datepicker-close ui-state-default ui-priority-primary ui-corner-all" onclick="DP_jQuery_'+dpuuid+'.datepicker._hideDatepicker();">'+this._get(a,"closeText")+"</button>",x=d?'<div class="ui-datepicker-buttonpane ui-widget-content">'+(c?w:"")+(this._isInRange(a,v)?'<button type="button" class="ui-datepicker-current ui-state-default ui-priority-secondary ui-corner-all" onclick="DP_jQuery_'+dpuuid+".datepicker._gotoToday('#"+a.id+"');\""+">"+u+"</button>":"")+(c?"":w)+"</div>":"",y=parseInt(this._get(a,"firstDay"),10);y=isNaN(y)?0:y;var z=this._get(a,"showWeek"),A=this._get(a,"dayNames"),B=this._get(a,"dayNamesShort"),C=this._get(a,"dayNamesMin"),D=this._get(a,"monthNames"),E=this._get(a,"monthNamesShort"),F=this._get(a,"beforeShowDay"),G=this._get(a,"showOtherMonths"),H=this._get(a,"selectOtherMonths"),I=this._get(a,"calculateWeek")||this.iso8601Week,J=this._getDefaultDate(a),K="";for(var L=0;L<g[0];L++){var M="";this.maxRows=4;for(var N=0;N<g[1];N++){var O=this._daylightSavingAdjust(new Date(o,n,a.selectedDay)),P=" ui-corner-all",Q="";if(j){Q+='<div class="ui-datepicker-group';if(g[1]>1)switch(N){case 0:Q+=" ui-datepicker-group-first",P=" ui-corner-"+(c?"right":"left");break;case g[1]-1:Q+=" ui-datepicker-group-last",P=" ui-corner-"+(c?"left":"right");break;default:Q+=" ui-datepicker-group-middle",P=""}Q+='">'}Q+='<div class="ui-datepicker-header ui-widget-header ui-helper-clearfix'+P+'">'+(/all|left/.test(P)&&L==0?c?t:r:"")+(/all|right/.test(P)&&L==0?c?r:t:"")+this._generateMonthYearHeader(a,n,o,l,m,L>0||N>0,D,E)+'</div><table class="ui-datepicker-calendar"><thead>'+"<tr>";var R=z?'<th class="ui-datepicker-week-col">'+this._get(a,"weekHeader")+"</th>":"";for(var S=0;S<7;S++){var T=(S+y)%7;R+="<th"+((S+y+6)%7>=5?' class="ui-datepicker-week-end"':"")+">"+'<span title="'+A[T]+'">'+C[T]+"</span></th>"}Q+=R+"</tr></thead><tbody>";var U=this._getDaysInMonth(o,n);o==a.selectedYear&&n==a.selectedMonth&&(a.selectedDay=Math.min(a.selectedDay,U));var V=(this._getFirstDayOfMonth(o,n)-y+7)%7,W=Math.ceil((V+U)/7),X=j?this.maxRows>W?this.maxRows:W:W;this.maxRows=X;var Y=this._daylightSavingAdjust(new Date(o,n,1-V));for(var Z=0;Z<X;Z++){Q+="<tr>";var _=z?'<td class="ui-datepicker-week-col">'+this._get(a,"calculateWeek")(Y)+"</td>":"";for(var S=0;S<7;S++){var ba=F?F.apply(a.input?a.input[0]:null,[Y]):[!0,""],bb=Y.getMonth()!=n,bc=bb&&!H||!ba[0]||l&&Y<l||m&&Y>m;_+='<td class="'+((S+y+6)%7>=5?" ui-datepicker-week-end":"")+(bb?" ui-datepicker-other-month":"")+(Y.getTime()==O.getTime()&&n==a.selectedMonth&&a._keyEvent||J.getTime()==Y.getTime()&&J.getTime()==O.getTime()?" "+this._dayOverClass:"")+(bc?" "+this._unselectableClass+" ui-state-disabled":"")+(bb&&!G?"":" "+ba[1]+(Y.getTime()==k.getTime()?" "+this._currentClass:"")+(Y.getTime()==b.getTime()?" ui-datepicker-today":""))+'"'+((!bb||G)&&ba[2]?' title="'+ba[2]+'"':"")+(bc?"":' onclick="DP_jQuery_'+dpuuid+".datepicker._selectDay('#"+a.id+"',"+Y.getMonth()+","+Y.getFullYear()+', this);return false;"')+">"+(bb&&!G?" ":bc?'<span class="ui-state-default">'+Y.getDate()+"</span>":'<a class="ui-state-default'+(Y.getTime()==b.getTime()?" ui-state-highlight":"")+(Y.getTime()==k.getTime()?" ui-state-active":"")+(bb?" ui-priority-secondary":"")+'" href="#">'+Y.getDate()+"</a>")+"</td>",Y.setDate(Y.getDate()+1),Y=this._daylightSavingAdjust(Y)}Q+=_+"</tr>"}n++,n>11&&(n=0,o++),Q+="</tbody></table>"+(j?"</div>"+(g[0]>0&&N==g[1]-1?'<div class="ui-datepicker-row-break"></div>':""):""),M+=Q}K+=M}K+=x+($.browser.msie&&parseInt($.browser.version,10)<7&&!a.inline?'<iframe src="javascript:false;" class="ui-datepicker-cover" frameborder="0"></iframe>':""),a._keyEvent=!1;return K},_generateMonthYearHeader:function(a,b,c,d,e,f,g,h){var i=this._get(a,"changeMonth"),j=this._get(a,"changeYear"),k=this
+._get(a,"showMonthAfterYear"),l='<div class="ui-datepicker-title">',m="";if(f||!i)m+='<span class="ui-datepicker-month">'+g[b]+"</span>";else{var n=d&&d.getFullYear()==c,o=e&&e.getFullYear()==c;m+='<select class="ui-datepicker-month" onchange="DP_jQuery_'+dpuuid+".datepicker._selectMonthYear('#"+a.id+"', this, 'M');\" "+">";for(var p=0;p<12;p++)(!n||p>=d.getMonth())&&(!o||p<=e.getMonth())&&(m+='<option value="'+p+'"'+(p==b?' selected="selected"':"")+">"+h[p]+"</option>");m+="</select>"}k||(l+=m+(f||!i||!j?" ":""));if(!a.yearshtml){a.yearshtml="";if(f||!j)l+='<span class="ui-datepicker-year">'+c+"</span>";else{var q=this._get(a,"yearRange").split(":"),r=(new Date).getFullYear(),s=function(a){var b=a.match(/c[+-].*/)?c+parseInt(a.substring(1),10):a.match(/[+-].*/)?r+parseInt(a,10):parseInt(a,10);return isNaN(b)?r:b},t=s(q[0]),u=Math.max(t,s(q[1]||""));t=d?Math.max(t,d.getFullYear()):t,u=e?Math.min(u,e.getFullYear()):u,a.yearshtml+='<select class="ui-datepicker-year" onchange="DP_jQuery_'+dpuuid+".datepicker._selectMonthYear('#"+a.id+"', this, 'Y');\" "+">";for(;t<=u;t++)a.yearshtml+='<option value="'+t+'"'+(t==c?' selected="selected"':"")+">"+t+"</option>";a.yearshtml+="</select>",l+=a.yearshtml,a.yearshtml=null}}l+=this._get(a,"yearSuffix"),k&&(l+=(f||!i||!j?" ":"")+m),l+="</div>";return l},_adjustInstDate:function(a,b,c){var d=a.drawYear+(c=="Y"?b:0),e=a.drawMonth+(c=="M"?b:0),f=Math.min(a.selectedDay,this._getDaysInMonth(d,e))+(c=="D"?b:0),g=this._restrictMinMax(a,this._daylightSavingAdjust(new Date(d,e,f)));a.selectedDay=g.getDate(),a.drawMonth=a.selectedMonth=g.getMonth(),a.drawYear=a.selectedYear=g.getFullYear(),(c=="M"||c=="Y")&&this._notifyChange(a)},_restrictMinMax:function(a,b){var c=this._getMinMaxDate(a,"min"),d=this._getMinMaxDate(a,"max"),e=c&&b<c?c:b;e=d&&e>d?d:e;return e},_notifyChange:function(a){var b=this._get(a,"onChangeMonthYear");b&&b.apply(a.input?a.input[0]:null,[a.selectedYear,a.selectedMonth+1,a])},_getNumberOfMonths:function(a){var b=this._get(a,"numberOfMonths");return b==null?[1,1]:typeof b=="number"?[1,b]:b},_getMinMaxDate:function(a,b){return this._determineDate(a,this._get(a,b+"Date"),null)},_getDaysInMonth:function(a,b){return 32-this._daylightSavingAdjust(new Date(a,b,32)).getDate()},_getFirstDayOfMonth:function(a,b){return(new Date(a,b,1)).getDay()},_canAdjustMonth:function(a,b,c,d){var e=this._getNumberOfMonths(a),f=this._daylightSavingAdjust(new Date(c,d+(b<0?b:e[0]*e[1]),1));b<0&&f.setDate(this._getDaysInMonth(f.getFullYear(),f.getMonth()));return this._isInRange(a,f)},_isInRange:function(a,b){var c=this._getMinMaxDate(a,"min"),d=this._getMinMaxDate(a,"max");return(!c||b.getTime()>=c.getTime())&&(!d||b.getTime()<=d.getTime())},_getFormatConfig:function(a){var b=this._get(a,"shortYearCutoff");b=typeof b!="string"?b:(new Date).getFullYear()%100+parseInt(b,10);return{shortYearCutoff:b,dayNamesShort:this._get(a,"dayNamesShort"),dayNames:this._get(a,"dayNames"),monthNamesShort:this._get(a,"monthNamesShort"),monthNames:this._get(a,"monthNames")}},_formatDate:function(a,b,c,d){b||(a.currentDay=a.selectedDay,a.currentMonth=a.selectedMonth,a.currentYear=a.selectedYear);var e=b?typeof b=="object"?b:this._daylightSavingAdjust(new Date(d,c,b)):this._daylightSavingAdjust(new Date(a.currentYear,a.currentMonth,a.currentDay));return this.formatDate(this._get(a,"dateFormat"),e,this._getFormatConfig(a))}}),$.fn.datepicker=function(a){if(!this.length)return this;$.datepicker.initialized||($(document).mousedown($.datepicker._checkExternalClick).find("body").append($.datepicker.dpDiv),$.datepicker.initialized=!0);var b=Array.prototype.slice.call(arguments,1);if(typeof a=="string"&&(a=="isDisabled"||a=="getDate"||a=="widget"))return $.datepicker["_"+a+"Datepicker"].apply($.datepicker,[this[0]].concat(b));if(a=="option"&&arguments.length==2&&typeof arguments[1]=="string")return $.datepicker["_"+a+"Datepicker"].apply($.datepicker,[this[0]].concat(b));return this.each(function(){typeof a=="string"?$.datepicker["_"+a+"Datepicker"].apply($.datepicker,[this].concat(b)):$.datepicker._attachDatepicker(this,a)})},$.datepicker=new Datepicker,$.datepicker.initialized=!1,$.datepicker.uuid=(new Date).getTime(),$.datepicker.version="1.8.17",window["DP_jQuery_"+dpuuid]=$})(jQuery);/*
+ * jQuery UI Progressbar 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Progressbar
+ *
+ * Depends:
+ * jquery.ui.core.js
+ * jquery.ui.widget.js
+ */(function(a,b){a.widget("ui.progressbar",{options:{value:0,max:100},min:0,_create:function(){this.element.addClass("ui-progressbar ui-widget ui-widget-content ui-corner-all").attr({role:"progressbar","aria-valuemin":this.min,"aria-valuemax":this.options.max,"aria-valuenow":this._value()}),this.valueDiv=a("<div class='ui-progressbar-value ui-widget-header ui-corner-left'></div>").appendTo(this.element),this.oldValue=this._value(),this._refreshValue()},destroy:function(){this.element.removeClass("ui-progressbar ui-widget ui-widget-content ui-corner-all").removeAttr("role").removeAttr("aria-valuemin").removeAttr("aria-valuemax").removeAttr("aria-valuenow"),this.valueDiv.remove(),a.Widget.prototype.destroy.apply(this,arguments)},value:function(a){if(a===b)return this._value();this._setOption("value",a);return this},_setOption:function(b,c){b==="value"&&(this.options.value=c,this._refreshValue(),this._value()===this.options.max&&this._trigger("complete")),a.Widget.prototype._setOption.apply(this,arguments)},_value:function(){var a=this.options.value;typeof a!="number"&&(a=0);return Math.min(this.options.max,Math.max(this.min,a))},_percentage:function(){return 100*this._value()/this.options.max},_refreshValue:function(){var a=this.value(),b=this._percentage();this.oldValue!==a&&(this.oldValue=a,this._trigger("change")),this.valueDiv.toggle(a>this.min).toggleClass("ui-corner-right",a===this.options.max).width(b.toFixed(0)+"%"),this.element.attr("aria-valuenow",a)}}),a.extend(a.ui.progressbar,{version:"1.8.17"})})(jQuery);/*
+ * jQuery UI Effects 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/
+ */jQuery.effects||function(a,b){function l(b){if(!b||typeof b=="number"||a.fx.speeds[b])return!0;if(typeof b=="string"&&!a.effects[b])return!0;return!1}function k(b,c,d,e){typeof b=="object"&&(e=c,d=null,c=b,b=c.effect),a.isFunction(c)&&(e=c,d=null,c={});if(typeof c=="number"||a.fx.speeds[c])e=d,d=c,c={};a.isFunction(d)&&(e=d,d=null),c=c||{},d=d||c.duration,d=a.fx.off?0:typeof d=="number"?d:d in a.fx.speeds?a.fx.speeds[d]:a.fx.speeds._default,e=e||c.complete;return[b,c,d,e]}function j(a,b){var c={_:0},d;for(d in b)a[d]!=b[d]&&(c[d]=b[d]);return c}function i(b){var c,d;for(c in b)d=b[c],(d==null||a.isFunction(d)||c in g||/scrollbar/.test(c)||!/color/i.test(c)&&isNaN(parseFloat(d)))&&delete b[c];return b}function h(){var a=document.defaultView?document.defaultView.getComputedStyle(this,null):this.currentStyle,b={},c,d;if(a&&a.length&&a[0]&&a[a[0]]){var e=a.length;while(e--)c=a[e],typeof a[c]=="string"&&(d=c.replace(/\-(\w)/g,function(a,b){return b.toUpperCase()}),b[d]=a[c])}else for(c in a)typeof a[c]=="string"&&(b[c]=a[c]);return b}function d(b,d){var e;do{e=a.curCSS(b,d);if(e!=""&&e!="transparent"||a.nodeName(b,"body"))break;d="backgroundColor"}while(b=b.parentNode);return c(e)}function c(b){var c;if(b&&b.constructor==Array&&b.length==3)return b;if(c=/rgb\(\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*,\s*([0-9]{1,3})\s*\)/.exec(b))return[parseInt(c[1],10),parseInt(c[2],10),parseInt(c[3],10)];if(c=/rgb\(\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*,\s*([0-9]+(?:\.[0-9]+)?)\%\s*\)/.exec(b))return[parseFloat(c[1])*2.55,parseFloat(c[2])*2.55,parseFloat(c[3])*2.55];if(c=/#([a-fA-F0-9]{2})([a-fA-F0-9]{2})([a-fA-F0-9]{2})/.exec(b))return[parseInt(c[1],16),parseInt(c[2],16),parseInt(c[3],16)];if(c=/#([a-fA-F0-9])([a-fA-F0-9])([a-fA-F0-9])/.exec(b))return[parseInt(c[1]+c[1],16),parseInt(c[2]+c[2],16),parseInt(c[3]+c[3],16)];if(c=/rgba\(0, 0, 0, 0\)/.exec(b))return e.transparent;return e[a.trim(b).toLowerCase()]}a.effects={},a.each(["backgroundColor","borderBottomColor","borderLeftColor","borderRightColor","borderTopColor","borderColor","color","outlineColor"],function(b,e){a.fx.step[e]=function(a){a.colorInit||(a.start=d(a.elem,e),a.end=c(a.end),a.colorInit=!0),a.elem.style[e]="rgb("+Math.max(Math.min(parseInt(a.pos*(a.end[0]-a.start[0])+a.start[0],10),255),0)+","+Math.max(Math.min(parseInt(a.pos*(a.end[1]-a.start[1])+a.start[1],10),255),0)+","+Math.max(Math.min(parseInt(a.pos*(a.end[2]-a.start[2])+a.start[2],10),255),0)+")"}});var e={aqua:[0,255,255],azure:[240,255,255],beige:[245,245,220],black:[0,0,0],blue:[0,0,255],brown:[165,42,42],cyan:[0,255,255],darkblue:[0,0,139],darkcyan:[0,139,139],darkgrey:[169,169,169],darkgreen:[0,100,0],darkkhaki:[189,183,107],darkmagenta:[139,0,139],darkolivegreen:[85,107,47],darkorange:[255,140,0],darkorchid:[153,50,204],darkred:[139,0,0],darksalmon:[233,150,122],darkviolet:[148,0,211],fuchsia:[255,0,255],gold:[255,215,0],green:[0,128,0],indigo:[75,0,130],khaki:[240,230,140],lightblue:[173,216,230],lightcyan:[224,255,255],lightgreen:[144,238,144],lightgrey:[211,211,211],lightpink:[255,182,193],lightyellow:[255,255,224],lime:[0,255,0],magenta:[255,0,255],maroon:[128,0,0],navy:[0,0,128],olive:[128,128,0],orange:[255,165,0],pink:[255,192,203],purple:[128,0,128],violet:[128,0,128],red:[255,0,0],silver:[192,192,192],white:[255,255,255],yellow:[255,255,0],transparent:[255,255,255]},f=["add","remove","toggle"],g={border:1,borderBottom:1,borderColor:1,borderLeft:1,borderRight:1,borderTop:1,borderWidth:1,margin:1,padding:1};a.effects.animateClass=function(b,c,d,e){a.isFunction(d)&&(e=d,d=null);return this.queue(function(){var g=a(this),k=g.attr("style")||" ",l=i(h.call(this)),m,n=g.attr("class");a.each(f,function(a,c){b[c]&&g[c+"Class"](b[c])}),m=i(h.call(this)),g.attr("class",n),g.animate(j(l,m),{queue:!1,duration:c,easing:d,complete:function(){a.each(f,function(a,c){b[c]&&g[c+"Class"](b[c])}),typeof g.attr("style")=="object"?(g.attr("style").cssText="",g.attr("style").cssText=k):g.attr("style",k),e&&e.apply(this,arguments),a.dequeue(this)}})})},a.fn.extend({_addClass:a.fn.addClass,addClass:function(b,c,d,e){return c?a.effects.animateClass.apply(this,[{add:b},c,d,e]):this._addClass(b)},_removeClass:a.fn.removeClass,removeClass:function(b,c,d,e){return c?a.effects.animateClass.apply(this,[{remove:b},c,d,e]):this._removeClass(b)},_toggleClass:a.fn.toggleClass,toggleClass:function(c,d,e,f,g){return typeof d=="boolean"||d===b?e?a.effects.animateClass.apply(this,[d?{add:c}:{remove:c},e,f,g]):this._toggleClass(c,d):a.effects.animateClass.apply(this,[{toggle:c},d,e,f])},switchClass:function(b,c,d,e,f){return a.effects.animateClass.apply(this,[{add:c,remove:b},d,e,f])}}),a.extend(a.effects,{version:"1.8.17",save:function(a,b){for(var c=0;c<b.length;c++)b[c]!==null&&a.data("ec.storage."+b[c],a[0].style[b[c]])},restore:function(a,b){for(var c=0;c<b.length;c++)b[c]!==null&&a.css(b[c],a.data("ec.storage."+b[c]))},setMode:function(a,b){b=="toggle"&&(b=a.is(":hidden")?"show":"hide");return b},getBaseline:function(a,b){var c,d;switch(a[0]){case"top":c=0;break;case"middle":c=.5;break;case"bottom":c=1;break;default:c=a[0]/b.height}switch(a[1]){case"left":d=0;break;case"center":d=.5;break;case"right":d=1;break;default:d=a[1]/b.width}return{x:d,y:c}},createWrapper:function(b){if(b.parent().is(".ui-effects-wrapper"))return b.parent();var c={width:b.outerWidth(!0),height:b.outerHeight(!0),"float":b.css("float")},d=a("<div></div>").addClass("ui-effects-wrapper").css({fontSize:"100%",background:"transparent",border:"none",margin:0,padding:0}),e=document.activeElement;b.wrap(d),(b[0]===e||a.contains(b[0],e))&&a(e).focus(),d=b.parent(),b.css("position")=="static"?(d.css({position:"relative"}),b.css({position:"relative"})):(a.extend(c,{position:b.css("position"),zIndex:b.css("z-index")}),a.each(["top","left","bottom","right"],function(a,d){c[d]=b.css(d),isNaN(parseInt(c[d],10))&&(c[d]="auto")}),b.css({position:"relative",top:0,left:0,right:"auto",bottom:"auto"}));return d.css(c).show()},removeWrapper:function(b){var c,d=document.activeElement;if(b.parent().is(".ui-effects-wrapper")){c=b.parent().replaceWith(b),(b[0]===d||a.contains(b[0],d))&&a(d).focus();return c}return b},setTransition:function(b,c,d,e){e=e||{},a.each(c,function(a,c){unit=b.cssUnit(c),unit[0]>0&&(e[c]=unit[0]*d+unit[1])});return e}}),a.fn.extend({effect:function(b,c,d,e){var f=k.apply(this,arguments),g={options:f[1],duration:f[2],callback:f[3]},h=g.options.mode,i=a.effects[b];if(a.fx.off||!i)return h?this[h](g.duration,g.callback):this.each(function(){g.callback&&g.callback.call(this)});return i.call(this,g)},_show:a.fn.show,show:function(a){if(l(a))return this._show.apply(this,arguments);var b=k.apply(this,arguments);b[1].mode="show";return this.effect.apply(this,b)},_hide:a.fn.hide,hide:function(a){if(l(a))return this._hide.apply(this,arguments);var b=k.apply(this,arguments);b[1].mode="hide";return this.effect.apply(this,b)},__toggle:a.fn.toggle,toggle:function(b){if(l(b)||typeof b=="boolean"||a.isFunction(b))return this.__toggle.apply(this,arguments);var c=k.apply(this,arguments);c[1].mode="toggle";return this.effect.apply(this,c)},cssUnit:function(b){var c=this.css(b),d=[];a.each(["em","px","%","pt"],function(a,b){c.indexOf(b)>0&&(d=[parseFloat(c),b])});return d}}),a.easing.jswing=a.easing.swing,a.extend(a.easing,{def:"easeOutQuad",swing:function(b,c,d,e,f){return a.easing[a.easing.def](b,c,d,e,f)},easeInQuad:function(a,b,c,d,e){return d*(b/=e)*b+c},easeOutQuad:function(a,b,c,d,e){return-d*(b/=e)*(b-2)+c},easeInOutQuad:function(a,b,c,d,e){if((b/=e/2)<1)return d/2*b*b+c;return-d/2*(--b*(b-2)-1)+c},easeInCubic:function(a,b,c,d,e){return d*(b/=e)*b*b+c},easeOutCubic:function(a,b,c,d,e){return d*((b=b/e-1)*b*b+1)+c},easeInOutCubic:function(a,b,c,d,e){if((b/=e/2)<1)return d/2*b*b*b+c;return d/2*((b-=2)*b*b+2)+c},easeInQuart:function(a,b,c,d,e){return d*(b/=e)*b*b*b+c},easeOutQuart:function(a,b,c,d,e){return-d*((b=b/e-1)*b*b*b-1)+c},easeInOutQuart:function(a,b,c,d,e){if((b/=e/2)<1)return d/2*b*b*b*b+c;return-d/2*((b-=2)*b*b*b-2)+c},easeInQuint:function(a,b,c,d,e){return d*(b/=e)*b*b*b*b+c},easeOutQuint:function(a,b,c,d,e){return d*((b=b/e-1)*b*b*b*b+1)+c},easeInOutQuint:function(a,b,c,d,e){if((b/=e/2)<1)return d/2*b*b*b*b*b+c;return d/2*((b-=2)*b*b*b*b+2)+c},easeInSine:function(a,b,c,d,e){return-d*Math.cos(b/e*(Math.PI/2))+d+c},easeOutSine:function(a,b,c,d,e){return d*Math.sin(b/e*(Math.PI/2))+c},easeInOutSine:function(a,b,c,d,e){return-d/2*(Math.cos(Math.PI*b/e)-1)+c},easeInExpo:function(a,b,c,d,e){return b==0?c:d*Math.pow(2,10*(b/e-1))+c},easeOutExpo:function(a,b,c,d,e){return b==e?c+d:d*(-Math.pow(2,-10*b/e)+1)+c},easeInOutExpo:function(a,b,c,d,e){if(b==0)return c;if(b==e)return c+d;if((b/=e/2)<1)return d/2*Math.pow(2,10*(b-1))+c;return d/2*(-Math.pow(2,-10*--b)+2)+c},easeInCirc:function(a,b,c,d,e){return-d*(Math.sqrt(1-(b/=e)*b)-1)+c},easeOutCirc:function(a,b,c,d,e){return d*Math.sqrt(1-(b=b/e-1)*b)+c},easeInOutCirc:function(a,b,c,d,e){if((b/=e/2)<1)return-d/2*(Math.sqrt(1-b*b)-1)+c;return d/2*(Math.sqrt(1-(b-=2)*b)+1)+c},easeInElastic:function(a,b,c,d,e){var f=1.70158,g=0,h=d;if(b==0)return c;if((b/=e)==1)return c+d;g||(g=e*.3);if(h<Math.abs(d)){h=d;var f=g/4}else var f=g/(2*Math.PI)*Math.asin(d/h);return-(h*Math.pow(2,10*(b-=1))*Math.sin((b*e-f)*2*Math.PI/g))+c},easeOutElastic:function(a,b,c,d,e){var f=1.70158,g=0,h=d;if(b==0)return c;if((b/=e)==1)return c+d;g||(g=e*.3);if(h<Math.abs(d)){h=d;var f=g/4}else var f=g/(2*Math.PI)*Math.asin(d/h);return h*Math.pow(2,-10*b)*Math.sin((b*e-f)*2*Math.PI/g)+d+c},easeInOutElastic:function(a,b,c,d,e){var f=1.70158,g=0,h=d;if(b==0)return c;if((b/=e/2)==2)return c+d;g||(g=e*.3*1.5);if(h<Math.abs(d)){h=d;var f=g/4}else var f=g/(2*Math.PI)*Math.asin(d/h);if(b<1)return-0.5*h*Math.pow(2,10*(b-=1))*Math.sin((b*e-f)*2*Math.PI/g)+c;return h*Math.pow(2,-10*(b-=1))*Math.sin((b*e-f)*2*Math.PI/g)*.5+d+c},easeInBack:function(a,c,d,e,f,g){g==b&&(g=1.70158);return e*(c/=f)*c*((g+1)*c-g)+d},easeOutBack:function(a,c,d,e,f,g){g==b&&(g=1.70158);return e*((c=c/f-1)*c*((g+1)*c+g)+1)+d},easeInOutBack:function(a,c,d,e,f,g){g==b&&(g=1.70158);if((c/=f/2)<1)return e/2*c*c*(((g*=1.525)+1)*c-g)+d;return e/2*((c-=2)*c*(((g*=1.525)+1)*c+g)+2)+d},easeInBounce:function(b,c,d,e,f){return e-a.easing.easeOutBounce(b,f-c,0,e,f)+d},easeOutBounce:function(a,b,c,d,e){return(b/=e)<1/2.75?d*7.5625*b*b+c:b<2/2.75?d*(7.5625*(b-=1.5/2.75)*b+.75)+c:b<2.5/2.75?d*(7.5625*(b-=2.25/2.75)*b+.9375)+c:d*(7.5625*(b-=2.625/2.75)*b+.984375)+c},easeInOutBounce:function(b,c,d,e,f){if(c<f/2)return a.easing.easeInBounce(b,c*2,0,e,f)*.5+d;return a.easing.easeOutBounce(b,c*2-f,0,e,f)*.5+e*.5+d}})}(jQuery);/*
+ * jQuery UI Effects Blind 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Blind
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.blind=function(b){return this.queue(function(){var c=a(this),d=["position","top","bottom","left","right"],e=a.effects.setMode(c,b.options.mode||"hide"),f=b.options.direction||"vertical";a.effects.save(c,d),c.show();var g=a.effects.createWrapper(c).css({overflow:"hidden"}),h=f=="vertical"?"height":"width",i=f=="vertical"?g.height():g.width();e=="show"&&g.css(h,0);var j={};j[h]=e=="show"?i:0,g.animate(j,b.duration,b.options.easing,function(){e=="hide"&&c.hide(),a.effects.restore(c,d),a.effects.removeWrapper(c),b.callback&&b.callback.apply(c[0],arguments),c.dequeue()})})}})(jQuery);/*
+ * jQuery UI Effects Bounce 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Bounce
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.bounce=function(b){return this.queue(function(){var c=a(this),d=["position","top","bottom","left","right"],e=a.effects.setMode(c,b.options.mode||"effect"),f=b.options.direction||"up",g=b.options.distance||20,h=b.options.times||5,i=b.duration||250;/show|hide/.test(e)&&d.push("opacity"),a.effects.save(c,d),c.show(),a.effects.createWrapper(c);var j=f=="up"||f=="down"?"top":"left",k=f=="up"||f=="left"?"pos":"neg",g=b.options.distance||(j=="top"?c.outerHeight({margin:!0})/3:c.outerWidth({margin:!0})/3);e=="show"&&c.css("opacity",0).css(j,k=="pos"?-g:g),e=="hide"&&(g=g/(h*2)),e!="hide"&&h--;if(e=="show"){var l={opacity:1};l[j]=(k=="pos"?"+=":"-=")+g,c.animate(l,i/2,b.options.easing),g=g/2,h--}for(var m=0;m<h;m++){var n={},p={};n[j]=(k=="pos"?"-=":"+=")+g,p[j]=(k=="pos"?"+=":"-=")+g,c.animate(n,i/2,b.options.easing).animate(p,i/2,b.options.easing),g=e=="hide"?g*2:g/2}if(e=="hide"){var l={opacity:0};l[j]=(k=="pos"?"-=":"+=")+g,c.animate(l,i/2,b.options.easing,function(){c.hide(),a.effects.restore(c,d),a.effects.removeWrapper(c),b.callback&&b.callback.apply(this,arguments)})}else{var n={},p={};n[j]=(k=="pos"?"-=":"+=")+g,p[j]=(k=="pos"?"+=":"-=")+g,c.animate(n,i/2,b.options.easing).animate(p,i/2,b.options.easing,function(){a.effects.restore(c,d),a.effects.removeWrapper(c),b.callback&&b.callback.apply(this,arguments)})}c.queue("fx",function(){c.dequeue()}),c.dequeue()})}})(jQuery);/*
+ * jQuery UI Effects Clip 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Clip
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.clip=function(b){return this.queue(function(){var c=a(this),d=["position","top","bottom","left","right","height","width"],e=a.effects.setMode(c,b.options.mode||"hide"),f=b.options.direction||"vertical";a.effects.save(c,d),c.show();var g=a.effects.createWrapper(c).css({overflow:"hidden"}),h=c[0].tagName=="IMG"?g:c,i={size:f=="vertical"?"height":"width",position:f=="vertical"?"top":"left"},j=f=="vertical"?h.height():h.width();e=="show"&&(h.css(i.size,0),h.css(i.position,j/2));var k={};k[i.size]=e=="show"?j:0,k[i.position]=e=="show"?0:j/2,h.animate(k,{queue:!1,duration:b.duration,easing:b.options.easing,complete:function(){e=="hide"&&c.hide(),a.effects.restore(c,d),a.effects.removeWrapper(c),b.callback&&b.callback.apply(c[0],arguments),c.dequeue()}})})}})(jQuery);/*
+ * jQuery UI Effects Drop 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Drop
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.drop=function(b){return this.queue(function(){var c=a(this),d=["position","top","bottom","left","right","opacity"],e=a.effects.setMode(c,b.options.mode||"hide"),f=b.options.direction||"left";a.effects.save(c,d),c.show(),a.effects.createWrapper(c);var g=f=="up"||f=="down"?"top":"left",h=f=="up"||f=="left"?"pos":"neg",i=b.options.distance||(g=="top"?c.outerHeight({margin:!0})/2:c.outerWidth({margin:!0})/2);e=="show"&&c.css("opacity",0).css(g,h=="pos"?-i:i);var j={opacity:e=="show"?1:0};j[g]=(e=="show"?h=="pos"?"+=":"-=":h=="pos"?"-=":"+=")+i,c.animate(j,{queue:!1,duration:b.duration,easing:b.options.easing,complete:function(){e=="hide"&&c.hide(),a.effects.restore(c,d),a.effects.removeWrapper(c),b.callback&&b.callback.apply(this,arguments),c.dequeue()}})})}})(jQuery);/*
+ * jQuery UI Effects Explode 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Explode
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.explode=function(b){return this.queue(function(){var c=b.options.pieces?Math.round(Math.sqrt(b.options.pieces)):3,d=b.options.pieces?Math.round(Math.sqrt(b.options.pieces)):3;b.options.mode=b.options.mode=="toggle"?a(this).is(":visible")?"hide":"show":b.options.mode;var e=a(this).show().css("visibility","hidden"),f=e.offset();f.top-=parseInt(e.css("marginTop"),10)||0,f.left-=parseInt(e.css("marginLeft"),10)||0;var g=e.outerWidth(!0),h=e.outerHeight(!0);for(var i=0;i<c;i++)for(var j=0;j<d;j++)e.clone().appendTo("body").wrap("<div></div>").css({position:"absolute",visibility:"visible",left:-j*(g/d),top:-i*(h/c)}).parent().addClass("ui-effects-explode").css({position:"absolute",overflow:"hidden",width:g/d,height:h/c,left:f.left+j*(g/d)+(b.options.mode=="show"?(j-Math.floor(d/2))*(g/d):0),top:f.top+i*(h/c)+(b.options.mode=="show"?(i-Math.floor(c/2))*(h/c):0),opacity:b.options.mode=="show"?0:1}).animate({left:f.left+j*(g/d)+(b.options.mode=="show"?0:(j-Math.floor(d/2))*(g/d)),top:f.top+i*(h/c)+(b.options.mode=="show"?0:(i-Math.floor(c/2))*(h/c)),opacity:b.options.mode=="show"?1:0},b.duration||500);setTimeout(function(){b.options.mode=="show"?e.css({visibility:"visible"}):e.css({visibility:"visible"}).hide(),b.callback&&b.callback.apply(e[0]),e.dequeue(),a("div.ui-effects-explode").remove()},b.duration||500)})}})(jQuery);/*
+ * jQuery UI Effects Fade 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Fade
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.fade=function(b){return this.queue(function(){var c=a(this),d=a.effects.setMode(c,b.options.mode||"hide");c.animate({opacity:d},{queue:!1,duration:b.duration,easing:b.options.easing,complete:function(){b.callback&&b.callback.apply(this,arguments),c.dequeue()}})})}})(jQuery);/*
+ * jQuery UI Effects Fold 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Fold
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.fold=function(b){return this.queue(function(){var c=a(this),d=["position","top","bottom","left","right"],e=a.effects.setMode(c,b.options.mode||"hide"),f=b.options.size||15,g=!!b.options.horizFirst,h=b.duration?b.duration/2:a.fx.speeds._default/2;a.effects.save(c,d),c.show();var i=a.effects.createWrapper(c).css({overflow:"hidden"}),j=e=="show"!=g,k=j?["width","height"]:["height","width"],l=j?[i.width(),i.height()]:[i.height(),i.width()],m=/([0-9]+)%/.exec(f);m&&(f=parseInt(m[1],10)/100*l[e=="hide"?0:1]),e=="show"&&i.css(g?{height:0,width:f}:{height:f,width:0});var n={},p={};n[k[0]]=e=="show"?l[0]:f,p[k[1]]=e=="show"?l[1]:0,i.animate(n,h,b.options.easing).animate(p,h,b.options.easing,function(){e=="hide"&&c.hide(),a.effects.restore(c,d),a.effects.removeWrapper(c),b.callback&&b.callback.apply(c[0],arguments),c.dequeue()})})}})(jQuery);/*
+ * jQuery UI Effects Highlight 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Highlight
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.highlight=function(b){return this.queue(function(){var c=a(this),d=["backgroundImage","backgroundColor","opacity"],e=a.effects.setMode(c,b.options.mode||"show"),f={backgroundColor:c.css("backgroundColor")};e=="hide"&&(f.opacity=0),a.effects.save(c,d),c.show().css({backgroundImage:"none",backgroundColor:b.options.color||"#ffff99"}).animate(f,{queue:!1,duration:b.duration,easing:b.options.easing,complete:function(){e=="hide"&&c.hide(),a.effects.restore(c,d),e=="show"&&!a.support.opacity&&this.style.removeAttribute("filter"),b.callback&&b.callback.apply(this,arguments),c.dequeue()}})})}})(jQuery);/*
+ * jQuery UI Effects Pulsate 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Pulsate
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.pulsate=function(b){return this.queue(function(){var c=a(this),d=a.effects.setMode(c,b.options.mode||"show");times=(b.options.times||5)*2-1,duration=b.duration?b.duration/2:a.fx.speeds._default/2,isVisible=c.is(":visible"),animateTo=0,isVisible||(c.css("opacity",0).show(),animateTo=1),(d=="hide"&&isVisible||d=="show"&&!isVisible)&×--;for(var e=0;e<times;e++)c.animate({opacity:animateTo},duration,b.options.easing),animateTo=(animateTo+1)%2;c.animate({opacity:animateTo},duration,b.options.easing,function(){animateTo==0&&c.hide(),b.callback&&b.callback.apply(this,arguments)}),c.queue("fx",function(){c.dequeue()}).dequeue()})}})(jQuery);/*
+ * jQuery UI Effects Scale 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Scale
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.puff=function(b){return this.queue(function(){var c=a(this),d=a.effects.setMode(c,b.options.mode||"hide"),e=parseInt(b.options.percent,10)||150,f=e/100,g={height:c.height(),width:c.width()};a.extend(b.options,{fade:!0,mode:d,percent:d=="hide"?e:100,from:d=="hide"?g:{height:g.height*f,width:g.width*f}}),c.effect("scale",b.options,b.duration,b.callback),c.dequeue()})},a.effects.scale=function(b){return this.queue(function(){var c=a(this),d=a.extend(!0,{},b.options),e=a.effects.setMode(c,b.options.mode||"effect"),f=parseInt(b.options.percent,10)||(parseInt(b.options.percent,10)==0?0:e=="hide"?0:100),g=b.options.direction||"both",h=b.options.origin;e!="effect"&&(d.origin=h||["middle","center"],d.restore=!0);var i={height:c.height(),width:c.width()};c.from=b.options.from||(e=="show"?{height:0,width:0}:i);var j={y:g!="horizontal"?f/100:1,x:g!="vertical"?f/100:1};c.to={height:i.height*j.y,width:i.width*j.x},b.options.fade&&(e=="show"&&(c.from.opacity=0,c.to.opacity=1),e=="hide"&&(c.from.opacity=1,c.to.opacity=0)),d.from=c.from,d.to=c.to,d.mode=e,c.effect("size",d,b.duration,b.callback),c.dequeue()})},a.effects.size=function(b){return this.queue(function(){var c=a(this),d=["position","top","bottom","left","right","width","height","overflow","opacity"],e=["position","top","bottom","left","right","overflow","opacity"],f=["width","height","overflow"],g=["fontSize"],h=["borderTopWidth","borderBottomWidth","paddingTop","paddingBottom"],i=["borderLeftWidth","borderRightWidth","paddingLeft","paddingRight"],j=a.effects.setMode(c,b.options.mode||"effect"),k=b.options.restore||!1,l=b.options.scale||"both",m=b.options.origin,n={height:c.height(),width:c.width()};c.from=b.options.from||n,c.to=b.options.to||n;if(m){var p=a.effects.getBaseline(m,n);c.from.top=(n.height-c.from.height)*p.y,c.from.left=(n.width-c.from.width)*p.x,c.to.top=(n.height-c.to.height)*p.y,c.to.left=(n.width-c.to.width)*p.x}var q={from:{y:c.from.height/n.height,x:c.from.width/n.width},to:{y:c.to.height/n.height,x:c.to.width/n.width}};if(l=="box"||l=="both")q.from.y!=q.to.y&&(d=d.concat(h),c.from=a.effects.setTransition(c,h,q.from.y,c.from),c.to=a.effects.setTransition(c,h,q.to.y,c.to)),q.from.x!=q.to.x&&(d=d.concat(i),c.from=a.effects.setTransition(c,i,q.from.x,c.from),c.to=a.effects.setTransition(c,i,q.to.x,c.to));(l=="content"||l=="both")&&q.from.y!=q.to.y&&(d=d.concat(g),c.from=a.effects.setTransition(c,g,q.from.y,c.from),c.to=a.effects.setTransition(c,g,q.to.y,c.to)),a.effects.save(c,k?d:e),c.show(),a.effects.createWrapper(c),c.css("overflow","hidden").css(c.from);if(l=="content"||l=="both")h=h.concat(["marginTop","marginBottom"]).concat(g),i=i.concat(["marginLeft","marginRight"]),f=d.concat(h).concat(i),c.find("*[width]").each(function(){child=a(this),k&&a.effects.save(child,f);var c={height:child.height(),width:child.width()};child.from={height:c.height*q.from.y,width:c.width*q.from.x},child.to={height:c.height*q.to.y,width:c.width*q.to.x},q.from.y!=q.to.y&&(child.from=a.effects.setTransition(child,h,q.from.y,child.from),child.to=a.effects.setTransition(child,h,q.to.y,child.to)),q.from.x!=q.to.x&&(child.from=a.effects.setTransition(child,i,q.from.x,child.from),child.to=a.effects.setTransition(child,i,q.to.x,child.to)),child.css(child.from),child.animate(child.to,b.duration,b.options.easing,function(){k&&a.effects.restore(child,f)})});c.animate(c.to,{queue:!1,duration:b.duration,easing:b.options.easing,complete:function(){c.to.opacity===0&&c.css("opacity",c.from.opacity),j=="hide"&&c.hide(),a.effects.restore(c,k?d:e),a.effects.removeWrapper(c),b.callback&&b.callback.apply(this,arguments),c.dequeue()}})})}})(jQuery);/*
+ * jQuery UI Effects Shake 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Shake
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.shake=function(b){return this.queue(function(){var c=a(this),d=["position","top","bottom","left","right"],e=a.effects.setMode(c,b.options.mode||"effect"),f=b.options.direction||"left",g=b.options.distance||20,h=b.options.times||3,i=b.duration||b.options.duration||140;a.effects.save(c,d),c.show(),a.effects.createWrapper(c);var j=f=="up"||f=="down"?"top":"left",k=f=="up"||f=="left"?"pos":"neg",l={},m={},n={};l[j]=(k=="pos"?"-=":"+=")+g,m[j]=(k=="pos"?"+=":"-=")+g*2,n[j]=(k=="pos"?"-=":"+=")+g*2,c.animate(l,i,b.options.easing);for(var p=1;p<h;p++)c.animate(m,i,b.options.easing).animate(n,i,b.options.easing);c.animate(m,i,b.options.easing).animate(l,i/2,b.options.easing,function(){a.effects.restore(c,d),a.effects.removeWrapper(c),b.callback&&b.callback.apply(this,arguments)}),c.queue("fx",function(){c.dequeue()}),c.dequeue()})}})(jQuery);/*
+ * jQuery UI Effects Slide 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Slide
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.slide=function(b){return this.queue(function(){var c=a(this),d=["position","top","bottom","left","right"],e=a.effects.setMode(c,b.options.mode||"show"),f=b.options.direction||"left";a.effects.save(c,d),c.show(),a.effects.createWrapper(c).css({overflow:"hidden"});var g=f=="up"||f=="down"?"top":"left",h=f=="up"||f=="left"?"pos":"neg",i=b.options.distance||(g=="top"?c.outerHeight({margin:!0}):c.outerWidth({margin:!0}));e=="show"&&c.css(g,h=="pos"?isNaN(i)?"-"+i:-i:i);var j={};j[g]=(e=="show"?h=="pos"?"+=":"-=":h=="pos"?"-=":"+=")+i,c.animate(j,{queue:!1,duration:b.duration,easing:b.options.easing,complete:function(){e=="hide"&&c.hide(),a.effects.restore(c,d),a.effects.removeWrapper(c),b.callback&&b.callback.apply(this,arguments),c.dequeue()}})})}})(jQuery);/*
+ * jQuery UI Effects Transfer 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Effects/Transfer
+ *
+ * Depends:
+ * jquery.effects.core.js
+ */(function(a,b){a.effects.transfer=function(b){return this.queue(function(){var c=a(this),d=a(b.options.to),e=d.offset(),f={top:e.top,left:e.left,height:d.innerHeight(),width:d.innerWidth()},g=c.offset(),h=a('<div class="ui-effects-transfer"></div>').appendTo(document.body).addClass(b.options.className).css({top:g.top,left:g.left,height:c.innerHeight(),width:c.innerWidth(),position:"absolute"}).animate(f,b.duration,b.options.easing,function(){h.remove(),b.callback&&b.callback.apply(c[0],arguments),c.dequeue()})})}})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/jquery/plugins/jquery.getParams.js b/hyracks-control-cc/src/main/resources/static/javascript/jquery/plugins/jquery.getParams.js
new file mode 100644
index 0000000..16e8c7a
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/jquery/plugins/jquery.getParams.js
@@ -0,0 +1,39 @@
+/* Copyright (c) 2006 Mathias Bank (http://www.mathias-bank.de)
+ * Dual licensed under the MIT (http://www.opensource.org/licenses/mit-license.php)
+ * and GPL (http://www.opensource.org/licenses/gpl-license.php) licenses.
+ *
+ * Thanks to Hinnerk Ruemenapf - http://hinnerk.ruemenapf.de/ for bug reporting and fixing.
+ */
+jQuery.extend({
+/**
+* Returns get parameters.
+*
+* If the desired param does not exist, null will be returned
+*
+* @example value = $.getURLParam("paramName");
+*/
+ getURLParam: function(strParamName){
+ var strReturn = "";
+ var strHref = window.location.href;
+ var bFound=false;
+
+ var cmpstring = strParamName + "=";
+ var cmplen = cmpstring.length;
+
+ if ( strHref.indexOf("?") > -1 ){
+ var strQueryString = strHref.substr(strHref.indexOf("?")+1);
+ var aQueryString = strQueryString.split("&");
+ for ( var iParam = 0; iParam < aQueryString.length; iParam++ ){
+ if (aQueryString[iParam].substr(0,cmplen)==cmpstring){
+ var aParam = aQueryString[iParam].split("=");
+ strReturn = aParam[1];
+ bFound=true;
+ break;
+ }
+
+ }
+ }
+ if (bFound==false) return null;
+ return strReturn;
+ }
+});
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/jquery/plugins/jquery.timer.js b/hyracks-control-cc/src/main/resources/static/javascript/jquery/plugins/jquery.timer.js
new file mode 100644
index 0000000..55aedf4
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/jquery/plugins/jquery.timer.js
@@ -0,0 +1,111 @@
+/**
+ * jquery.timer.js
+ *
+ * Copyright (c) 2011 Jason Chavannes <jason.chavannes@gmail.com>
+ *
+ * http://jchavannes.com/jquery-timer
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use, copy,
+ * modify, merge, publish, distribute, sublicense, and/or sell copies
+ * of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+;(function($) {
+ $.timer = function(func, time, autostart) {
+ this.set = function(func, time, autostart) {
+ this.init = true;
+ if(typeof func == 'object') {
+ var paramList = ['autostart', 'time'];
+ for(var arg in paramList) {if(func[paramList[arg]] != undefined) {eval(paramList[arg] + " = func[paramList[arg]]");}};
+ func = func.action;
+ }
+ if(typeof func == 'function') {this.action = func;}
+ if(!isNaN(time)) {this.intervalTime = time;}
+ if(autostart && !this.active) {
+ this.active = true;
+ this.setTimer();
+ }
+ return this;
+ };
+ this.once = function(time) {
+ var timer = this;
+ if(isNaN(time)) {time = 0;}
+ window.setTimeout(function() {timer.action();}, time);
+ return this;
+ };
+ this.play = function(reset) {
+ if(!this.active) {
+ if(reset) {this.setTimer();}
+ else {this.setTimer(this.remaining);}
+ this.active = true;
+ }
+ return this;
+ };
+ this.pause = function() {
+ if(this.active) {
+ this.active = false;
+ this.remaining -= new Date() - this.last;
+ this.clearTimer();
+ }
+ return this;
+ };
+ this.stop = function() {
+ this.active = false;
+ this.remaining = this.intervalTime;
+ this.clearTimer();
+ return this;
+ };
+ this.toggle = function(reset) {
+ if(this.active) {this.pause();}
+ else if(reset) {this.play(true);}
+ else {this.play();}
+ return this;
+ };
+ this.reset = function() {
+ this.active = false;
+ this.play(true);
+ return this;
+ };
+ this.clearTimer = function() {
+ window.clearTimeout(this.timeoutObject);
+ };
+ this.setTimer = function(time) {
+ var timer = this;
+ if(typeof this.action != 'function') {return;}
+ if(isNaN(time)) {time = this.intervalTime;}
+ this.remaining = time;
+ this.last = new Date();
+ this.clearTimer();
+ this.timeoutObject = window.setTimeout(function() {timer.go();}, time);
+ };
+ this.go = function() {
+ if(this.active) {
+ this.action();
+ this.setTimer();
+ }
+ };
+
+ if(this.init) {
+ return new $.timer(func, time, autostart);
+ } else {
+ this.set(func, time, autostart);
+ return this;
+ }
+ };
+})(jQuery);
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/javascript/jsplumb/jquery.jsPlumb-1.3.5-all-min.js b/hyracks-control-cc/src/main/resources/static/javascript/jsplumb/jquery.jsPlumb-1.3.5-all-min.js
new file mode 100644
index 0000000..47725f9
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/javascript/jsplumb/jquery.jsPlumb-1.3.5-all-min.js
@@ -0,0 +1 @@
+(function(){var w=!!document.createElement("canvas").getContext,e=!!window.SVGAngle||document.implementation.hasFeature("http://www.w3.org/TR/SVG11/feature#BasicStructure","1.1"),a=!(w|e);var p=function(F,G,D,J){var I=function(M,L){if(M===L){return true}else{if(typeof M=="object"&&typeof L=="object"){var N=true;for(var K in M){if(!I(M[K],L[K])){N=false;break}}for(var K in L){if(!I(L[K],M[K])){N=false;break}}return N}}};for(var H=+D||0,E=F.length;H<E;H++){if(I(F[H],G)){return H}}return -1},i=function(D,F){if(D){for(var E=0;E<D.length;E++){if(F(D[E])){return E}}}return -1},y=function(E,F){var D=i(E,F);if(D>-1){E.splice(D,1)}return D!=-1},r=function(F,E,D){if(i(F,D)==-1){F.push(E)}};if(!window.console){window.console={time:function(){},timeEnd:function(){},group:function(){},groupEnd:function(){},log:function(){}}}var l=function(G,E,F){var D=G[E];if(D==null){D=[],G[E]=D}D.push(F);return D},u=null,d=function(D,E){return n.CurrentLibrary.getAttribute(A(D),E)},f=function(E,F,D){n.CurrentLibrary.setAttribute(A(E),F,D)},x=function(E,D){n.CurrentLibrary.addClass(A(E),D)},k=function(E,D){return n.CurrentLibrary.hasClass(A(E),D)},m=function(E,D){n.CurrentLibrary.removeClass(A(E),D)},A=function(D){return n.CurrentLibrary.getElementObject(D)},s=function(D){return n.CurrentLibrary.getOffset(A(D))},b=function(D){return n.CurrentLibrary.getSize(A(D))},z=true,o=function(){if(z&&typeof console!="undefined"){try{var E=arguments[arguments.length-1];console.log(E)}catch(D){}}},C=function(D){if(z&&typeof console!="undefined"){console.group(D)}},h=function(D){if(z&&typeof console!="undefined"){console.groupEnd(D)}},B=function(D){if(z&&typeof console!="undefined"){console.time(D)}},t=function(D){if(z&&typeof console!="undefined"){console.timeEnd(D)}};EventGenerator=function(){var F={},E=this;var D=["ready"];this.bind=function(G,H){l(F,G,H)};this.fire=function(I,J,G){if(F[I]){for(var H=0;H<F[I].length;H++){if(p(D,I)!=-1){F[I][H](J,G)}else{try{F[I][H](J,G)}catch(K){o("jsPlumb: fire failed for event "+I+" : "+K)}}}}};this.clearListeners=function(G){if(G){delete F[G]}else{delete F;F={}}};this.getListener=function(G){return F[G]}},_timestamp=function(){return""+(new Date()).getTime()},jsPlumbUIComponent=function(H){var S=this,O=arguments,M=false,Q=H.parameters||{},E=S.idPrefix,G=E+(new Date()).getTime();S._jsPlumb=H._jsPlumb;S.getId=function(){return G};S.tooltip=H.tooltip;S.hoverClass=H.hoverClass;EventGenerator.apply(this);this.clone=function(){var T=new Object();S.constructor.apply(T,O);return T};this.getParameter=function(T){return Q[T]},this.getParameters=function(){return Q},this.setParameter=function(T,U){Q[T]=U},this.setParameters=function(T){Q=T},this.overlayPlacements=[],this.paintStyle=null,this.hoverPaintStyle=null;var I=H.beforeDetach;this.isDetachAllowed=function(T){var U=true;if(I){try{U=I(T)}catch(V){o("jsPlumb: beforeDetach callback failed",V)}}return U};var K=H.beforeDrop;this.isDropAllowed=function(W,T,U){var V=S._jsPlumb.checkCondition("beforeDrop",{sourceId:W,targetId:T,scope:U});if(K){try{V=K({sourceId:W,targetId:T,scope:U})}catch(X){o("jsPlumb: beforeDrop callback failed",X)}}return V};var P=function(){if(S.paintStyle&&S.hoverPaintStyle){var T={};n.extend(T,S.paintStyle);n.extend(T,S.hoverPaintStyle);delete S.hoverPaintStyle;if(T.gradient&&S.paintStyle.fillStyle){delete T.gradient}S.hoverPaintStyle=T}};this.setPaintStyle=function(T,U){S.paintStyle=T;S.paintStyleInUse=S.paintStyle;P();if(!U){S.repaint()}};this.setHoverPaintStyle=function(T,U){S.hoverPaintStyle=T;P();if(!U){S.repaint()}};this.setHover=function(T,V,U){if(!S._jsPlumb.currentlyDragging&&!S._jsPlumb.isHoverSuspended()){M=T;if(S.hoverClass!=null&&S.canvas!=null){if(T){J.addClass(S.canvas,S.hoverClass)}else{J.removeClass(S.canvas,S.hoverClass)}}if(S.hoverPaintStyle!=null){S.paintStyleInUse=T?S.hoverPaintStyle:S.paintStyle;U=U||_timestamp();S.repaint({timestamp:U,recalc:false})}if(S.getAttachedElements&&!V){N(T,_timestamp(),S)}}};this.isHover=function(){return M};var J=n.CurrentLibrary,R=["click","dblclick","mouseenter","mouseout","mousemove","mousedown","mouseup","contextmenu"],L={mouseout:"mouseexit"},F=function(V,W,U){var T=L[U]||U;J.bind(V,U,function(X){W.fire(T,W,X)})},D=function(V,U){var T=L[U]||U;J.unbind(V,U)};this.attachListeners=function(U,V){for(var T=0;T<R.length;T++){F(U,V,R[T])}};var N=function(X,W,T){var V=S.getAttachedElements();if(V){for(var U=0;U<V.length;U++){if(!T||T!=V[U]){V[U].setHover(X,true,W)}}}};this.reattachListenersForElement=function(U){if(arguments.length>1){for(var T=0;T<R.length;T++){D(U,R[T])}for(var T=1;T<arguments.length;T++){S.attachListeners(U,arguments[T])}}}},overlayCapableJsPlumbUIComponent=function(I){jsPlumbUIComponent.apply(this,arguments);var N=this;this.overlays=[];var G=function(S){var Q=null;if(S.constructor==Array){var P=S[0],R=n.extend({component:N,_jsPlumb:N._jsPlumb},S[1]);if(S.length==3){n.extend(R,S[2])}Q=new n.Overlays[N._jsPlumb.getRenderMode()][P](R);if(R.events){for(var O in R.events){Q.bind(O,R.events[O])}}}else{if(S.constructor==String){Q=new n.Overlays[N._jsPlumb.getRenderMode()][S]({component:N,_jsPlumb:N._jsPlumb})}else{Q=S}}N.overlays.push(Q)},H=function(S){var O=N.defaultOverlayKeys||[],R=S.overlays,P=function(T){return N._jsPlumb.Defaults[T]||n.Defaults[T]||[]};if(!R){R=[]}for(var Q=0;Q<O.length;Q++){R.unshift.apply(R,P(O[Q]))}return R};var E=H(I);if(E){for(var J=0;J<E.length;J++){G(E[J])}}var D=function(Q){var O=-1;for(var P=0;P<N.overlays.length;P++){if(Q===N.overlays[P].id){O=P;break}}return O};this.addOverlay=function(O){G(O);N.repaint()};this.getOverlay=function(P){var O=D(P);return O>=0?N.overlays[O]:null};this.hideOverlay=function(P){var O=N.getOverlay(P);if(O){O.hide()}};this.showOverlay=function(P){var O=N.getOverlay(P);if(O){O.show()}};this.removeAllOverlays=function(){N.overlays.splice(0,N.overlays.length);N.repaint()};this.removeOverlay=function(P){var O=D(P);if(O!=-1){var Q=N.overlays[O];Q.cleanup();N.overlays.splice(O,1)}};this.removeOverlays=function(){for(var O=0;O<arguments.length;O++){N.removeOverlay(arguments[O])}};var F="__label",M=function(Q){var O={cssClass:Q.cssClass,labelStyle:this.labelStyle,id:F,component:N,_jsPlumb:N._jsPlumb},P=n.extend(O,Q);return new n.Overlays[N._jsPlumb.getRenderMode()].Label(P)};if(I.label){var K=I.labelLocation||N.defaultLabelLocation||0.5,L=I.labelStyle||N._jsPlumb.Defaults.LabelStyle||n.Defaults.LabelStyle;this.overlays.push(M({label:I.label,location:K,labelStyle:L}))}this.setLabel=function(O){var P=N.getOverlay(F);if(!P){var Q=O.constructor==String||O.constructor==Function?{label:O}:O;P=M(Q);this.overlays.push(P)}else{if(O.constructor==String||O.constructor==Function){P.setLabel(O)}else{if(O.label){P.setLabel(O.label)}if(O.location){P.setLocation(O.location)}}}N.repaint()};this.getLabel=function(){var O=N.getOverlay(F);return O!=null?O.getLabel():null};this.getLabelOverlay=function(){return N.getOverlay(F)}},_bindListeners=function(F,D,E){F.bind("click",function(G,H){D.fire("click",D,H)});F.bind("dblclick",function(G,H){D.fire("dblclick",D,H)});F.bind("contextmenu",function(G,H){D.fire("contextmenu",D,H)});F.bind("mouseenter",function(G,H){if(!D.isHover()){E(true);D.fire("mouseenter",D,H)}});F.bind("mouseexit",function(G,H){if(D.isHover()){E(false);D.fire("mouseexit",self,H)}})};var g=0,c=function(){var D=g+1;g++;return D};var v=function(E){this.Defaults={Anchor:"BottomCenter",Anchors:[null,null],ConnectionsDetachable:true,Connector:"Bezier",Container:null,DragOptions:{},DropOptions:{},Endpoint:"Dot",Endpoints:[null,null],EndpointStyle:{fillStyle:"#456"},EndpointStyles:[null,null],EndpointHoverStyle:null,EndpointHoverStyles:[null,null],HoverPaintStyle:null,LabelStyle:{color:"black"},LogEnabled:false,Overlays:[],MaxConnections:1,MouseEventsEnabled:true,PaintStyle:{lineWidth:8,strokeStyle:"#456"},RenderMode:"svg",Scope:"jsPlumb_DefaultScope"};if(E){n.extend(this.Defaults,E)}this.logEnabled=this.Defaults.LogEnabled;EventGenerator.apply(this);var av=this.bind;this.bind=function(a7,a6){if("ready"===a7&&F){a6()}else{av(a7,a6)}};var aZ=this,at=c(),H=null,ae=function(){n.repaintEverything()},a1=true,aw=function(){if(a1){ae()}},aQ=null,F=false,aD={},az={},aA={},X={},a2={},aR={},aY={},aW=this.Defaults.MouseEventsEnabled,a5=[],U=[],I=this.Defaults.Scope,O=null,M=function(a9,a7,a8){var a6=a9[a7];if(a6==null){a6=[];a9[a7]=a6}a6.push(a8);return a6},aB=function(a7,a6){if(aZ.Defaults.Container){n.CurrentLibrary.appendElement(a7,aZ.Defaults.Container)}else{if(!a6){document.body.appendChild(a7)}else{n.CurrentLibrary.appendElement(a7,a6)}}},am=1,Z=function(){return""+am++},aq=function(a6){return a6._nodes?a6._nodes:a6},aJ=false,aS=function(a7,a6){aJ=a7;if(a6){aZ.repaintEverything()}},aM=function(a6,a8,a7){if(!aJ){var a9=d(a6,"id");aZ.anchorManager.redraw(a9,a8,a7)}},ap=function(a7,a9){var ba=null;if(a7.constructor==Array){ba=[];for(var a6=0;a6<a7.length;a6++){var a8=A(a7[a6]),bb=d(a8,"id");ba.push(a9(a8,bb))}}else{var a8=A(a7),bb=d(a8,"id");ba=a9(a8,bb)}return ba},ah=function(a6){return aA[a6]},aN=function(ba,a9,a8){var a6=a9==null?false:a9;if(a6){if(n.CurrentLibrary.isDragSupported(ba)&&!n.CurrentLibrary.isAlreadyDraggable(ba)){var a7=a8||aZ.Defaults.DragOptions||n.Defaults.DragOptions;a7=n.extend({},a7);var bc=n.CurrentLibrary.dragEvents.drag,bb=n.CurrentLibrary.dragEvents.stop;a7[bc]=ab(a7[bc],function(){var bd=n.CurrentLibrary.getUIPosition(arguments);aM(ba,bd);x(ba,"jsPlumb_dragged")});a7[bb]=ab(a7[bb],function(){var bd=n.CurrentLibrary.getUIPosition(arguments);aM(ba,bd);m(ba,"jsPlumb_dragged")});aY[D(ba)]=true;var a6=aY[D(ba)];a7.disabled=a6==null?false:!a6;n.CurrentLibrary.initDraggable(ba,a7,false)}}},al=function(bb,a7){var a6=n.extend({},bb);if(a7){n.extend(a6,a7)}if(a6.source&&a6.source.endpoint){a6.sourceEndpoint=a6.source}if(a6.source&&a6.target.endpoint){a6.targetEndpoint=a6.target}if(bb.uuids){a6.sourceEndpoint=ah(bb.uuids[0]);a6.targetEndpoint=ah(bb.uuids[1])}if(a6.sourceEndpoint&&a6.sourceEndpoint.isFull()){o(aZ,"could not add connection; source endpoint is full");return}if(a6.targetEndpoint&&a6.targetEndpoint.isFull()){o(aZ,"could not add connection; target endpoint is full");return}if(a6.sourceEndpoint&&a6.sourceEndpoint.connectorOverlays){a6.overlays=a6.overlays||[];for(var a9=0;a9<a6.sourceEndpoint.connectorOverlays.length;a9++){a6.overlays.push(a6.sourceEndpoint.connectorOverlays[a9])}}a6.tooltip=bb.tooltip;if(!a6.tooltip&&a6.sourceEndpoint&&a6.sourceEndpoint.connectorTooltip){a6.tooltip=a6.sourceEndpoint.connectorTooltip}if(a6.target&&!a6.target.endpoint){var ba=D(a6.target),bc=aE[ba];var a8=function(bf,bd,be,bg){if(bg[be]){if(a6[bd]){a6[bd][1]=bg[be]}else{if(a6[bf]){a6[bd]=[a6[bf],bg[be]];a6[bf]=null}else{a6[bd]=[null,bg[be]]}}}};if(bc){a8("endpoint","endpoints","endpoint",bc);a8("endpointStyle","endpointStyles","paintStyle",bc);a8("endpointHoverStyle","endpointHoverStyles","hoverPaintStyle",bc)}}return a6},T=function(ba){var a9=aZ.Defaults.ConnectionType||aZ.getDefaultConnectionType(),a8=aZ.Defaults.EndpointType||V,a7=n.CurrentLibrary.getParent;if(ba.container){ba.parent=ba.container}else{if(ba.sourceEndpoint){ba.parent=ba.sourceEndpoint.parent}else{if(ba.source.constructor==a8){ba.parent=ba.source.parent}else{ba.parent=a7(ba.source)}}}ba._jsPlumb=aZ;var a6=new a9(ba);a6.id="con_"+Z();a3("click","click",a6);a3("dblclick","dblclick",a6);a3("contextmenu","contextmenu",a6);return a6},a4=function(a7,a8,a6){a8=a8||{};if(!a7.suspendedEndpoint){M(aD,a7.scope,a7)}if(!a8.doNotFireConnectionEvent&&a8.fireEvent!==false){aZ.fire("jsPlumbConnection",{connection:a7,source:a7.source,target:a7.target,sourceId:a7.sourceId,targetId:a7.targetId,sourceEndpoint:a7.endpoints[0],targetEndpoint:a7.endpoints[1]},a6)}aZ.anchorManager.newConnection(a7);aM(a7.source)},a3=function(a6,a7,a8){a8.bind(a6,function(ba,a9){aZ.fire(a7,a8,a9)})},ai=function(a8){if(a8.container){return a8.container}else{var a6=n.CurrentLibrary.getTagName(a8.source),a7=n.CurrentLibrary.getParent(a8.source);if(a6&&a6.toLowerCase()==="td"){return n.CurrentLibrary.getParent(a7)}else{return a7}}},ao=function(a8){var a7=aZ.Defaults.EndpointType||V;a8.parent=ai(a8);a8._jsPlumb=aZ;var a6=new a7(a8);a6.id="ep_"+Z();a3("click","endpointClick",a6);a3("dblclick","endpointDblClick",a6);a3("contextmenu","contextmenu",a6);return a6},L=function(a8,bb,ba){var a6=az[a8];if(a6&&a6.length){for(var a9=0;a9<a6.length;a9++){for(var a7=0;a7<a6[a9].connections.length;a7++){var bc=bb(a6[a9].connections[a7]);if(bc){return}}if(ba){ba(a6[a9])}}}},P=function(a7){for(var a6 in az){L(a6,a7)}},ag=function(a6,a7){if(a6!=null&&a6.parentNode!=null){a6.parentNode.removeChild(a6)}},aC=function(a8,a7){for(var a6=0;a6<a8.length;a6++){ag(a8[a6],a7)}},aV=function(a7,a6){return ap(a7,function(a8,a9){aY[a9]=a6;if(n.CurrentLibrary.isDragSupported(a8)){n.CurrentLibrary.setDraggable(a8,a6)}})},aH=function(a8,a9,a6){a9=a9==="block";var a7=null;if(a6){if(a9){a7=function(bb){bb.setVisible(true,true,true)}}else{a7=function(bb){bb.setVisible(false,true,true)}}}var ba=d(a8,"id");L(ba,function(bc){if(a9&&a6){var bb=bc.sourceId===ba?1:0;if(bc.endpoints[bb].isVisible()){bc.setVisible(true)}}else{bc.setVisible(a9)}},a7)},aT=function(a6){return ap(a6,function(a8,a7){var a9=aY[a7]==null?false:aY[a7];a9=!a9;aY[a7]=a9;n.CurrentLibrary.setDraggable(a8,a9);return a9})},ax=function(a6,a8){var a7=null;if(a8){a7=function(a9){var ba=a9.isVisible();a9.setVisible(!ba)}}L(a6,function(ba){var a9=ba.isVisible();ba.setVisible(!a9)},a7)},N=function(bb){var a9=bb.timestamp,a6=bb.recalc,ba=bb.offset,a7=bb.elId;if(!a6){if(a9&&a9===a2[a7]){return X[a7]}}if(a6||!ba){var a8=A(a7);if(a8!=null){U[a7]=b(a8);X[a7]=s(a8);a2[a7]=a9}}else{X[a7]=ba;if(U[a7]==null){var a8=A(a7);if(a8!=null){U[a7]=b(a8)}}}if(X[a7]&&!X[a7].right){X[a7].right=X[a7].left+U[a7][0];X[a7].bottom=X[a7].top+U[a7][1];X[a7].width=U[a7][0];X[a7].height=U[a7][1];X[a7].centerx=X[a7].left+(X[a7].width/2);X[a7].centery=X[a7].top+(X[a7].height/2)}return X[a7]},au=function(a6){var a7=X[a6];if(!a7){a7=N({elId:a6})}return{o:a7,s:U[a6]}},D=function(a6,a7){var a8=A(a6);var a9=d(a8,"id");if(!a9||a9=="undefined"){if(arguments.length==2&&arguments[1]!=undefined){a9=a7}else{a9="jsPlumb_"+at+"_"+Z()}f(a8,"id",a9)}return a9},ab=function(a8,a6,a7){a8=a8||function(){};a6=a6||function(){};return function(){var a9=null;try{a9=a6.apply(this,arguments)}catch(ba){o(aZ,"jsPlumb function failed : "+ba)}if(a7==null||(a9!==a7)){try{a8.apply(this,arguments)}catch(ba){o(aZ,"wrapped function failed : "+ba)}}return a9}};this.connectorClass="_jsPlumb_connector";this.endpointClass="_jsPlumb_endpoint";this.overlayClass="_jsPlumb_overlay";this.Anchors={};this.Connectors={canvas:{},svg:{},vml:{}};this.Endpoints={canvas:{},svg:{},vml:{}};this.Overlays={canvas:{},svg:{},vml:{}};this.addClass=function(a7,a6){return n.CurrentLibrary.addClass(a7,a6)};this.removeClass=function(a7,a6){return n.CurrentLibrary.removeClass(a7,a6)};this.hasClass=function(a7,a6){return n.CurrentLibrary.hasClass(a7,a6)};this.addEndpoint=function(a8,a9,bi){bi=bi||{};var a7=n.extend({},bi);n.extend(a7,a9);a7.endpoint=a7.endpoint||aZ.Defaults.Endpoint||n.Defaults.Endpoint;a7.paintStyle=a7.paintStyle||aZ.Defaults.EndpointStyle||n.Defaults.EndpointStyle;a8=aq(a8);var ba=[],bd=a8.length&&a8.constructor!=String?a8:[a8];for(var bb=0;bb<bd.length;bb++){var bg=A(bd[bb]),a6=D(bg);a7.source=bg;N({elId:a6});var bf=ao(a7);if(a7.parentAnchor){bf.parentAnchor=a7.parentAnchor}M(az,a6,bf);var be=X[a6],bc=U[a6];var bh=bf.anchor.compute({xy:[be.left,be.top],wh:bc,element:bf});bf.paint({anchorLoc:bh});ba.push(bf)}return ba.length==1?ba[0]:ba};this.addEndpoints=function(ba,a7,a6){var a9=[];for(var a8=0;a8<a7.length;a8++){var bb=aZ.addEndpoint(ba,a7[a8],a6);if(bb.constructor==Array){Array.prototype.push.apply(a9,bb)}else{a9.push(bb)}}return a9};this.animate=function(a8,a7,a6){var a9=A(a8),bc=d(a8,"id");a6=a6||{};var bb=n.CurrentLibrary.dragEvents.step;var ba=n.CurrentLibrary.dragEvents.complete;a6[bb]=ab(a6[bb],function(){aZ.repaint(bc)});a6[ba]=ab(a6[ba],function(){aZ.repaint(bc)});n.CurrentLibrary.animate(a9,a7,a6)};this.checkCondition=function(a8,ba){var a6=aZ.getListener(a8);var a9=true;if(a6&&a6.length>0){try{for(var a7=0;a7<a6.length;a7++){a9=a9&&a6[a7](ba)}}catch(bb){o(aZ,"cannot check condition ["+a8+"]"+bb)}}return a9};this.connect=function(a9,a7){var a6=al(a9,a7);if(a6){if(a6.deleteEndpointsOnDetach==null){a6.deleteEndpointsOnDetach=true}var a8=T(a6);a4(a8,a6);return a8}};this.deleteEndpoint=function(a7){var bc=(typeof a7=="string")?aA[a7]:a7;if(bc){var a9=bc.getUuid();if(a9){aA[a9]=null}bc.detachAll();aC(bc.endpoint.getDisplayElements());aZ.anchorManager.deleteEndpoint(bc);for(var bb in az){var a6=az[bb];if(a6){var ba=[];for(var a8=0;a8<a6.length;a8++){if(a6[a8]!=bc){ba.push(a6[a8])}}az[bb]=ba}}delete bc}};this.deleteEveryEndpoint=function(){for(var a8 in az){var a6=az[a8];if(a6&&a6.length){for(var a7=0;a7<a6.length;a7++){aZ.deleteEndpoint(a6[a7])}}}delete az;az={};delete aA;aA={}};var aK=function(a8,ba){var a7=aZ.Defaults.ConnectionType||aZ.getDefaultConnectionType(),a6=a8.constructor==a7,a9=a6?{connection:a8,source:a8.source,target:a8.target,sourceId:a8.sourceId,targetId:a8.targetId,sourceEndpoint:a8.endpoints[0],targetEndpoint:a8.endpoints[1]}:a8;if(ba){aZ.fire("jsPlumbConnectionDetached",a9)}aZ.anchorManager.connectionDetached(a9)};this.detach=function(){if(arguments.length==0){return}var ba=aZ.Defaults.ConnectionType||aZ.getDefaultConnectionType(),bb=arguments[0].constructor==ba,a9=arguments.length==2?bb?(arguments[1]||{}):arguments[0]:arguments[0],be=(a9.fireEvent!==false),a8=a9.forceDetach,a7=bb?arguments[0]:a9.connection;if(a7){if(a8||(a7.isDetachAllowed(a7)&&a7.endpoints[0].isDetachAllowed(a7)&&a7.endpoints[1].isDetachAllowed(a7))){if(a8||aZ.checkCondition("beforeDetach",a7)){a7.endpoints[0].detach(a7,false,true,be)}}}else{var a6=n.extend({},a9);if(a6.uuids){ah(a6.uuids[0]).detachFrom(ah(a6.uuids[1]),be)}else{if(a6.sourceEndpoint&&a6.targetEndpoint){a6.sourceEndpoint.detachFrom(a6.targetEndpoint)}else{var bd=D(a6.source),bc=D(a6.target);L(bd,function(bf){if((bf.sourceId==bd&&bf.targetId==bc)||(bf.targetId==bd&&bf.sourceId==bc)){if(aZ.checkCondition("beforeDetach",bf)){bf.endpoints[0].detach(bf,false,true,be)}}})}}}};this.detachAllConnections=function(a8,a9){a9=a9||{};a8=A(a8);var ba=d(a8,"id"),a6=az[ba];if(a6&&a6.length){for(var a7=0;a7<a6.length;a7++){a6[a7].detachAll(a9.fireEvent)}}};this.detachEveryConnection=function(a8){a8=a8||{};for(var a9 in az){var a6=az[a9];if(a6&&a6.length){for(var a7=0;a7<a6.length;a7++){a6[a7].detachAll(a8.fireEvent)}}}delete aD;aD={}};this.draggable=function(a8,a6){if(typeof a8=="object"&&a8.length){for(var a7=0;a7<a8.length;a7++){var a9=A(a8[a7]);if(a9){aN(a9,true,a6)}}}else{if(a8._nodes){for(var a7=0;a7<a8._nodes.length;a7++){var a9=A(a8._nodes[a7]);if(a9){aN(a9,true,a6)}}}else{var a9=A(a8);if(a9){aN(a9,true,a6)}}}};this.extend=function(a7,a6){return n.CurrentLibrary.extend(a7,a6)};this.getDefaultEndpointType=function(){return V};this.getDefaultConnectionType=function(){return aj};this.getConnections=function(bh){if(!bh){bh={}}else{if(bh.constructor==String){bh={scope:bh}}}var be=function(bi){var bj=[];if(bi){if(typeof bi=="string"){bj.push(bi)}else{bj=bi}}return bj},bf=bh.scope||aZ.getDefaultScope(),bd=be(bf),a6=be(bh.source),bb=be(bh.target),a7=function(bj,bi){return bj.length>0?p(bj,bi)!=-1:true},ba=bd.length>1?{}:[],bg=function(bj,bk){if(bd.length>1){var bi=ba[bj];if(bi==null){bi=[];ba[bj]=bi}bi.push(bk)}else{ba.push(bk)}};for(var a9 in aD){if(a7(bd,a9)){for(var a8=0;a8<aD[a9].length;a8++){var bc=aD[a9][a8];if(a7(a6,bc.sourceId)&&a7(bb,bc.targetId)){bg(a9,bc)}}}}return ba};this.getAllConnections=function(){return aD};this.getDefaultScope=function(){return I};this.getEndpoint=ah;this.getEndpoints=function(a6){return az[D(a6)]};this.getId=D;this.getOffset=function(a7){var a6=X[a7];return N({elId:a7})};this.getSelector=function(a6){return n.CurrentLibrary.getSelector(a6)};this.getSize=function(a7){var a6=U[a7];if(!a6){N({elId:a7})}return U[a7]};this.appendElement=aB;var ay=false;this.isHoverSuspended=function(){return ay};this.setHoverSuspended=function(a6){ay=a6};this.hide=function(a6,a7){aH(a6,"none",a7)};this.idstamp=Z;this.init=function(){if(!F){aZ.setRenderMode(aZ.Defaults.RenderMode);var a6=function(a7){n.CurrentLibrary.bind(document,a7,function(bd){if(!aZ.currentlyDragging&&aW&&O==n.CANVAS){for(var bc in aD){var be=aD[bc];for(var ba=0;ba<be.length;ba++){var a9=be[ba].connector[a7](bd);if(a9){return}}}for(var bb in az){var a8=az[bb];for(var ba=0;ba<a8.length;ba++){if(a8[ba].endpoint[a7](bd)){return}}}}})};a6("click");a6("dblclick");a6("mousemove");a6("mousedown");a6("mouseup");a6("contextmenu");F=true;aZ.fire("ready")}};this.log=H;this.jsPlumbUIComponent=jsPlumbUIComponent;this.EventGenerator=EventGenerator;this.makeAnchor=function(){if(arguments.length==0){return null}var bb=arguments[0],a8=arguments[1],a7=arguments[2],a9=null;if(!a7){throw"NO JSPLUMB SET"}if(bb.compute&&bb.getOrientation){return bb}else{if(typeof bb=="string"){a9=n.Anchors[arguments[0]]({elementId:a8,jsPlumbInstance:aZ})}else{if(bb.constructor==Array){if(bb[0].constructor==Array||bb[0].constructor==String){if(bb.length==2&&bb[0].constructor==String&&bb[1].constructor==Object){var a6=n.extend({elementId:a8,jsPlumbInstance:aZ},bb[1]);a9=n.Anchors[bb[0]](a6)}else{a9=new ad(bb,null,a8)}}else{var ba={x:bb[0],y:bb[1],orientation:(bb.length>=4)?[bb[2],bb[3]]:[0,0],offsets:(bb.length==6)?[bb[4],bb[5]]:[0,0],elementId:a8};a9=new Q(ba);a9.clone=function(){return new Q(ba)}}}}}if(!a9.id){a9.id="anchor_"+Z()}return a9};this.makeAnchors=function(a9,a7,a6){var ba=[];for(var a8=0;a8<a9.length;a8++){if(typeof a9[a8]=="string"){ba.push(n.Anchors[a9[a8]]({elementId:a7,jsPlumbInstance:a6}))}else{if(a9[a8].constructor==Array){ba.push(aZ.makeAnchor(a9[a8],a7,a6))}}}return ba};this.makeDynamicAnchor=function(a6,a7){return new ad(a6,a7)};var aE={};var S=function(a6,a7){a6.paintStyle=a6.paintStyle||aZ.Defaults.EndpointStyles[a7]||aZ.Defaults.EndpointStyle||n.Defaults.EndpointStyles[a7]||n.Defaults.EndpointStyle;a6.hoverPaintStyle=a6.hoverPaintStyle||aZ.Defaults.EndpointHoverStyles[a7]||aZ.Defaults.EndpointHoverStyle||n.Defaults.EndpointHoverStyles[a7]||n.Defaults.EndpointHoverStyle;a6.anchor=a6.anchor||aZ.Defaults.Anchors[a7]||aZ.Defaults.Anchor||n.Defaults.Anchors[a7]||n.Defaults.Anchor};this.makeTarget=function(a8,a9,bf){var a7=n.extend({},bf);n.extend(a7,a9);var bd=n.CurrentLibrary,be=a7.scope||aZ.Defaults.Scope,ba=a7.deleteEndpointsOnDetach||true,a6=function(bk){var bi=D(bk);aE[bi]=a7.endpoint;var bh=n.extend({},a7.dropOptions||{}),bg=function(){aZ.currentlyDragging=false;var bx=A(bd.getDragObject(arguments)),bn=d(bx,"dragId"),bv=d(bx,"originalScope"),bs=aR[bn],bm=bs.endpoints[0],bl=a7.endpoint?n.extend({},a7.endpoint):{};S(bl,1);bm.anchor.locked=false;if(bv){bd.setDragScope(bx,bv)}var bq=bs.isDropAllowed(bs.sourceId,D(bk),bs.scope);if(bs.endpointsToDeleteOnDetach){if(bm===bs.endpointsToDeleteOnDetach[0]){bs.endpointsToDeleteOnDetach[0]=null}else{if(bm===bs.endpointsToDeleteOnDetach[1]){bs.endpointsToDeleteOnDetach[1]=null}}}if(bs.suspendedEndpoint){bs.targetId=bs.suspendedEndpoint.elementId;bs.target=bd.getElementObject(bs.suspendedEndpoint.elementId);bs.endpoints[1]=bs.suspendedEndpoint}if(bq){bm.detach(bs,false,true,true);var bw=aZ.addEndpoint(bk,bl);if(bw.anchor.positionFinder!=null){var bt=bd.getUIPosition(arguments),bp=bd.getOffset(bk),bu=bd.getSize(bk),bo=bw.anchor.positionFinder(bt,bp,bu,bw.anchor);bw.anchor.x=bo[0];bw.anchor.y=bo[1]}var br=aZ.connect({source:bm,target:bw,scope:bv,previousConnection:bs,container:bs.parent,doNotFireConnectionEvent:bm.endpointWillMoveAfterConnection});if(ba){br.endpointsToDeleteOnDetach=[bm,bw]}}else{if(bs.suspendedEndpoint){if(bm.isReattach){bs.setHover(false);bs.floatingAnchorIndex=null;bs.suspendedEndpoint.addConnection(bs);aZ.repaint(bm.elementId)}else{bm.detach(bs,false,true,true)}}}};var bj=bd.dragEvents.drop;bh.scope=bh.scope||be;bh[bj]=ab(bh[bj],bg);bd.initDroppable(bk,bh,true)};a8=aq(a8);var bc=a8.length&&a8.constructor!=String?a8:[a8];for(var bb=0;bb<bc.length;bb++){a6(A(bc[bb]))}};this.makeTargets=function(a8,a9,a6){for(var a7=0;a7<a8.length;a7++){aZ.makeTarget(a8[a7],a9,a6)}};var ak={};this.makeSource=function(ba,bd,a6){var bb=n.extend({},a6);n.extend(bb,bd);var a9=n.CurrentLibrary,bc=function(bk){var be=D(bk);ak[be]=bb.endpoint||{};var bf=a9.dragEvents.stop,bj=a9.dragEvents.drag,bl=n.extend({},bb.dragOptions||ak[be].dragOptions||{}),bh=bl.drag,bm=bl.stop,bn=null,bi=false;bl.scope=bl.scope||bb.scope;ak[be].endpoint=ak[be].endpoint||aZ.Defaults.Endpoints[0]||aZ.Defaults.Endpoint;S(ak[be],0);bl[bj]=ab(bl[bj],function(){if(bh){bh.apply(this,arguments)}bi=false});bl[bf]=function(){if(bm){bm.apply(this,arguments)}aZ.currentlyDragging=false;if(bn.connections.length==0){aZ.deleteEndpoint(bn)}else{a9.unbind(bn.canvas,"mousedown");var bp=ak[be].anchor||aZ.Defaults.Anchor;bn.anchor=aZ.makeAnchor(bp,be,aZ);if(bb.parent){var bq=a9.getElementObject(bb.parent);if(bq){var bo=bn.elementId;bn.setElement(bq);bn.endpointWillMoveAfterConnection=false;aZ.anchorManager.rehomeEndpoint(bo,bq);bn.connections[0].previousConnection=null;a4(bn.connections[0])}}aZ.repaint(be)}};var bg=function(bp){var bu=N({elId:be});var bt=((bp.pageX||bp.page.x)-bu.left)/bu.width,bs=((bp.pageY||bp.page.y)-bu.top)/bu.height,by=bt,bx=bs;if(bb.parent){var br=n.CurrentLibrary.getElementObject(bb.parent),bq=D(br);bu=N({elId:bq});by=((bp.pageX||bp.page.x)-bu.left)/bu.width,bx=((bp.pageY||bp.page.y)-bu.top)/bu.height}var bw={};n.extend(bw,ak[be]);bw.isSource=true;bw.anchor=[bt,bs,0,0];bw.parentAnchor=[by,bx,0,0];bw.dragOptions=bl;if(bb.parent){var bo=bw.container||aZ.Defaults.Container;if(bo){bw.container=bo}else{bw.container=n.CurrentLibrary.getParent(bb.parent)}}bn=aZ.addEndpoint(be,bw);bi=true;bn.endpointWillMoveAfterConnection=bb.parent!=null;bn.endpointWillMoveTo=bb.parent?a9.getElementObject(bb.parent):null;var bv=function(){if(bi){aZ.deleteEndpoint(bn)}};aZ.registerListener(bn.canvas,"mouseup",bv);aZ.registerListener(bk,"mouseup",bv);a9.trigger(bn.canvas,"mousedown",bp)};aZ.registerListener(bk,"mousedown",bg)};ba=aq(ba);var a7=ba.length&&ba.constructor!=String?ba:[ba];for(var a8=0;a8<a7.length;a8++){bc(A(a7[a8]))}};this.makeSources=function(a8,a9,a6){for(var a7=0;a7<a8.length;a7++){aZ.makeSource(a8[a7],a9,a6)}};this.ready=function(a6){aZ.bind("ready",a6)},this.repaint=function(a7){var a8=function(a9){aM(A(a9))};if(typeof a7=="object"){for(var a6=0;a6<a7.length;a6++){a8(a7[a6])}}else{a8(a7)}};this.repaintEverything=function(){var a7=_timestamp();for(var a6 in az){aM(A(a6),null,a7)}};this.removeAllEndpoints=function(a8){var a6=d(a8,"id"),a9=az[a6];if(a9){for(var a7=0;a7<a9.length;a7++){aZ.deleteEndpoint(a9[a7])}}az[a6]=[]};this.removeEveryEndpoint=this.deleteEveryEndpoint;this.removeEndpoint=function(a6,a7){aZ.deleteEndpoint(a7)};var W={},aP=function(){for(var a7 in W){for(var a6=0;a6<W[a7].length;a6++){var a8=W[a7][a6];n.CurrentLibrary.unbind(a8.el,a8.event,a8.listener)}}W={}};this.registerListener=function(a7,a6,a8){n.CurrentLibrary.bind(a7,a6,a8);M(W,a6,{el:a7,event:a6,listener:a8})};this.reset=function(){aZ.deleteEveryEndpoint();aZ.clearListeners();aP();aZ.anchorManager.reset()};this.setAutomaticRepaint=function(a6){a1=a6};this.setDefaultScope=function(a6){I=a6};this.setDraggable=aV;this.setDebugLog=function(a6){H=a6};this.setRepaintFunction=function(a6){ae=a6};this.setMouseEventsEnabled=function(a6){aW=a6};this.setSuspendDrawing=aS;this.CANVAS="canvas";this.SVG="svg";this.VML="vml";this.setRenderMode=function(a6){if(a6){a6=a6.toLowerCase()}else{return}if(a6!==n.CANVAS&&a6!==n.SVG&&a6!==n.VML){throw new Error("render mode must be one of jsPlumb.CANVAS, jsPlumb.SVG or jsPlumb.VML")}if(a6===n.CANVAS&&w){O=n.CANVAS}else{if(a6===n.SVG&&e){O=n.SVG}else{if(a){O=n.VML}}}return O};this.getRenderMode=function(){return O};this.show=function(a6,a7){aH(a6,"block",a7)};this.sizeCanvas=function(a8,a6,ba,a7,a9){if(a8){a8.style.height=a9+"px";a8.height=a9;a8.style.width=a7+"px";a8.width=a7;a8.style.left=a6+"px";a8.style.top=ba+"px"}};this.getTestHarness=function(){return{endpointsByElement:az,endpointCount:function(a6){var a7=az[a6];return a7?a7.length:0},connectionCount:function(a6){a6=a6||I;var a7=aD[a6];return a7?a7.length:0},findIndex:p,getId:D,makeAnchor:self.makeAnchor,makeDynamicAnchor:self.makeDynamicAnchor}};this.toggle=ax;this.toggleVisible=ax;this.toggleDraggable=aT;this.unload=function(){};this.wrap=ab;this.addListener=this.bind;var a0=function(bb,a8){var a9=null,a6=bb;if(a8.tagName.toLowerCase()==="svg"&&a8.parentNode){a9=a8.parentNode}else{if(a8.offsetParent){a9=a8.offsetParent}}if(a9!=null){var a7=a9.tagName.toLowerCase()==="body"?{left:0,top:0}:s(a9),ba=a9.tagName.toLowerCase()==="body"?{left:0,top:0}:{left:a9.scrollLeft,top:a9.scrollTop};a6[0]=bb[0]-a7.left+ba.left;a6[1]=bb[1]-a7.top+ba.top}return a6};var Q=function(ba){var a8=this;this.x=ba.x||0;this.y=ba.y||0;this.elementId=ba.elementId;var a7=ba.orientation||[0,0];var a9=null,a6=null;this.offsets=ba.offsets||[0,0];a8.timestamp=null;this.compute=function(bf){var be=bf.xy,bb=bf.wh,bc=bf.element,bd=bf.timestamp;if(bd&&bd===a8.timestamp){return a6}a6=[be[0]+(a8.x*bb[0])+a8.offsets[0],be[1]+(a8.y*bb[1])+a8.offsets[1]];a6=a0(a6,bc.canvas);a8.timestamp=bd;return a6};this.getOrientation=function(bb){return a7};this.equals=function(bb){if(!bb){return false}var bc=bb.getOrientation();var bd=this.getOrientation();return this.x==bb.x&&this.y==bb.y&&this.offsets[0]==bb.offsets[0]&&this.offsets[1]==bb.offsets[1]&&bd[0]==bc[0]&&bd[1]==bc[1]};this.getCurrentLocation=function(){return a6}};var aG=function(bc){var ba=bc.reference,bb=bc.referenceCanvas,a8=b(A(bb)),a7=0,bd=0,a6=null,a9=null;this.x=0;this.y=0;this.isFloating=true;this.compute=function(bh){var bg=bh.xy,bf=bh.element,be=[bg[0]+(a8[0]/2),bg[1]+(a8[1]/2)];be=a0(be,bf.canvas);a9=be;return be};this.getOrientation=function(bf){if(a6){return a6}else{var be=ba.getOrientation(bf);return[Math.abs(be[0])*a7*-1,Math.abs(be[1])*bd*-1]}};this.over=function(be){a6=be.getOrientation()};this.out=function(){a6=null};this.getCurrentLocation=function(){return a9}};var ad=function(a8,a7,bd){this.isSelective=true;this.isDynamic=true;var bg=[],bf=this,be=function(bh){return bh.constructor==Q?bh:aZ.makeAnchor(bh,bd,aZ)};for(var bc=0;bc<a8.length;bc++){bg[bc]=be(a8[bc])}this.addAnchor=function(bh){bg.push(be(bh))};this.getAnchors=function(){return bg};this.locked=false;var a9=bg.length>0?bg[0]:null,bb=bg.length>0?0:-1,bf=this,ba=function(bj,bh,bn,bm,bi){var bl=bm[0]+(bj.x*bi[0]),bk=bm[1]+(bj.y*bi[1]);return Math.sqrt(Math.pow(bh-bl,2)+Math.pow(bn-bk,2))},a6=a7||function(br,bi,bj,bk,bh){var bm=bj[0]+(bk[0]/2),bl=bj[1]+(bk[1]/2);var bo=-1,bq=Infinity;for(var bn=0;bn<bh.length;bn++){var bp=ba(bh[bn],bm,bl,br,bi);if(bp<bq){bo=bn+0;bq=bp}}return bh[bo]};this.compute=function(bl){var bk=bl.xy,bh=bl.wh,bj=bl.timestamp,bi=bl.txy,bm=bl.twh;if(bf.locked||bi==null||bm==null){return a9.compute(bl)}else{bl.timestamp=null}a9=a6(bk,bh,bi,bm,bg);bf.x=a9.x;bf.y=a9.y;return a9.compute(bl)};this.getCurrentLocation=function(){return a9!=null?a9.getCurrentLocation():null};this.getOrientation=function(bh){return a9!=null?a9.getOrientation(bh):[0,0]};this.over=function(bh){if(a9!=null){a9.over(bh)}};this.out=function(){if(a9!=null){a9.out()}}};var aU={},Y={},ar={},K={HORIZONTAL:"horizontal",VERTICAL:"vertical",DIAGONAL:"diagonal",IDENTITY:"identity"},aX=function(bf,bg,bc,a9){if(bf===bg){return{orientation:K.IDENTITY,a:["top","top"]}}var a7=Math.atan2((a9.centery-bc.centery),(a9.centerx-bc.centerx)),ba=Math.atan2((bc.centery-a9.centery),(bc.centerx-a9.centerx)),bb=((bc.left<=a9.left&&bc.right>=a9.left)||(bc.left<=a9.right&&bc.right>=a9.right)||(bc.left<=a9.left&&bc.right>=a9.right)||(a9.left<=bc.left&&a9.right>=bc.right)),bh=((bc.top<=a9.top&&bc.bottom>=a9.top)||(bc.top<=a9.bottom&&bc.bottom>=a9.bottom)||(bc.top<=a9.top&&bc.bottom>=a9.bottom)||(a9.top<=bc.top&&a9.bottom>=bc.bottom));if(!(bb||bh)){var be=null,a8=false,a6=false,bd=null;if(a9.left>bc.left&&a9.top>bc.top){be=["right","top"]}else{if(a9.left>bc.left&&bc.top>a9.top){be=["top","left"]}else{if(a9.left<bc.left&&a9.top<bc.top){be=["top","right"]}else{if(a9.left<bc.left&&a9.top>bc.top){be=["left","top"]}}}}return{orientation:K.DIAGONAL,a:be,theta:a7,theta2:ba}}else{if(bb){return{orientation:K.HORIZONTAL,a:bc.top<a9.top?["bottom","top"]:["top","bottom"],theta:a7,theta2:ba}}else{return{orientation:K.VERTICAL,a:bc.left<a9.left?["right","left"]:["left","right"],theta:a7,theta2:ba}}}},aI=function(bk,bg,be,bf,bl,bh,a8){var bm=[],a7=bg[bl?0:1]/(bf.length+1);for(var bi=0;bi<bf.length;bi++){var bn=(bi+1)*a7,a6=bh*bg[bl?1:0];if(a8){bn=bg[bl?0:1]-bn}var bd=(bl?bn:a6),ba=be[0]+bd,bc=bd/bg[0],bb=(bl?a6:bn),a9=be[1]+bb,bj=bb/bg[1];bm.push([ba,a9,bc,bj,bf[bi][1],bf[bi][2]])}return bm},aL=function(a7,a6){return a7[0]>a6[0]?1:-1},R=function(a6){return function(a8,a7){var a9=true;if(a6){if(a8[0][0]<a7[0][0]){a9=true}else{a9=a8[0][1]>a7[0][1]}}else{if(a8[0][0]>a7[0][0]){a9=true}else{a9=a8[0][1]>a7[0][1]}}return a9===false?-1:1}},G=function(a7,a6){var a9=a7[0][0]<0?-Math.PI-a7[0][0]:Math.PI-a7[0][0],a8=a6[0][0]<0?-Math.PI-a6[0][0]:Math.PI-a6[0][0];if(a9>a8){return 1}else{return a7[0][1]>a6[0][1]?1:-1}},aF={top:aL,right:R(true),bottom:R(true),left:G},ac=function(a6,a7){return a6.sort(a7)},aa=function(a7,a6){var a9=U[a7],ba=X[a7],a8=function(bg,bn,bc,bf,bl,bk){if(bf.length>0){var bj=ac(bf,aF[bg]),bh=bg==="right"||bg==="top",bb=aI(bg,bn,bc,bj,bl,bk,bh);var bo=function(br,bq){var bp=a0([bq[0],bq[1]],br.canvas);Y[br.id]=[bp[0],bp[1],bq[2],bq[3]]};for(var bd=0;bd<bb.length;bd++){var bi=bb[bd][4],bm=bi.endpoints[0].elementId===a7,be=bi.endpoints[1].elementId===a7;if(bm){bo(bi.endpoints[0],bb[bd])}else{if(be){bo(bi.endpoints[1],bb[bd])}}}}};a8("bottom",a9,[ba.left,ba.top],a6.bottom,true,1);a8("top",a9,[ba.left,ba.top],a6.top,true,0);a8("left",a9,[ba.left,ba.top],a6.left,false,0);a8("right",a9,[ba.left,ba.top],a6.right,false,1)},an=function(){var a6={},a8={},a9={},bc=[],a7=this,bb={};this.reset=function(){a8={};a9={};bc=[];bb={}};this.newConnection=function(bg){var bi=bg.sourceId,bf=bg.targetId,bd=bg.endpoints,bh=true,be=function(bj,bk,bm,bl,bn){if(bm.constructor==ad||bm.constructor==Q){M(a8,bl,[bn,bk,bm.constructor==ad])}else{if(bi==bf){n.CurrentLibrary.removeElement(bd[1].canvas);bh=false}M(a9,bl,bn)}};be(0,bd[0],bd[0].anchor,bf,bg);if(bh){be(1,bd[1],bd[1].anchor,bi,bg)}};this.connectionDetached=function(bd){var bi=bd.sourceId,bj=bd.targetId,bm=bd.connection.endpoints,bh=function(bn,bo,bq,bp,bs){if(bq.constructor==aG){}else{if(bq.constructor==ad||bq.constructor==Q){var br=a8[bp];if(br){y(br,function(bt){return bt[0].id==bs.id})}}else{y(a9[bp],function(bt){return bt.id==bs.id})}}};bh(1,bm[1],bm[1].anchor,bi,bd.connection);bh(0,bm[0],bm[0].anchor,bj,bd.connection);var be=bd.connection.sourceId,bf=bd.connection.targetId,bl=bd.connection.endpoints[0].id,bg=bd.connection.endpoints[1].id,bk=function(bp,bn){if(bp){var bo=function(bq){return bq[4]==bn};y(bp.top,bo);y(bp.left,bo);y(bp.bottom,bo);y(bp.right,bo)}};bk(bb[be],bl);bk(bb[bf],bg);a7.redraw(be);a7.redraw(bf)};this.add=function(be,bd){M(a6,bd,be)};this.get=function(bd){return{standard:a8[bd]||[],continuous:a9[bd]||[],endpoints:a6[bd],continuousAnchorEndpoints:bc}};this.deleteEndpoint=function(be){var bd=p(bc,be);if(bd>-1){bc.splice(bd,1)}else{y(a6[be.elementId],function(bf){return bf.id==be.id})}};this.clearFor=function(bd){delete a6[bd];a6[bd]=[]};var ba=function(bx,bk,bs,bh,bn,bo,bq,bm,bz,bp,bg,bw){var bu=-1,bf=-1,bi=bh.endpoints[bq],br=bi.id,bl=[1,0][bq],bd=[[bk,bs],bh,bn,bo,br],be=bx[bz],by=bi._continuousAnchorEdge?bx[bi._continuousAnchorEdge]:null;if(by){var bv=i(by,function(bA){return bA[4]==br});if(bv!=-1){by.splice(bv,1);for(var bt=0;bt<by.length;bt++){r(bg,by[bt][1],function(bA){return bA.id==by[bt][1].id});r(bw,by[bt][1].endpoints[bq],function(bA){return bA.id==by[bt][1].endpoints[bq].id})}}}for(var bt=0;bt<be.length;bt++){if(bq==1&&be[bt][3]===bo&&bf==-1){bf=bt}r(bg,be[bt][1],function(bA){return bA.id==be[bt][1].id});r(bw,be[bt][1].endpoints[bq],function(bA){return bA.id==be[bt][1].endpoints[bq].id})}if(bu!=-1){be[bu]=bd}else{var bj=bm?bf!=-1?bf:0:be.length;be.splice(bj,0,bd)}bi._continuousAnchorEdge=bz};this.redraw=function(bp,br,bf){var bB=a6[bp]||[],bA=a8[bp]||[],bz=a9[bp]||[],be=[],by=[],bg=[];bf=bf||_timestamp();N({elId:bp,offset:br,recalc:false,timestamp:bf});var bl=X[bp],bi=U[bp],bn={};for(var bw=0;bw<bz.length;bw++){var bj=bz[bw],bk=bj.sourceId,bh=bj.targetId,bx=bk+"_"+bh,bu=bh+"_"+bk,bt=bn[bx],bm=au(bh),bd=au(bk),bo=bj.sourceId==bp?1:0;if(!bb[bk]){bb[bk]={top:[],right:[],bottom:[],left:[]}}if(!bb[bh]){bb[bh]={top:[],right:[],bottom:[],left:[]}}if(bh==bk){ba(bb[bk],-Math.PI/2,0,bj,false,bh,0,false,"top",bk,be,by)}else{if(!bt){bt=aX(bk,bh,bd.o,bm.o);bn[bx]=bt}ba(bb[bk],bt.theta,0,bj,false,bh,0,false,bt.a[0],bk,be,by);ba(bb[bh],bt.theta2,-1,bj,true,bk,1,true,bt.a[1],bh,be,by)}r(bg,bk,function(bC){return bC===bk});r(bg,bh,function(bC){return bC===bh});r(be,bj,function(bC){return bC.id==bj.id});r(by,bj.endpoints[bo],function(bC){return bC.id==bj.endpoints[bo].id})}for(var bw=0;bw<bg.length;bw++){aa(bg[bw],bb[bg[bw]])}for(var bw=0;bw<bB.length;bw++){bB[bw].paint({timestamp:bf,offset:bl,dimensions:bi})}for(var bw=0;bw<by.length;bw++){by[bw].paint({timestamp:bf,offset:bl,dimensions:bi})}for(var bw=0;bw<bA.length;bw++){var bq=bA[bw][1];if(bq.anchor.constructor==ad){bq.paint({elementWithPrecedence:bp});r(be,bA[bw][0],function(bC){return bC.id==bA[bw][0].id});for(var bv=0;bv<bq.connections.length;bv++){if(bq.connections[bv]!==bA[bw][0]){r(be,bq.connections[bv],function(bC){return bC.id==bq.connections[bv].id})}}}else{if(bq.anchor.constructor==Q){r(be,bA[bw][0],function(bC){return bC.id==bA[bw][0].id})}}}var bs=aR[bp];if(bs){bs.paint({timestamp:bf,recalc:false,elId:bp})}for(var bw=0;bw<be.length;bw++){be[bw].paint({elId:bp,timestamp:bf,recalc:false})}};this.rehomeEndpoint=function(bd,bh){var be=a6[bd]||[],bf=aZ.getId(bh);for(var bg=0;bg<be.length;bg++){a7.add(be[bg],bf)}be.splice(0,be.length)}};aZ.anchorManager=new an();aZ.continuousAnchorFactory={get:function(a7){var a6=aU[a7.elementId];if(!a6){a6={type:"Continuous",compute:function(a8){return Y[a8.element.id]||[0,0]},getCurrentLocation:function(a8){return Y[a8.id]||[0,0]},getOrientation:function(a8){return ar[a8.id]||[0,0]},isDynamic:true,isContinuous:true};aU[a7.elementId]=a6}return a6}};var aj=function(bo){var bh=this,a8=true;bh.idPrefix="_jsplumb_c_";bh.defaultLabelLocation=0.5;bh.defaultOverlayKeys=["Overlays","ConnectionOverlays"];this.parent=bo.parent;overlayCapableJsPlumbUIComponent.apply(this,arguments);this.isVisible=function(){return a8};this.setVisible=function(bq){a8=bq;if(bh.connector&&bh.connector.canvas){bh.connector.canvas.style.display=bq?"block":"none"}};this.source=A(bo.source);this.target=A(bo.target);if(bo.sourceEndpoint){this.source=bo.sourceEndpoint.endpointWillMoveTo||bo.sourceEndpoint.getElement()}if(bo.targetEndpoint){this.target=bo.targetEndpoint.getElement()}bh.previousConnection=bo.previousConnection;var be=bo.cost;bh.getCost=function(){return be};bh.setCost=function(bq){be=bq};var bc=bo.bidirectional===true;bh.isBidirectional=function(){return bc};this.sourceId=d(this.source,"id");this.targetId=d(this.target,"id");this.getAttachedElements=function(){return bh.endpoints};this.scope=bo.scope;this.endpoints=[];this.endpointStyles=[];var bn=function(br,bq){if(br){return aZ.makeAnchor(br,bq,aZ)}},bl=function(bq,bw,br,bt,bu,bs,bv){if(bq){bh.endpoints[bw]=bq;bq.addConnection(bh)}else{if(!br.endpoints){br.endpoints=[null,null]}var bC=br.endpoints[bw]||br.endpoint||aZ.Defaults.Endpoints[bw]||n.Defaults.Endpoints[bw]||aZ.Defaults.Endpoint||n.Defaults.Endpoint;if(!br.endpointStyles){br.endpointStyles=[null,null]}if(!br.endpointHoverStyles){br.endpointHoverStyles=[null,null]}var bA=br.endpointStyles[bw]||br.endpointStyle||aZ.Defaults.EndpointStyles[bw]||n.Defaults.EndpointStyles[bw]||aZ.Defaults.EndpointStyle||n.Defaults.EndpointStyle;if(bA.fillStyle==null&&bs!=null){bA.fillStyle=bs.strokeStyle}if(bA.outlineColor==null&&bs!=null){bA.outlineColor=bs.outlineColor}if(bA.outlineWidth==null&&bs!=null){bA.outlineWidth=bs.outlineWidth}var bz=br.endpointHoverStyles[bw]||br.endpointHoverStyle||aZ.Defaults.EndpointHoverStyles[bw]||n.Defaults.EndpointHoverStyles[bw]||aZ.Defaults.EndpointHoverStyle||n.Defaults.EndpointHoverStyle;if(bv!=null){if(bz==null){bz={}}if(bz.fillStyle==null){bz.fillStyle=bv.strokeStyle}}var by=br.anchors?br.anchors[bw]:br.anchor?br.anchor:bn(aZ.Defaults.Anchors[bw],bu)||bn(n.Defaults.Anchors[bw],bu)||bn(aZ.Defaults.Anchor,bu)||bn(n.Defaults.Anchor,bu),bB=br.uuids?br.uuids[bw]:null,bx=ao({paintStyle:bA,hoverPaintStyle:bz,endpoint:bC,connections:[bh],uuid:bB,anchor:by,source:bt,container:br.container,reattach:br.reattach,detachable:br.detachable});bh.endpoints[bw]=bx;if(br.drawEndpoints===false){bx.setVisible(false,true,true)}return bx}};var bj=bl(bo.sourceEndpoint,0,bo,bh.source,bh.sourceId,bo.paintStyle,bo.hoverPaintStyle);if(bj){M(az,this.sourceId,bj)}var a9=((bh.sourceId==bh.targetId)&&bo.targetEndpoint==null)?bj:bo.targetEndpoint,bi=bl(a9,1,bo,bh.target,bh.targetId,bo.paintStyle,bo.hoverPaintStyle);if(bi){M(az,this.targetId,bi)}if(!this.scope){this.scope=this.endpoints[0].scope}if(bo.deleteEndpointsOnDetach){bh.endpointsToDeleteOnDetach=[bj,bi]}var a7=aZ.Defaults.ConnectionsDetachable;if(bo.detachable===false){a7=false}if(bh.endpoints[0].connectionsDetachable===false){a7=false}if(bh.endpoints[1].connectionsDetachable===false){a7=false}if(be==null){be=bh.endpoints[0].getConnectionCost()}if(bo.bidirectional==null){bc=bh.endpoints[0].areConnectionsBidirectional()}this.isDetachable=function(){return a7===true};this.setDetachable=function(bq){a7=bq===true};var bp=n.extend({},this.endpoints[0].getParameters());n.extend(bp,this.endpoints[1].getParameters());n.extend(bp,bh.getParameters());bh.setParameters(bp);var bf=bh.setHover;bh.setHover=function(){bh.connector.setHover.apply(bh.connector,arguments);bf.apply(bh,arguments)};var bm=function(bq){if(u==null){bh.setHover(bq,false)}};this.setConnector=function(bq,br){if(bh.connector!=null){aC(bh.connector.getDisplayElements(),bh.parent)}var bs={_jsPlumb:bh._jsPlumb,parent:bo.parent,cssClass:bo.cssClass,container:bo.container,tooltip:bh.tooltip};if(bq.constructor==String){this.connector=new n.Connectors[O][bq](bs)}else{if(bq.constructor==Array){this.connector=new n.Connectors[O][bq[0]](n.extend(bq[1],bs))}}bh.canvas=bh.connector.canvas;_bindListeners(bh.connector,bh,bm);if(!br){bh.repaint()}};bh.setConnector(this.endpoints[0].connector||this.endpoints[1].connector||bo.connector||aZ.Defaults.Connector||n.Defaults.Connector,true);this.setPaintStyle(this.endpoints[0].connectorStyle||this.endpoints[1].connectorStyle||bo.paintStyle||aZ.Defaults.PaintStyle||n.Defaults.PaintStyle,true);this.setHoverPaintStyle(this.endpoints[0].connectorHoverStyle||this.endpoints[1].connectorHoverStyle||bo.hoverPaintStyle||aZ.Defaults.HoverPaintStyle||n.Defaults.HoverPaintStyle,true);this.paintStyleInUse=this.paintStyle;this.moveParent=function(bt){var bs=n.CurrentLibrary,br=bs.getParent(bh.connector.canvas);bs.removeElement(bh.connector.canvas,br);bs.appendElement(bh.connector.canvas,bt);if(bh.connector.bgCanvas){bs.removeElement(bh.connector.bgCanvas,br);bs.appendElement(bh.connector.bgCanvas,bt)}for(var bq=0;bq<bh.overlays.length;bq++){if(bh.overlays[bq].isAppendedAtTopLevel){bs.removeElement(bh.overlays[bq].canvas,br);bs.appendElement(bh.overlays[bq].canvas,bt);if(bh.overlays[bq].reattachListeners){bh.overlays[bq].reattachListeners(bh.connector)}}}if(bh.connector.reattachListeners){bh.connector.reattachListeners()}};N({elId:this.sourceId});N({elId:this.targetId});var bb=X[this.sourceId],ba=U[this.sourceId],a6=X[this.targetId],bd=U[this.targetId],bg=_timestamp(),bk=this.endpoints[0].anchor.compute({xy:[bb.left,bb.top],wh:ba,element:this.endpoints[0],elementId:this.endpoints[0].elementId,txy:[a6.left,a6.top],twh:bd,tElement:this.endpoints[1],timestamp:bg});this.endpoints[0].paint({anchorLoc:bk,timestamp:bg});bk=this.endpoints[1].anchor.compute({xy:[a6.left,a6.top],wh:bd,element:this.endpoints[1],elementId:this.endpoints[1].elementId,txy:[bb.left,bb.top],twh:ba,tElement:this.endpoints[0],timestamp:bg});this.endpoints[1].paint({anchorLoc:bk,timestamp:bg});this.paint=function(bH){bH=bH||{};var by=bH.elId,bz=bH.ui,bw=bH.recalc,br=bH.timestamp,bA=false,bG=bA?this.sourceId:this.targetId,bv=bA?this.targetId:this.sourceId,bs=bA?0:1,bI=bA?1:0;var bJ=N({elId:by,offset:bz,recalc:bw,timestamp:br}),bx=N({elId:bG,timestamp:br});var bC=this.endpoints[bI],bq=this.endpoints[bs],bu=bC.anchor.getCurrentLocation(bC),bF=bq.anchor.getCurrentLocation(bq);var bt=0;for(var bE=0;bE<bh.overlays.length;bE++){var bB=bh.overlays[bE];if(bB.isVisible()){bt=Math.max(bt,bB.computeMaxSize(bh.connector))}}var bD=this.connector.compute(bu,bF,this.endpoints[bI],this.endpoints[bs],this.endpoints[bI].anchor,this.endpoints[bs].anchor,bh.paintStyleInUse.lineWidth,bt,bJ,bx);bh.connector.paint(bD,bh.paintStyleInUse);for(var bE=0;bE<bh.overlays.length;bE++){var bB=bh.overlays[bE];if(bB.isVisible){bh.overlayPlacements[bE]=bB.draw(bh.connector,bh.paintStyleInUse,bD)}}};this.repaint=function(br){br=br||{};var bq=!(br.recalc===false);this.paint({elId:this.sourceId,recalc:bq,timestamp:br.timestamp})};bh.repaint()};var aO=function(a7){var a6=false;return{drag:function(){if(a6){return true}var a8=n.CurrentLibrary.getUIPosition(arguments),a9=a7.element;if(a9){n.CurrentLibrary.setOffset(a9,a8);aM(A(a9),a8)}},stopDrag:function(){a6=true}}};var af=function(ba,a9,bb,a8,a6){var a7=new aG({reference:a9,referenceCanvas:a8});return ao({paintStyle:ba,endpoint:bb,anchor:a7,source:a6,scope:"__floating"})};var J=function(a8,a6){var ba=document.createElement("div");ba.style.position="absolute";var a7=A(ba);aB(ba,a6);var a9=D(a7);N({elId:a9});a8.id=a9;a8.element=a7};var V=function(bA){var bp=this;bp.idPrefix="_jsplumb_e_";bp.defaultLabelLocation=[0.5,0.5];bp.defaultOverlayKeys=["Overlays","EndpointOverlays"];this.parent=bA.parent;overlayCapableJsPlumbUIComponent.apply(this,arguments);bA=bA||{};var ba=true;this.isVisible=function(){return ba};this.setVisible=function(bD,bG,bC){ba=bD;if(bp.canvas){bp.canvas.style.display=bD?"block":"none"}if(!bG){for(var bF=0;bF<bp.connections.length;bF++){bp.connections[bF].setVisible(bD);if(!bC){var bE=bp===bp.connections[bF].endpoints[0]?1:0;if(bp.connections[bF].endpoints[bE].connections.length==1){bp.connections[bF].endpoints[bE].setVisible(bD,true,true)}}}}};var bo=bA.source,bi=bA.uuid,by=null,bc=null;if(bi){aA[bi]=bp}var bf=d(bo,"id");this.elementId=bf;this.element=bo;var a8=bA.connectionCost;this.getConnectionCost=function(){return a8};this.setConnectionCost=function(bC){a8=bC};var bx=bA.connectionsBidirectional===true;this.areConnectionsBidirectional=function(){return bx};this.setConnectionsBidirectional=function(bC){bx=bC};bp.anchor=bA.anchor?aZ.makeAnchor(bA.anchor,bf,aZ):bA.anchors?aZ.makeAnchor(bA.anchors,bf,aZ):aZ.makeAnchor("TopCenter",bf,aZ);if(!bA._transient){aZ.anchorManager.add(bp,bf)}var bm=bA.endpoint||aZ.Defaults.Endpoint||n.Defaults.Endpoint||"Dot",bg={_jsPlumb:bp._jsPlumb,parent:bA.parent,container:bA.container,tooltip:bA.tooltip,connectorTooltip:bA.connectorTooltip,endpoint:bp};if(bm.constructor==String){bm=new n.Endpoints[O][bm](bg)}else{if(bm.constructor==Array){bg=n.extend(bm[1],bg);bm=new n.Endpoints[O][bm[0]](bg)}else{bm=bm.clone()}}var bj=n.extend({},bg);bm.clone=function(){var bC=new Object();bm.constructor.apply(bC,[bj]);return bC};bp.endpoint=bm;bp.type=bp.endpoint.type;var bn=bp.setHover;bp.setHover=function(){bp.endpoint.setHover.apply(bp.endpoint,arguments);bn.apply(bp,arguments)};var bB=function(bC){if(bp.connections.length>0){bp.connections[0].setHover(bC,false)}else{bp.setHover(bC)}};_bindListeners(bp.endpoint,bp,bB);this.setPaintStyle(bA.paintStyle||bA.style||aZ.Defaults.EndpointStyle||n.Defaults.EndpointStyle,true);this.setHoverPaintStyle(bA.hoverPaintStyle||aZ.Defaults.EndpointHoverStyle||n.Defaults.EndpointHoverStyle,true);this.paintStyleInUse=this.paintStyle;this.connectorStyle=bA.connectorStyle;this.connectorHoverStyle=bA.connectorHoverStyle;this.connectorOverlays=bA.connectorOverlays;this.connector=bA.connector;this.connectorTooltip=bA.connectorTooltip;this.isSource=bA.isSource||false;this.isTarget=bA.isTarget||false;var bu=bA.maxConnections||aZ.Defaults.MaxConnections;this.getAttachedElements=function(){return bp.connections};this.canvas=this.endpoint.canvas;this.connections=bA.connections||[];this.scope=bA.scope||I;this.timestamp=null;bp.isReattach=bA.reattach||false;bp.connectionsDetachable=aZ.Defaults.ConnectionsDetachable;if(bA.connectionsDetachable===false||bA.detachable===false){bp.connectionsDetachable=false}var bk=bA.dragAllowedWhenFull||true;this.computeAnchor=function(bC){return bp.anchor.compute(bC)};this.addConnection=function(bC){bp.connections.push(bC)};this.detach=function(bC,bG,bD,bK){var bJ=i(bp.connections,function(bM){return bM.id==bC.id}),bI=false;bK=(bK!==false);if(bJ>=0){if(bD||bC._forceDetach||bC.isDetachable()||bC.isDetachAllowed(bC)){var bL=bC.endpoints[0]==bp?bC.endpoints[1]:bC.endpoints[0];if(bD||bC._forceDetach||(bp.isDetachAllowed(bC))){bp.connections.splice(bJ,1);if(!bG){bL.detach(bC,true,bD);if(bC.endpointsToDeleteOnDetach){for(var bH=0;bH<bC.endpointsToDeleteOnDetach.length;bH++){var bE=bC.endpointsToDeleteOnDetach[bH];if(bE&&bE.connections.length==0){aZ.deleteEndpoint(bE)}}}}aC(bC.connector.getDisplayElements(),bC.parent);y(aD[bC.scope],function(bM){return bM.id==bC.id});bI=true;var bF=(!bG&&bK);aK(bC,bF)}}}return bI};this.detachAll=function(bC){while(bp.connections.length>0){bp.detach(bp.connections[0],false,true,bC)}};this.detachFrom=function(bE,bD){var bF=[];for(var bC=0;bC<bp.connections.length;bC++){if(bp.connections[bC].endpoints[1]==bE||bp.connections[bC].endpoints[0]==bE){bF.push(bp.connections[bC])}}for(var bC=0;bC<bF.length;bC++){if(bp.detach(bF[bC],false,true,bD)){bF[bC].setHover(false,false)}}};this.detachFromConnection=function(bD){var bC=p(bp.connections,bD);if(bC>=0){bp.connections.splice(bC,1)}};this.getElement=function(){return bo};this.setElement=function(bE){var bG=D(bE);y(az[bf],function(bH){return bH.id==bp.id});bo=A(bE);bf=D(bo);bp.elementId=bf;var bF=ai({source:bG}),bD=a9.getParent(bp.canvas);a9.removeElement(bp.canvas,bD);a9.appendElement(bp.canvas,bF);for(var bC=0;bC<bp.connections.length;bC++){bp.connections[bC].moveParent(bF);bp.connections[bC].sourceId=bf;bp.connections[bC].source=bo}M(az,bG,bp);aZ.repaint(bG)};this.getUuid=function(){return bi};this.makeInPlaceCopy=function(){return ao({anchor:bp.anchor,source:bo,paintStyle:this.paintStyle,endpoint:bm,_transient:true,scope:bp.scope})};this.isConnectedTo=function(bE){var bD=false;if(bE){for(var bC=0;bC<bp.connections.length;bC++){if(bp.connections[bC].endpoints[1]==bE){bD=true;break}}}return bD};this.isFloating=function(){return by!=null};this.connectorSelector=function(){var bC=bp.connections[0];if(bp.isTarget&&bC){return bC}else{return(bp.connections.length<bu)||bu==-1?null:bC}};this.isFull=function(){return !(bp.isFloating()||bu<1||bp.connections.length<bu)};this.setDragAllowedWhenFull=function(bC){bk=bC};this.setStyle=bp.setPaintStyle;this.equals=function(bC){return this.anchor.equals(bC.anchor)};var bl=function(bD){var bC=0;if(bD!=null){for(var bE=0;bE<bp.connections.length;bE++){if(bp.connections[bE].sourceId==bD||bp.connections[bE].targetId==bD){bC=bE;break}}}return bp.connections[bC]};this.paint=function(bF){bF=bF||{};var bL=bF.timestamp,bK=!(bF.recalc===false);if(!bL||bp.timestamp!==bL){N({elId:bf,timestamp:bL,recalc:bK});var bR=bF.offset||X[bf];if(bR){var bI=bF.anchorPoint,bG=bF.connectorPaintStyle;if(bI==null){var bC=bF.dimensions||U[bf];if(bR==null||bC==null){N({elId:bf,timestamp:bL});bR=X[bf];bC=U[bf]}var bE={xy:[bR.left,bR.top],wh:bC,element:bp,timestamp:bL};if(bK&&bp.anchor.isDynamic&&bp.connections.length>0){var bO=bl(bF.elementWithPrecedence),bQ=bO.endpoints[0]==bp?1:0,bH=bQ==0?bO.sourceId:bO.targetId,bN=X[bH],bP=U[bH];bE.txy=[bN.left,bN.top];bE.twh=bP;bE.tElement=bO.endpoints[bQ]}bI=bp.anchor.compute(bE)}var bM=bm.compute(bI,bp.anchor.getOrientation(bm),bp.paintStyleInUse,bG||bp.paintStyleInUse);bm.paint(bM,bp.paintStyleInUse,bp.anchor);bp.timestamp=bL;for(var bJ=0;bJ<bp.overlays.length;bJ++){var bD=bp.overlays[bJ];if(bD.isVisible){bp.overlayPlacements[bJ]=bD.draw(bp.endpoint,bp.paintStyleInUse,bM)}}}}};this.repaint=this.paint;this.removeConnection=this.detach;if(n.CurrentLibrary.isDragSupported(bo)){var bt={id:null,element:null},bs=null,a7=false,bb=null,a6=aO(bt);var bd=function(){bs=bp.connectorSelector();var bC=true;if(bs==null&&!bA.isSource){bC=false}if(bA.isSource&&bp.isFull()&&!bk){bC=false}if(bs!=null&&!bs.isDetachable()){bC=false}if(bC===false){if(n.CurrentLibrary.stopDrag){n.CurrentLibrary.stopDrag()}a6.stopDrag();return false}if(bs&&!bp.isFull()&&bA.isSource){bs=null}N({elId:bf});bc=bp.makeInPlaceCopy();bc.paint();J(bt,bp.parent);var bI=A(bc.canvas),bG=n.CurrentLibrary.getOffset(bI),bD=a0([bG.left,bG.top],bc.canvas);n.CurrentLibrary.setOffset(bt.element,{left:bD[0],top:bD[1]});if(bp.parentAnchor){bp.anchor=aZ.makeAnchor(bp.parentAnchor,bp.elementId,aZ)}f(A(bp.canvas),"dragId",bt.id);f(A(bp.canvas),"elId",bf);by=af(bp.paintStyle,bp.anchor,bm,bp.canvas,bt.element);if(bs==null){bp.anchor.locked=true;bp.setHover(false,false);bs=T({sourceEndpoint:bp,targetEndpoint:by,source:bp.endpointWillMoveTo||A(bo),target:bt.element,anchors:[bp.anchor,by.anchor],paintStyle:bA.connectorStyle,hoverPaintStyle:bA.connectorHoverStyle,connector:bA.connector,overlays:bA.connectorOverlays})}else{a7=true;bs.connector.setHover(false,false);be(A(bc.canvas),false,true);var bF=bs.endpoints[0].id==bp.id?0:1;bs.floatingAnchorIndex=bF;bp.detachFromConnection(bs);var bJ=A(bp.canvas),bH=n.CurrentLibrary.getDragScope(bJ);f(bJ,"originalScope",bH);var bE=n.CurrentLibrary.getDropScope(bJ);n.CurrentLibrary.setDragScope(bJ,bE);if(bF==0){bb=[bs.source,bs.sourceId,bw,bH];bs.source=bt.element;bs.sourceId=bt.id}else{bb=[bs.target,bs.targetId,bw,bH];bs.target=bt.element;bs.targetId=bt.id}bs.endpoints[bF==0?1:0].anchor.locked=true;bs.suspendedEndpoint=bs.endpoints[bF];bs.suspendedEndpoint.setHover(false);bs.endpoints[bF]=by}aR[bt.id]=bs;by.addConnection(bs);M(az,bt.id,by);aZ.currentlyDragging=true};var a9=n.CurrentLibrary,bv=bA.dragOptions||{},bq=n.extend({},a9.defaultDragOptions),br=a9.dragEvents.start,bz=a9.dragEvents.stop,bh=a9.dragEvents.drag;bv=n.extend(bq,bv);bv.scope=bv.scope||bp.scope;bv[br]=ab(bv[br],bd);bv[bh]=ab(bv[bh],a6.drag);bv[bz]=ab(bv[bz],function(){aZ.currentlyDragging=false;y(az[bt.id],function(bD){return bD.id==by.id});aC([bt.element[0],by.canvas],bo);ag(bc.canvas,bo);aZ.anchorManager.clearFor(bt.id);var bC=bs.floatingAnchorIndex==null?1:bs.floatingAnchorIndex;bs.endpoints[bC==0?1:0].anchor.locked=false;if(bs.endpoints[bC]==by){if(a7&&bs.suspendedEndpoint){if(bC==0){bs.source=bb[0];bs.sourceId=bb[1]}else{bs.target=bb[0];bs.targetId=bb[1]}n.CurrentLibrary.setDragScope(bb[2],bb[3]);bs.endpoints[bC]=bs.suspendedEndpoint;if(bp.isReattach||bs._forceDetach||!bs.endpoints[bC==0?1:0].detach(bs)){bs.setHover(false);bs.floatingAnchorIndex=null;bs.suspendedEndpoint.addConnection(bs);aZ.repaint(bb[1])}bs._forceDetach=null}else{aC(bs.connector.getDisplayElements(),bp.parent);bp.detachFromConnection(bs)}}bp.anchor.locked=false;bp.paint({recalc:false});bs.setHover(false,false);bs=null;bc=null;delete az[by.elementId];by.anchor=null;by=null;aZ.currentlyDragging=false});var bw=A(bp.canvas);n.CurrentLibrary.initDraggable(bw,bv,true)}var be=function(bG,bJ,bE){if((bA.isTarget||bJ)&&n.CurrentLibrary.isDropSupported(bo)){var bC=bA.dropOptions||aZ.Defaults.DropOptions||n.Defaults.DropOptions;bC=n.extend({},bC);bC.scope=bC.scope||bp.scope;var bH=n.CurrentLibrary.dragEvents.drop,bI=n.CurrentLibrary.dragEvents.over,bD=n.CurrentLibrary.dragEvents.out,bF=function(bK){var bU=A(n.CurrentLibrary.getDragObject(arguments)),bL=d(bU,"dragId"),bN=d(bU,"elId"),bT=d(bU,"originalScope"),bQ=aR[bL],bR=bQ.floatingAnchorIndex==null?1:bQ.floatingAnchorIndex,bS=bR==0?1:0;if(bT){n.CurrentLibrary.setDragScope(bU,bT)}if(!bp.isFull()&&!(bR==0&&!bp.isSource)&&!(bR==1&&!bp.isTarget)){var bO=true;if(bQ.suspendedEndpoint&&bQ.suspendedEndpoint.id!=bp.id){if(!bQ.isDetachAllowed(bQ)||!bQ.endpoints[bR].isDetachAllowed(bQ)||!bQ.suspendedEndpoint.isDetachAllowed(bQ)||!aZ.checkCondition("beforeDetach",bQ)){bO=false}}if(bR==0){bQ.source=bo;bQ.sourceId=bf}else{bQ.target=bo;bQ.targetId=bf}bO=bO&&bp.isDropAllowed(bQ.sourceId,bQ.targetId,bQ.scope);if(bO){bQ.endpoints[bR].detachFromConnection(bQ);if(bQ.suspendedEndpoint){bQ.suspendedEndpoint.detachFromConnection(bQ)}bQ.endpoints[bR]=bp;bp.addConnection(bQ);if(!bQ.suspendedEndpoint){aN(bo,bA.draggable,{})}else{var bP=bQ.suspendedEndpoint.getElement(),bM=bQ.suspendedEndpoint.elementId;aK({source:bR==0?bP:bQ.source,target:bR==1?bP:bQ.target,sourceId:bR==0?bM:bQ.sourceId,targetId:bR==1?bM:bQ.targetId,sourceEndpoint:bR==0?bQ.suspendedEndpoint:bQ.endpoints[0],targetEndpoint:bR==1?bQ.suspendedEndpoint:bQ.endpoints[1],connection:bQ},true)}a4(bQ,null,bK)}else{if(bQ.suspendedEndpoint){bQ.endpoints[bR]=bQ.suspendedEndpoint;bQ.setHover(false);bQ._forceDetach=true;if(bR==0){bQ.source=bQ.suspendedEndpoint.element;bQ.sourceId=bQ.suspendedEndpoint.elementId}else{bQ.target=bQ.suspendedEndpoint.element;bQ.targetId=bQ.suspendedEndpoint.elementId}bQ.suspendedEndpoint.addConnection(bQ);bQ.endpoints[0].repaint();bQ.repaint();aZ.repaint(bQ.source.elementId);bQ._forceDetach=false}}bQ.floatingAnchorIndex=null}aZ.currentlyDragging=false;delete aR[bL]};bC[bH]=ab(bC[bH],bF);bC[bI]=ab(bC[bI],function(){if(bp.isTarget){var bL=n.CurrentLibrary.getDragObject(arguments),bN=d(A(bL),"dragId"),bM=aR[bN];if(bM!=null){var bK=bM.floatingAnchorIndex==null?1:bM.floatingAnchorIndex;bM.endpoints[bK].anchor.over(bp.anchor)}}});bC[bD]=ab(bC[bD],function(){if(bp.isTarget){var bL=n.CurrentLibrary.getDragObject(arguments),bN=d(A(bL),"dragId"),bM=aR[bN];if(bM!=null){var bK=bM.floatingAnchorIndex==null?1:bM.floatingAnchorIndex;bM.endpoints[bK].anchor.out()}}});n.CurrentLibrary.initDroppable(bG,bC,true,bE)}};be(A(bp.canvas),true,!(bA._transient||bp.anchor.isFloating));return bp}};var n=window.jsPlumb=new v();n.getInstance=function(E){var D=new v(E);D.init();return D};n.util={convertStyle:function(E,D){if("transparent"===E){return E}var J=E,I=function(K){return K.length==1?"0"+K:K},F=function(K){return I(Number(K).toString(16))},G=/(rgb[a]?\()(.*)(\))/;if(E.match(G)){var H=E.match(G)[2].split(",");J="#"+F(H[0])+F(H[1])+F(H[2]);if(!D&&H.length==4){J=J+F(H[3])}}return J},gradient:function(E,D){E=E.constructor==Array?E:[E.x,E.y];D=D.constructor==Array?D:[D.x,D.y];return(D[1]-E[1])/(D[0]-E[0])},normal:function(E,D){return -1/n.util.gradient(E,D)},segment:function(E,D){E=E.constructor==Array?E:[E.x,E.y];D=D.constructor==Array?D:[D.x,D.y];if(D[0]>E[0]){return(D[1]>E[1])?2:1}else{return(D[1]>E[1])?3:4}},segmentMultipliers:[null,[1,-1],[1,1],[-1,1],[-1,-1]],inverseSegmentMultipliers:[null,[-1,-1],[-1,1],[1,1],[1,-1]],pointOnLine:function(D,H,E){var G=n.util.gradient(D,H),L=n.util.segment(D,H),K=E>0?n.util.segmentMultipliers[L]:n.util.inverseSegmentMultipliers[L],F=Math.atan(G),I=Math.abs(E*Math.sin(F))*K[1],J=Math.abs(E*Math.cos(F))*K[0];return{x:D.x+J,y:D.y+I}},perpendicularLineTo:function(F,G,H){var E=n.util.gradient(F,G),I=Math.atan(-1/E),J=H/2*Math.sin(I),D=H/2*Math.cos(I);return[{x:G.x+D,y:G.y+J},{x:G.x-D,y:G.y-J}]}};var q=function(D,I,F,E,H,G){return function(K){K=K||{};var J=K.jsPlumbInstance.makeAnchor([D,I,F,E,0,0],K.elementId,K.jsPlumbInstance);J.type=H;if(G){G(J,K)}return J}};n.Anchors.TopCenter=q(0.5,0,0,-1,"TopCenter");n.Anchors.BottomCenter=q(0.5,1,0,1,"BottomCenter");n.Anchors.LeftMiddle=q(0,0.5,-1,0,"LeftMiddle");n.Anchors.RightMiddle=q(1,0.5,1,0,"RightMiddle");n.Anchors.Center=q(0.5,0.5,0,0,"Center");n.Anchors.TopRight=q(1,0,0,-1,"TopRight");n.Anchors.BottomRight=q(1,1,0,1,"BottomRight");n.Anchors.TopLeft=q(0,0,0,-1,"TopLeft");n.Anchors.BottomLeft=q(0,1,0,1,"BottomLeft");n.Defaults.DynamicAnchors=function(D){return D.jsPlumbInstance.makeAnchors(["TopCenter","RightMiddle","BottomCenter","LeftMiddle"],D.elementId,D.jsPlumbInstance)};n.Anchors.AutoDefault=function(E){var D=E.jsPlumbInstance.makeDynamicAnchor(n.Defaults.DynamicAnchors(E));D.type="AutoDefault";return D};n.Anchors.Assign=q(0,0,0,0,"Assign",function(E,F){var D=F.position||"Fixed";E.positionFinder=D.constructor==String?F.jsPlumbInstance.AnchorPositionFinders[D]:D;E.constructorParams=F});n.Anchors.Continuous=function(D){return D.jsPlumbInstance.continuousAnchorFactory.get(D)};n.AnchorPositionFinders={Fixed:function(G,E,F,D){return[(G.left-E.left)/F[0],(G.top-E.top)/F[1]]},Grid:function(D,M,H,E){var L=D.left-M.left,K=D.top-M.top,J=H[0]/(E.constructorParams.grid[0]),I=H[1]/(E.constructorParams.grid[1]),G=Math.floor(L/J),F=Math.floor(K/I);return[((G*J)+(J/2))/H[0],((F*I)+(I/2))/H[1]]}}})();(function(){jsPlumb.DOMElementComponent=function(c){jsPlumb.jsPlumbUIComponent.apply(this,arguments);this.mousemove=this.dblclick=this.click=this.mousedown=this.mouseup=function(d){}};jsPlumb.Connectors.Straight=function(){this.type="Straight";var r=this,i=null,e,k,p,n,l,f,q,h,g,d,c,o,m;this.compute=function(A,J,s,z,F,t,D,v){var I=Math.abs(A[0]-J[0]),C=Math.abs(A[1]-J[1]),B=0.45*I,u=0.45*C;I*=1.9;C*=1.9;var G=Math.min(A[0],J[0])-B;var E=Math.min(A[1],J[1])-u;var H=Math.max(2*D,v);if(I<H){I=H;G=A[0]+((J[0]-A[0])/2)-(H/2);B=(I-Math.abs(A[0]-J[0]))/2}if(C<H){C=H;E=A[1]+((J[1]-A[1])/2)-(H/2);u=(C-Math.abs(A[1]-J[1]))/2}h=A[0]<J[0]?B:I-B;g=A[1]<J[1]?u:C-u;d=A[0]<J[0]?I-B:B;c=A[1]<J[1]?C-u:u;i=[G,E,I,C,h,g,d,c];n=d-h,l=c-g;e=jsPlumb.util.gradient({x:h,y:g},{x:d,y:c}),k=-1/e;p=-1*((e*h)-g);f=Math.atan(e);q=Math.atan(k);m=Math.sqrt((n*n)+(l*l));return i};this.pointOnPath=function(s){if(s==0){return{x:h,y:g}}else{if(s==1){return{x:d,y:c}}else{return jsPlumb.util.pointOnLine({x:h,y:g},{x:d,y:c},s*m)}}};this.gradientAtPoint=function(s){return e};this.pointAlongPathFrom=function(s,u){var t=r.pointOnPath(s);return jsPlumb.util.pointOnLine(t,{x:d,y:c},u)}};jsPlumb.Connectors.Bezier=function(g){var q=this;g=g||{};this.majorAnchor=g.curviness||150;this.minorAnchor=10;var k=null;this.type="Bezier";this._findControlPoint=function(C,r,x,s,v,A,t){var z=A.getOrientation(s),B=t.getOrientation(v),w=z[0]!=B[0]||z[1]==B[1],u=[],D=q.majorAnchor,y=q.minorAnchor;if(!w){if(z[0]==0){u.push(r[0]<x[0]?C[0]+y:C[0]-y)}else{u.push(C[0]-(D*z[0]))}if(z[1]==0){u.push(r[1]<x[1]?C[1]+y:C[1]-y)}else{u.push(C[1]+(D*B[1]))}}else{if(B[0]==0){u.push(x[0]<r[0]?C[0]+y:C[0]-y)}else{u.push(C[0]+(D*B[0]))}if(B[1]==0){u.push(x[1]<r[1]?C[1]+y:C[1]-y)}else{u.push(C[1]+(D*z[1]))}}return u};var p,o,l,d,c,l,h,f,e,n,i;this.compute=function(N,u,H,v,L,s,r,G){r=r||0;n=Math.abs(N[0]-u[0])+r;i=Math.abs(N[1]-u[1])+r;f=Math.min(N[0],u[0])-(r/2);e=Math.min(N[1],u[1])-(r/2);l=N[0]<u[0]?n-(r/2):(r/2);h=N[1]<u[1]?i-(r/2):(r/2);d=N[0]<u[0]?(r/2):n-(r/2);c=N[1]<u[1]?(r/2):i-(r/2);p=q._findControlPoint([l,h],N,u,H,v,L,s);o=q._findControlPoint([d,c],u,N,H,v,s,L);var F=Math.min(l,d),E=Math.min(p[0],o[0]),A=Math.min(F,E),M=Math.max(l,d),J=Math.max(p[0],o[0]),x=Math.max(M,J);if(x>n){n=x}if(A<0){f+=A;var C=Math.abs(A);n+=C;p[0]+=C;l+=C;d+=C;o[0]+=C}var K=Math.min(h,c),I=Math.min(p[1],o[1]),w=Math.min(K,I),B=Math.max(h,c),z=Math.max(p[1],o[1]),t=Math.max(B,z);if(t>i){i=t}if(w<0){e+=w;var y=Math.abs(w);i+=y;p[1]+=y;h+=y;c+=y;o[1]+=y}if(G&&n<G){var D=(G-n)/2;n=G;f-=D;l=l+D;d=d+D;p[0]=p[0]+D;o[0]=o[0]+D}if(G&&i<G){var D=(G-i)/2;i=G;e-=D;h=h+D;c=c+D;p[1]=p[1]+D;o[1]=o[1]+D}k=[f,e,n,i,l,h,d,c,p[0],p[1],o[0],o[1]];return k};var m=function(){return[{x:l,y:h},{x:p[0],y:p[1]},{x:o[0],y:o[1]},{x:d,y:c}]};this.pointOnPath=function(r){return jsBezier.pointOnCurve(m(),r)};this.gradientAtPoint=function(r){return jsBezier.gradientAtPoint(m(),r)};this.pointAlongPathFrom=function(r,s){return jsBezier.pointAlongCurveFrom(m(),r,s)}};jsPlumb.Connectors.Flowchart=function(h){this.type="Flowchart";h=h||{};var p=this,d=h.stub||h.minStubLength||30,k=[],o=0,m=[],c=[],n=[],f,e,i=function(s,r,w,v){var u=0;for(var t=0;t<k.length;t++){c[t]=k[t][5]/o;m[t]=[u,(u+=(k[t][5]/o))]}},q=function(){n.push(k.length);for(var r=0;r<k.length;r++){n.push(k[r][0]);n.push(k[r][1])}},g=function(C,z,B,A,w,v){var s=k.length==0?B:k[k.length-1][0],r=k.length==0?A:k[k.length-1][1],t=C==s?Infinity:0,u=Math.abs(C==s?z-r:C-s);k.push([C,z,s,r,t,u]);o+=u},l=function(t){var r=m.length-1,s=0;for(var u=0;u<m.length;u++){if(m[u][1]>=t){r=u;s=(t-m[u][0])/c[u];break}}return{segment:k[r],proportion:s,index:r}};this.compute=function(Q,ae,r,J,ao,D,O,I,aj,ag){k=[];o=0;c=[];f=ae[0]<Q[0];e=ae[1]<Q[1];var U=O||1,B=(U/2)+(d*2),z=(U/2)+(d*2),G=ao.orientation||ao.getOrientation(r),ap=D.orientation||D.getOrientation(J),ad=f?ae[0]:Q[0],ac=e?ae[1]:Q[1],af=Math.abs(ae[0]-Q[0])+2*B,an=Math.abs(ae[1]-Q[1])+2*z;if(G[0]==0&&G[1]==0||ap[0]==0&&ap[1]==0){var W=af>an?0:1,Y=[1,0][W];G=[];ap=[];G[W]=Q[W]>ae[W]?-1:1;ap[W]=Q[W]>ae[W]?1:-1;G[Y]=0;ap[Y]=0}if(af<I){B+=(I-af)/2;af=I}if(an<I){z+=(I-an)/2;an=I}var A=f?af-B:B,v=e?an-z:z,al=f?B:af-B,ak=e?z:an-z,T=A+(G[0]*d),S=v+(G[1]*d),E=al+(ap[0]*d),C=ak+(ap[1]*d),P=Math.abs(A-al)>2*d,R=Math.abs(v-ak)>2*d,ab=T+((E-T)/2),Z=S+((C-S)/2),H=((G[0]*ap[0])+(G[1]*ap[1])),V=H==-1,X=H==0,s=H==1;ad-=B;ac-=z;n=[ad,ac,af,an,A,v,al,ak];var ai=[];g(T,S,A,v,al,ak);var L=G[0]==0?"y":"x",F=V?"opposite":s?"orthogonal":"perpendicular",t=jsPlumb.util.segment([A,v],[al,ak]),aa=G[L=="x"?0:1]==-1,K={x:[null,4,3,2,1],y:[null,2,1,4,3]};if(aa){t=K[L][t]}var N=function(aq,y,w,x){return aq+(y*((1-w)*x)+d)},u={oppositex:function(){if(r.elementId==J.elementId){var w=S+((1-ao.y)*aj.height)+d;return[[T,w],[E,w]]}else{if(P&&(t==1||t==2)){return[[ab,v],[ab,ak]]}else{return[[T,Z],[E,Z]]}}},orthogonalx:function(){if(t==1||t==2){return[[E,S]]}else{return[[T,C]]}},perpendicularx:function(){var w=(ak+v)/2;if((t==1&&ap[1]==1)||(t==2&&ap[1]==-1)){if(Math.abs(al-A)>d){return[[E,S]]}else{return[[T,S],[T,w],[E,w]]}}else{if((t==3&&ap[1]==-1)||(t==4&&ap[1]==1)){return[[T,w],[E,w]]}else{if((t==3&&ap[1]==1)||(t==4&&ap[1]==-1)){return[[T,C]]}else{if((t==1&&ap[1]==-1)||(t==2&&ap[1]==1)){if(Math.abs(al-A)>d){return[[ab,S],[ab,C]]}else{return[[T,C]]}}}}}},oppositey:function(){if(r.elementId==J.elementId){var w=T+((1-ao.x)*aj.width)+d;return[[w,S],[w,C]]}else{if(R&&(t==2||t==3)){return[[A,Z],[al,Z]]}else{return[[ab,S],[ab,C]]}}},orthogonaly:function(){if(t==2||t==3){return[[T,C]]}else{return[[E,S]]}},perpendiculary:function(){var w=(al+A)/2;if((t==2&&ap[0]==-1)||(t==3&&ap[0]==1)){if(Math.abs(al-A)>d){return[[T,C]]}else{return[[T,Z],[E,Z]]}}else{if((t==1&&ap[0]==-1)||(t==4&&ap[0]==1)){var w=(al+A)/2;return[[w,S],[w,C]]}else{if((t==1&&ap[0]==1)||(t==4&&ap[0]==-1)){return[[E,S]]}else{if((t==2&&ap[0]==1)||(t==3&&ap[0]==-1)){if(Math.abs(ak-v)>d){return[[T,Z],[E,Z]]}else{return[[E,S]]}}}}}}};var M=u[F+L];var ah=M();if(ah){for(var am=0;am<ah.length;am++){g(ah[am][0],ah[am][1],A,v,al,ak)}}g(E,C,A,v,al,ak);g(al,ak,A,v,al,ak);q();i(A,v,al,ak);return n};this.pointOnPath=function(r){return p.pointAlongPathFrom(r,0)};this.gradientAtPoint=function(r){return k[l(r)["index"]][4]};this.pointAlongPathFrom=function(v,z){var w=l(v),u=w.segment,y=w.proportion,t=k[w.index][5],r=k[w.index][4];var x={x:r==Infinity?u[2]:u[2]>u[0]?u[0]+((1-y)*t)-z:u[2]+(y*t)+z,y:r==0?u[3]:u[3]>u[1]?u[1]+((1-y)*t)-z:u[3]+(y*t)+z,segmentInfo:w};return x}};jsPlumb.Endpoints.Dot=function(d){this.type="Dot";var c=this;d=d||{};this.radius=d.radius||10;this.defaultOffset=0.5*this.radius;this.defaultInnerRadius=this.radius/3;this.compute=function(i,f,l,h){var g=l.radius||c.radius,e=i[0]-g,k=i[1]-g;return[e,k,g*2,g*2,g]}};jsPlumb.Endpoints.Rectangle=function(d){this.type="Rectangle";var c=this;d=d||{};this.width=d.width||20;this.height=d.height||20;this.compute=function(k,g,m,i){var h=m.width||c.width,f=m.height||c.height,e=k[0]-(h/2),l=k[1]-(f/2);return[e,l,h,f]}};var a=function(e){jsPlumb.DOMElementComponent.apply(this,arguments);var c=this;var d=[];this.getDisplayElements=function(){return d};this.appendDisplayElement=function(f){d.push(f)}};jsPlumb.Endpoints.Image=function(g){this.type="Image";a.apply(this,arguments);var l=this,f=false,e=g.width,d=g.height,i=null,c=g.endpoint;this.img=new Image();l.ready=false;this.img.onload=function(){l.ready=true;e=e||l.img.width;d=d||l.img.height;if(i){i(l)}};c.setImage=function(m,o){var n=m.constructor==String?m:m.src;i=o;l.img.src=m};c.setImage(g.src||g.url,g.onload);this.compute=function(o,m,p,n){l.anchorPoint=o;if(l.ready){return[o[0]-e/2,o[1]-d/2,e,d]}else{return[0,0,0,0]}};l.canvas=document.createElement("img"),f=false;l.canvas.style.margin=0;l.canvas.style.padding=0;l.canvas.style.outline=0;l.canvas.style.position="absolute";var h=g.cssClass?" "+g.cssClass:"";l.canvas.className=jsPlumb.endpointClass+h;if(e){l.canvas.setAttribute("width",e)}if(d){l.canvas.setAttribute("height",d)}jsPlumb.appendElement(l.canvas,g.parent);l.attachListeners(l.canvas,l);var k=function(p,o,n){if(!f){l.canvas.setAttribute("src",l.img.src);f=true}var m=l.anchorPoint[0]-(e/2),q=l.anchorPoint[1]-(d/2);jsPlumb.sizeCanvas(l.canvas,m,q,e,d)};this.paint=function(o,n,m){if(l.ready){k(o,n,m)}else{window.setTimeout(function(){l.paint(o,n,m)},200)}}};jsPlumb.Endpoints.Blank=function(d){var c=this;this.type="Blank";a.apply(this,arguments);this.compute=function(g,e,h,f){return[g[0],g[1],10,0]};c.canvas=document.createElement("div");c.canvas.style.display="block";c.canvas.style.width="1px";c.canvas.style.height="1px";c.canvas.style.background="transparent";c.canvas.style.position="absolute";c.canvas.className=c._jsPlumb.endpointClass;jsPlumb.appendElement(c.canvas,d.parent);this.paint=function(g,f,e){jsPlumb.sizeCanvas(c.canvas,g[0],g[1],g[2],g[3])}};jsPlumb.Endpoints.Triangle=function(c){this.type="Triangle";c=c||{};c.width=c.width||55;c.height=c.height||55;this.width=c.width;this.height=c.height;this.compute=function(i,f,l,h){var g=l.width||self.width,e=l.height||self.height,d=i[0]-(g/2),k=i[1]-(e/2);return[d,k,g,e]}};var b=function(e){var d=true,c=this;this.isAppendedAtTopLevel=true;this.component=e.component;this.loc=e.location==null?0.5:e.location;this.endpointLoc=e.endpointLocation==null?[0.5,0.5]:e.endpointLocation;this.setVisible=function(f){d=f;c.component.repaint()};this.isVisible=function(){return d};this.hide=function(){c.setVisible(false)};this.show=function(){c.setVisible(true)};this.incrementLocation=function(f){c.loc+=f;c.component.repaint()};this.setLocation=function(f){c.loc=f;c.component.repaint()};this.getLocation=function(){return c.loc}};jsPlumb.Overlays.Arrow=function(g){this.type="Arrow";b.apply(this,arguments);this.isAppendedAtTopLevel=false;g=g||{};var d=this;this.length=g.length||20;this.width=g.width||20;this.id=g.id;var f=(g.direction||1)<0?-1:1,e=g.paintStyle||{lineWidth:1},c=g.foldback||0.623;this.computeMaxSize=function(){return d.width*1.5};this.cleanup=function(){};this.draw=function(i,x,s){var m,t,h,n,l;if(i.pointAlongPathFrom){if(d.loc==1){m=i.pointOnPath(d.loc);t=i.pointAlongPathFrom(d.loc,-1);h=jsPlumb.util.pointOnLine(m,t,d.length)}else{if(d.loc==0){h=i.pointOnPath(d.loc);t=i.pointAlongPathFrom(d.loc,1);m=jsPlumb.util.pointOnLine(h,t,d.length)}else{m=i.pointAlongPathFrom(d.loc,f*d.length/2),t=i.pointOnPath(d.loc),h=jsPlumb.util.pointOnLine(m,t,d.length)}}n=jsPlumb.util.perpendicularLineTo(m,h,d.width);l=jsPlumb.util.pointOnLine(m,h,c*d.length);var w=Math.min(m.x,n[0].x,n[1].x),q=Math.max(m.x,n[0].x,n[1].x),v=Math.min(m.y,n[0].y,n[1].y),p=Math.max(m.y,n[0].y,n[1].y);var o={hxy:m,tail:n,cxy:l},r=e.strokeStyle||x.strokeStyle,u=e.fillStyle||x.strokeStyle,k=e.lineWidth||x.lineWidth;d.paint(i,o,k,r,u,s);return[w,q,v,p]}else{return[0,0,0,0]}}};jsPlumb.Overlays.PlainArrow=function(d){d=d||{};var c=jsPlumb.extend(d,{foldback:1});jsPlumb.Overlays.Arrow.call(this,c);this.type="PlainArrow"};jsPlumb.Overlays.Diamond=function(e){e=e||{};var c=e.length||40,d=jsPlumb.extend(e,{length:c/2,foldback:2});jsPlumb.Overlays.Arrow.call(this,d);this.type="Diamond"};jsPlumb.Overlays.Label=function(i){this.type="Label";jsPlumb.DOMElementComponent.apply(this,arguments);b.apply(this,arguments);this.labelStyle=i.labelStyle||jsPlumb.Defaults.LabelStyle;this.id=i.id;this.cachedDimensions=null;var e=i.label||"",c=this,f=false,k=document.createElement("div"),g=null;k.style.position="absolute";var d=i._jsPlumb.overlayClass+" "+(c.labelStyle.cssClass?c.labelStyle.cssClass:i.cssClass?i.cssClass:"");k.className=d;jsPlumb.appendElement(k,i.component.parent);jsPlumb.getId(k);c.attachListeners(k,c);c.canvas=k;var h=c.setVisible;c.setVisible=function(l){h(l);k.style.display=l?"block":"none"};this.getElement=function(){return k};this.cleanup=function(){if(k!=null){jsPlumb.CurrentLibrary.removeElement(k)}};this.setLabel=function(m){e=m;g=null;c.component.repaint()};this.getLabel=function(){return e};this.paint=function(l,n,m){if(!f){l.appendDisplayElement(k);c.attachListeners(k,l);f=true}k.style.left=(m[0]+n.minx)+"px";k.style.top=(m[1]+n.miny)+"px"};this.getTextDimensions=function(){if(typeof e=="function"){var l=e(c);k.innerHTML=l.replace(/\r\n/g,"<br/>")}else{if(g==null){g=e;k.innerHTML=g.replace(/\r\n/g,"<br/>")}}var n=jsPlumb.CurrentLibrary.getElementObject(k),m=jsPlumb.CurrentLibrary.getSize(n);return{width:m[0],height:m[1]}};this.computeMaxSize=function(l){var m=c.getTextDimensions(l);return m.width?Math.max(m.width,m.height)*1.5:0};this.draw=function(m,n,o){var q=c.getTextDimensions(m);if(q.width!=null){var p={x:0,y:0};if(m.pointOnPath){p=m.pointOnPath(c.loc)}else{var l=c.loc.constructor==Array?c.loc:c.endpointLoc;p={x:l[0]*o[2],y:l[1]*o[3]}}minx=p.x-(q.width/2),miny=p.y-(q.height/2);c.paint(m,{minx:minx,miny:miny,td:q,cxy:p},o);return[minx,minx+q.width,miny,miny+q.height]}else{return[0,0,0,0]}};this.reattachListeners=function(l){if(k){c.reattachListenersForElement(k,c,l)}}};jsPlumb.Overlays.GuideLines=function(){var c=this;c.length=50;c.lineWidth=5;this.type="GuideLines";b.apply(this,arguments);jsPlumb.jsPlumbUIComponent.apply(this,arguments);this.draw=function(e,l,k){var i=e.pointAlongPathFrom(c.loc,c.length/2),h=e.pointOnPath(c.loc),g=jsPlumb.util.pointOnLine(i,h,c.length),f=jsPlumb.util.perpendicularLineTo(i,g,40),d=jsPlumb.util.perpendicularLineTo(g,i,20);c.paint(e,[i,g,f,d],c.lineWidth,"red",null,k);return[Math.min(i.x,g.x),Math.min(i.y,g.y),Math.max(i.x,g.x),Math.max(i.y,g.y)]};this.computeMaxSize=function(){return 50};this.cleanup=function(){}}})();(function(){var c=function(e,g,d,f){this.m=(f-g)/(d-e);this.b=-1*((this.m*e)-g);this.rectIntersect=function(q,p,s,o){var n=[];var k=(p-this.b)/this.m;if(k>=q&&k<=(q+s)){n.push([k,(this.m*k)+this.b])}var t=(this.m*(q+s))+this.b;if(t>=p&&t<=(p+o)){n.push([(t-this.b)/this.m,t])}var k=((p+o)-this.b)/this.m;if(k>=q&&k<=(q+s)){n.push([k,(this.m*k)+this.b])}var t=(this.m*q)+this.b;if(t>=p&&t<=(p+o)){n.push([(t-this.b)/this.m,t])}if(n.length==2){var m=(n[0][0]+n[1][0])/2,l=(n[0][1]+n[1][1])/2;n.push([m,l]);var i=m<=q+(s/2)?-1:1,r=l<=p+(o/2)?-1:1;n.push([i,r]);return n}return null}},a=function(e,g,d,f){if(e<=d&&f<=g){return 1}else{if(e<=d&&g<=f){return 2}else{if(d<=e&&f>=g){return 3}}}return 4},b=function(g,f,i,e,h,m,l,d,k){if(d<=k){return[g,f]}if(i==1){if(e[3]<=0&&h[3]>=1){return[g+(e[2]<0.5?-1*m:m),f]}else{if(e[2]>=1&&h[2]<=0){return[g,f+(e[3]<0.5?-1*l:l)]}else{return[g+(-1*m),f+(-1*l)]}}}else{if(i==2){if(e[3]>=1&&h[3]<=0){return[g+(e[2]<0.5?-1*m:m),f]}else{if(e[2]>=1&&h[2]<=0){return[g,f+(e[3]<0.5?-1*l:l)]}else{return[g+(1*m),f+(-1*l)]}}}else{if(i==3){if(e[3]>=1&&h[3]<=0){return[g+(e[2]<0.5?-1*m:m),f]}else{if(e[2]<=0&&h[2]>=1){return[g,f+(e[3]<0.5?-1*l:l)]}else{return[g+(-1*m),f+(-1*l)]}}}else{if(i==4){if(e[3]<=0&&h[3]>=1){return[g+(e[2]<0.5?-1*m:m),f]}else{if(e[2]<=0&&h[2]>=1){return[g,f+(e[3]<0.5?-1*l:l)]}else{return[g+(1*m),f+(-1*l)]}}}}}}};jsPlumb.Connectors.StateMachine=function(l){var s=this,n=null,o,m,g,e,p=[],d=l.curviness||10,k=l.margin||5,q=l.proximityLimit||80,f=l.orientation&&l.orientation=="clockwise",i=l.loopbackRadius||25,h=false;this.type="StateMachine";l=l||{};this.compute=function(ab,F,U,G,aa,u,t,S){var O=Math.abs(ab[0]-F[0]),W=Math.abs(ab[1]-F[1]),Q=0.45*O,Z=0.45*W;O*=1.9;W*=1.9;t=t||1;var M=Math.min(ab[0],F[0])-Q,K=Math.min(ab[1],F[1])-Z;if(U.elementId!=G.elementId){h=false;o=ab[0]<F[0]?Q:O-Q;m=ab[1]<F[1]?Z:W-Z;g=ab[0]<F[0]?O-Q:Q;e=ab[1]<F[1]?W-Z:Z;if(ab[2]==0){o-=k}if(ab[2]==1){o+=k}if(ab[3]==0){m-=k}if(ab[3]==1){m+=k}if(F[2]==0){g-=k}if(F[2]==1){g+=k}if(F[3]==0){e-=k}if(F[3]==1){e+=k}var L=(o+g)/2,J=(m+e)/2,v=(-1*L)/J,T=Math.atan(v),N=(v==Infinity||v==-Infinity)?0:Math.abs(d/2*Math.sin(T)),P=(v==Infinity||v==-Infinity)?0:Math.abs(d/2*Math.cos(T)),z=a(o,m,g,e),H=Math.sqrt(Math.pow(g-o,2)+Math.pow(e-m,2));p=b(L,J,z,ab,F,d,d,H,q);var E=Math.max(Math.abs(p[0]-o)*3,Math.abs(p[0]-g)*3,Math.abs(g-o),2*t,S),I=Math.max(Math.abs(p[1]-m)*3,Math.abs(p[1]-e)*3,Math.abs(e-m),2*t,S);if(O<E){var R=E-O;M-=(R/2);o+=(R/2);g+=(R/2);O=E;p[0]+=(R/2)}if(W<I){var Y=I-W;K-=(Y/2);m+=(Y/2);e+=(Y/2);W=I;p[1]+=(Y/2)}n=[M,K,O,W,o,m,g,e,p[0],p[1]]}else{h=true;var X=ab[0],V=ab[0],D=ab[1]-k,B=ab[1]-k,C=X,A=D-i;O=((2*t)+(4*i)),W=((2*t)+(4*i));M=C-i-t-i,K=A-i-t-i;n=[M,K,O,W,C-M,A-K,i,f,X-M,D-K,V-M,B-K]}return n};var r=function(){return[{x:g,y:e},{x:p[0],y:p[1]},{x:p[0]+1,y:p[1]+1},{x:o,y:m}]};this.pointOnPath=function(v){if(h){if(v>0&&v<1){v=1-v}var w=(v*2*Math.PI)+(Math.PI/2),u=n[4]+(n[6]*Math.cos(w)),t=n[5]+(n[6]*Math.sin(w));return{x:u,y:t}}else{return jsBezier.pointOnCurve(r(),v)}};this.gradientAtPoint=function(t){if(h){return Math.atan(t*2*Math.PI)}else{return jsBezier.gradientAtPoint(r(),t)}};this.pointAlongPathFrom=function(v,z){if(h){if(v>0&&v<1){v=1-v}var w=2*Math.PI*n[6],y=z/w*2*Math.PI,x=(v*2*Math.PI)-y+(Math.PI/2),u=n[4]+(n[6]*Math.cos(x)),t=n[5]+(n[6]*Math.sin(x));return{x:u,y:t}}return jsBezier.pointAlongCurveFrom(r(),v,z)}};jsPlumb.Connectors.canvas.StateMachine=function(f){f=f||{};var d=this,g=f.drawGuideline||true,e=f.avoidSelector;jsPlumb.Connectors.StateMachine.apply(this,arguments);jsPlumb.CanvasConnector.apply(this,arguments);this._paint=function(l){if(l.length==10){d.ctx.beginPath();d.ctx.moveTo(l[4],l[5]);d.ctx.quadraticCurveTo(l[8],l[9],l[6],l[7]);d.ctx.stroke()}else{d.ctx.save();d.ctx.beginPath();var k=0,i=2*Math.PI,h=l[7];d.ctx.arc(l[4],l[5],l[6],0,i,h);d.ctx.stroke();d.ctx.closePath();d.ctx.restore()}};this.createGradient=function(i,h){return h.createLinearGradient(i[4],i[5],i[6],i[7])}};jsPlumb.Connectors.svg.StateMachine=function(){var d=this;jsPlumb.Connectors.StateMachine.apply(this,arguments);jsPlumb.SvgConnector.apply(this,arguments);this.getPath=function(e){if(e.length==10){return"M "+e[4]+" "+e[5]+" C "+e[8]+" "+e[9]+" "+e[8]+" "+e[9]+" "+e[6]+" "+e[7]}else{return"M"+(e[8]+4)+" "+e[9]+" A "+e[6]+" "+e[6]+" 0 1,0 "+(e[8]-4)+" "+e[9]}}};jsPlumb.Connectors.vml.StateMachine=function(){jsPlumb.Connectors.StateMachine.apply(this,arguments);jsPlumb.VmlConnector.apply(this,arguments);var d=jsPlumb.vml.convertValue;this.getPath=function(k){if(k.length==10){return"m"+d(k[4])+","+d(k[5])+" c"+d(k[8])+","+d(k[9])+","+d(k[8])+","+d(k[9])+","+d(k[6])+","+d(k[7])+" e"}else{var h=d(k[8]-k[6]),g=d(k[9]-(2*k[6])),f=h+d(2*k[6]),e=g+d(2*k[6]),l=h+","+g+","+f+","+e;var i="ar "+l+","+d(k[8])+","+d(k[9])+","+d(k[8])+","+d(k[9])+" e";return i}}}})();(function(){var k={"stroke-linejoin":"joinstyle",joinstyle:"joinstyle",endcap:"endcap",miterlimit:"miterlimit"};if(document.createStyleSheet){document.createStyleSheet().addRule(".jsplumb_vml","behavior:url(#default#VML);position:absolute;");document.createStyleSheet().addRule("jsplumb\\:textbox","behavior:url(#default#VML);position:absolute;");document.createStyleSheet().addRule("jsplumb\\:oval","behavior:url(#default#VML);position:absolute;");document.createStyleSheet().addRule("jsplumb\\:rect","behavior:url(#default#VML);position:absolute;");document.createStyleSheet().addRule("jsplumb\\:stroke","behavior:url(#default#VML);position:absolute;");document.createStyleSheet().addRule("jsplumb\\:shape","behavior:url(#default#VML);position:absolute;");document.createStyleSheet().addRule("jsplumb\\:group","behavior:url(#default#VML);position:absolute;");document.namespaces.add("jsplumb","urn:schemas-microsoft-com:vml")}jsPlumb.vml={};var b=1000,d={},h=function(q,p){var s=jsPlumb.getId(q),r=d[s];if(!r){r=o("group",[0,0,b,b],{"class":p});r.style.backgroundColor="red";d[s]=r;jsPlumb.appendElement(r,q)}return r},c=function(q,r){for(var p in r){q[p]=r[p]}},o=function(p,r,s){s=s||{};var q=document.createElement("jsplumb:"+p);q.className=(s["class"]?s["class"]+" ":"")+"jsplumb_vml";n(q,r);c(q,s);return q},n=function(q,p){q.style.left=p[0]+"px";q.style.top=p[1]+"px";q.style.width=p[2]+"px";q.style.height=p[3]+"px";q.style.position="absolute"},i=jsPlumb.vml.convertValue=function(p){return Math.floor(p*b)},e=function(s,q,r,p){if("transparent"===q){p.setOpacity(r,"0.0")}else{p.setOpacity(r,"1.0")}},g=function(t,p,w){var s={};if(p.strokeStyle){s.stroked="true";var x=jsPlumb.util.convertStyle(p.strokeStyle,true);s.strokecolor=x;e(s,x,"stroke",w);s.strokeweight=p.lineWidth+"px"}else{s.stroked="false"}if(p.fillStyle){s.filled="true";var q=jsPlumb.util.convertStyle(p.fillStyle,true);s.fillcolor=q;e(s,q,"fill",w)}else{s.filled="false"}if(p.dashstyle){if(w.strokeNode==null){w.strokeNode=o("stroke",[0,0,0,0],{dashstyle:p.dashstyle});t.appendChild(w.strokeNode)}else{w.strokeNode.dashstyle=p.dashstyle}}else{if(p["stroke-dasharray"]&&p.lineWidth){var y=p["stroke-dasharray"].indexOf(",")==-1?" ":",",u=p["stroke-dasharray"].split(y),r="";for(var v=0;v<u.length;v++){r+=(Math.floor(u[v]/p.lineWidth)+y)}if(w.strokeNode==null){w.strokeNode=o("stroke",[0,0,0,0],{dashstyle:r});t.appendChild(w.strokeNode)}else{w.strokeNode.dashstyle=r}}}c(t,s)},l=function(){var p=this;jsPlumb.jsPlumbUIComponent.apply(this,arguments);this.opacityNodes={stroke:null,fill:null};this.initOpacityNodes=function(r){p.opacityNodes.stroke=o("stroke",[0,0,1,1],{opacity:"0.0"});p.opacityNodes.fill=o("fill",[0,0,1,1],{opacity:"0.0"});r.appendChild(p.opacityNodes.stroke);r.appendChild(p.opacityNodes.fill)};this.setOpacity=function(r,t){var s=p.opacityNodes[r];if(s){s.opacity=""+t}};var q=[];this.getDisplayElements=function(){return q};this.appendDisplayElement=function(s,r){if(!r){p.canvas.parentNode.appendChild(s)}q.push(s)}},f=jsPlumb.VmlConnector=function(r){var p=this;p.strokeNode=null;p.canvas=null;l.apply(this,arguments);var q=p._jsPlumb.connectorClass+(r.cssClass?(" "+r.cssClass):"");this.paint=function(w,t,v){if(t!=null){var A=p.getPath(w),u={path:A};if(t.outlineColor){var y=t.outlineWidth||1,z=t.lineWidth+(2*y),x={strokeStyle:jsPlumb.util.convertStyle(t.outlineColor),lineWidth:z};for(var s in k){x[s]=t[s]}if(p.bgCanvas==null){u["class"]=q;u.coordsize=(w[2]*b)+","+(w[3]*b);p.bgCanvas=o("shape",w,u);jsPlumb.appendElement(p.bgCanvas,r.parent);n(p.bgCanvas,w);p.appendDisplayElement(p.bgCanvas,true)}else{u.coordsize=(w[2]*b)+","+(w[3]*b);n(p.bgCanvas,w);c(p.bgCanvas,u)}g(p.bgCanvas,x,p)}if(p.canvas==null){u["class"]=q;u.coordsize=(w[2]*b)+","+(w[3]*b);if(p.tooltip){u.label=p.tooltip}p.canvas=o("shape",w,u);jsPlumb.appendElement(p.canvas,r.parent);p.appendDisplayElement(p.canvas,true);p.attachListeners(p.canvas,p);p.initOpacityNodes(p.canvas,["stroke"])}else{u.coordsize=(w[2]*b)+","+(w[3]*b);n(p.canvas,w);c(p.canvas,u)}g(p.canvas,t,p)}};this.reattachListeners=function(){if(p.canvas){p.reattachListenersForElement(p.canvas,p)}}},m=function(t){l.apply(this,arguments);var p=null,r=this,q=null,s=null;r.canvas=document.createElement("div");r.canvas.style.position="absolute";jsPlumb.appendElement(r.canvas,t.parent);if(r.tooltip){r.canvas.setAttribute("label",r.tooltip)}this.paint=function(x,v,u){var w={};jsPlumb.sizeCanvas(r.canvas,x[0],x[1],x[2],x[3]);if(p==null){w["class"]=jsPlumb.endpointClass;p=r.getVml([0,0,x[2],x[3]],w,u);r.canvas.appendChild(p);r.attachListeners(p,r);r.appendDisplayElement(p,true);r.appendDisplayElement(r.canvas,true);r.initOpacityNodes(p,["fill"])}else{n(p,[0,0,x[2],x[3]]);c(p,w)}g(p,v,r)};this.reattachListeners=function(){if(p){r.reattachListenersForElement(p,r)}}};jsPlumb.Connectors.vml.Bezier=function(){jsPlumb.Connectors.Bezier.apply(this,arguments);f.apply(this,arguments);this.getPath=function(p){return"m"+i(p[4])+","+i(p[5])+" c"+i(p[8])+","+i(p[9])+","+i(p[10])+","+i(p[11])+","+i(p[6])+","+i(p[7])+" e"}};jsPlumb.Connectors.vml.Straight=function(){jsPlumb.Connectors.Straight.apply(this,arguments);f.apply(this,arguments);this.getPath=function(p){return"m"+i(p[4])+","+i(p[5])+" l"+i(p[6])+","+i(p[7])+" e"}};jsPlumb.Connectors.vml.Flowchart=function(){jsPlumb.Connectors.Flowchart.apply(this,arguments);f.apply(this,arguments);this.getPath=function(r){var s="m "+i(r[4])+","+i(r[5])+" l";for(var q=0;q<r[8];q++){s=s+" "+i(r[9+(q*2)])+","+i(r[10+(q*2)])}s=s+" "+i(r[6])+","+i(r[7])+" e";return s}};jsPlumb.Endpoints.vml.Dot=function(){jsPlumb.Endpoints.Dot.apply(this,arguments);m.apply(this,arguments);this.getVml=function(q,r,p){return o("oval",q,r)}};jsPlumb.Endpoints.vml.Rectangle=function(){jsPlumb.Endpoints.Rectangle.apply(this,arguments);m.apply(this,arguments);this.getVml=function(q,r,p){return o("rect",q,r)}};jsPlumb.Endpoints.vml.Image=jsPlumb.Endpoints.Image;jsPlumb.Endpoints.vml.Blank=jsPlumb.Endpoints.Blank;jsPlumb.Overlays.vml.Label=jsPlumb.Overlays.Label;var a=function(t,r){t.apply(this,r);l.apply(this,arguments);var q=this,s=null;q.canvas=null;var p=function(v,u){return"m "+i(v.hxy.x)+","+i(v.hxy.y)+" l "+i(v.tail[0].x)+","+i(v.tail[0].y)+" "+i(v.cxy.x)+","+i(v.cxy.y)+" "+i(v.tail[1].x)+","+i(v.tail[1].y)+" x e"};this.paint=function(y,D,C,E,I,H){var v={};if(E){v.stroked="true";v.strokecolor=jsPlumb.util.convertStyle(E,true)}if(C){v.strokeweight=C+"px"}if(I){v.filled="true";v.fillcolor=I}var u=Math.min(D.hxy.x,D.tail[0].x,D.tail[1].x,D.cxy.x),G=Math.min(D.hxy.y,D.tail[0].y,D.tail[1].y,D.cxy.y),z=Math.max(D.hxy.x,D.tail[0].x,D.tail[1].x,D.cxy.x),x=Math.max(D.hxy.y,D.tail[0].y,D.tail[1].y,D.cxy.y),F=Math.abs(z-u),B=Math.abs(x-G),A=[u,G,F,B];v.path=p(D,H);v.coordsize=(H[2]*b)+","+(H[3]*b);A[0]=H[0];A[1]=H[1];A[2]=H[2];A[3]=H[3];if(q.canvas==null){q.canvas=o("shape",A,v);y.appendDisplayElement(q.canvas);q.attachListeners(q.canvas,y)}else{n(q.canvas,A);c(q.canvas,v)}};this.reattachListeners=function(){if(q.canvas){q.reattachListenersForElement(q.canvas,q)}}};jsPlumb.Overlays.vml.Arrow=function(){a.apply(this,[jsPlumb.Overlays.Arrow,arguments])};jsPlumb.Overlays.vml.PlainArrow=function(){a.apply(this,[jsPlumb.Overlays.PlainArrow,arguments])};jsPlumb.Overlays.vml.Diamond=function(){a.apply(this,[jsPlumb.Overlays.Diamond,arguments])}})();(function(){var l={joinstyle:"stroke-linejoin","stroke-linejoin":"stroke-linejoin","stroke-dashoffset":"stroke-dashoffset","stroke-linecap":"stroke-linecap"},w="stroke-dasharray",A="dashstyle",e="linearGradient",b="radialGradient",c="fill",a="stop",z="stroke",q="stroke-width",h="style",m="none",t="jsplumb_gradient_",o="lineWidth",C={svg:"http://www.w3.org/2000/svg",xhtml:"http://www.w3.org/1999/xhtml"},g=function(F,D){for(var E in D){F.setAttribute(E,""+D[E])}},f=function(E,D){var F=document.createElementNS(C.svg,E);D=D||{};D.version="1.1";D.xmnls=C.xhtml;g(F,D);return F},n=function(D){return"position:absolute;left:"+D[0]+"px;top:"+D[1]+"px"},i=function(E){for(var D=0;D<E.childNodes.length;D++){if(E.childNodes[D].tagName==e||E.childNodes[D].tagName==b){E.removeChild(E.childNodes[D])}}},v=function(N,I,F,D,J){var G=t+J._jsPlumb.idstamp();i(N);if(!F.gradient.offset){var L=f(e,{id:G});N.appendChild(L)}else{var L=f(b,{id:G});N.appendChild(L)}for(var K=0;K<F.gradient.stops.length;K++){var H=K;if(D.length==8){H=D[4]<D[6]?K:F.gradient.stops.length-1-K}else{H=D[4]<D[6]?F.gradient.stops.length-1-K:K}var M=jsPlumb.util.convertStyle(F.gradient.stops[H][1],true);var O=f(a,{offset:Math.floor(F.gradient.stops[K][0]*100)+"%","stop-color":M});L.appendChild(O)}var E=F.strokeStyle?z:c;I.setAttribute(h,E+":url(#"+G+")")},x=function(K,G,E,D,H){if(E.gradient){v(K,G,E,D,H)}else{i(K);G.setAttribute(h,"")}G.setAttribute(c,E.fillStyle?jsPlumb.util.convertStyle(E.fillStyle,true):m);G.setAttribute(z,E.strokeStyle?jsPlumb.util.convertStyle(E.strokeStyle,true):m);if(E.lineWidth){G.setAttribute(q,E.lineWidth)}if(E[A]&&E[o]&&!E[w]){var L=E[A].indexOf(",")==-1?" ":",",I=E[A].split(L),F="";I.forEach(function(M){F+=(Math.floor(M*E.lineWidth)+L)});G.setAttribute(w,F)}else{if(E[w]){G.setAttribute(w,E[w])}}for(var J in l){if(E[J]){G.setAttribute(l[J],E[J])}}},B=function(F){var D=/([0-9].)(p[xt])\s(.*)/;var E=F.match(D);return{size:E[1]+E[2],font:E[3]}},r=function(I,J,E){var K=E.split(" "),H=I.className,G=H.baseVal.split(" ");for(var F=0;F<K.length;F++){if(J){if(G.indexOf(K[F])==-1){G.push(K[F])}}else{var D=G.indexOf(K[F]);if(D!=-1){G.splice(D,1)}}}I.className.baseVal=G.join(" ")},u=function(E,D){r(E,true,D)},k=function(E,D){r(E,false,D)};jsPlumb.util.svg={addClass:u,removeClass:k};var s=function(H){var D=this,G=H.pointerEventsSpec||"all";jsPlumb.jsPlumbUIComponent.apply(this,H.originalArgs);D.canvas=null,D.path=null,D.svg=null;var F=H.cssClass+" "+(H.originalArgs[0].cssClass||""),I={style:"",width:0,height:0,"pointer-events":G,position:"absolute"};if(D.tooltip){I.title=D.tooltip}D.svg=f("svg",I);if(H.useDivWrapper){D.canvas=document.createElement("div");D.canvas.style.position="absolute";jsPlumb.sizeCanvas(D.canvas,0,0,1,1);D.canvas.className=F;if(D.tooltip){D.canvas.setAttribute("title",D.tooltip)}}else{g(D.svg,{"class":F});D.canvas=D.svg}H._jsPlumb.appendElement(D.canvas,H.originalArgs[0]["parent"]);if(H.useDivWrapper){D.canvas.appendChild(D.svg)}var E=[D.canvas];this.getDisplayElements=function(){return E};this.appendDisplayElement=function(J){E.push(J)};this.paint=function(M,L,K){if(L!=null){var J=M[0],N=M[1];if(H.useDivWrapper){jsPlumb.sizeCanvas(D.canvas,M[0],M[1],M[2],M[3]);J=0,N=0}g(D.svg,{style:n([J,N,M[2],M[3]]),width:M[2],height:M[3]});D._paint.apply(this,arguments)}}};var d=jsPlumb.SvgConnector=function(E){var D=this;s.apply(this,[{cssClass:E._jsPlumb.connectorClass,originalArgs:arguments,pointerEventsSpec:"none",tooltip:E.tooltip,_jsPlumb:E._jsPlumb}]);this._paint=function(L,H){var K=D.getPath(L),F={d:K},J=null;F["pointer-events"]="all";if(H.outlineColor){var I=H.outlineWidth||1,G=H.lineWidth+(2*I),J=jsPlumb.CurrentLibrary.extend({},H);J.strokeStyle=jsPlumb.util.convertStyle(H.outlineColor);J.lineWidth=G;if(D.bgPath==null){D.bgPath=f("path",F);D.svg.appendChild(D.bgPath);D.attachListeners(D.bgPath,D)}else{g(D.bgPath,F)}x(D.svg,D.bgPath,J,L,D)}if(D.path==null){D.path=f("path",F);D.svg.appendChild(D.path);D.attachListeners(D.path,D)}else{g(D.path,F)}x(D.svg,D.path,H,L,D)};this.reattachListeners=function(){if(D.bgPath){D.reattachListenersForElement(D.bgPath,D)}if(D.path){D.reattachListenersForElement(D.path,D)}}};jsPlumb.Connectors.svg.Bezier=function(D){jsPlumb.Connectors.Bezier.apply(this,arguments);d.apply(this,arguments);this.getPath=function(E){return"M "+E[4]+" "+E[5]+" C "+E[8]+" "+E[9]+" "+E[10]+" "+E[11]+" "+E[6]+" "+E[7]}};jsPlumb.Connectors.svg.Straight=function(D){jsPlumb.Connectors.Straight.apply(this,arguments);d.apply(this,arguments);this.getPath=function(E){return"M "+E[4]+" "+E[5]+" L "+E[6]+" "+E[7]}};jsPlumb.Connectors.svg.Flowchart=function(){var D=this;jsPlumb.Connectors.Flowchart.apply(this,arguments);d.apply(this,arguments);this.getPath=function(F){var G="M "+F[4]+","+F[5];for(var E=0;E<F[8];E++){G=G+" L "+F[9+(E*2)]+" "+F[10+(E*2)]}G=G+" "+F[6]+","+F[7];return G}};var y=function(E){var D=this;s.apply(this,[{cssClass:E._jsPlumb.endpointClass,originalArgs:arguments,pointerEventsSpec:"all",useDivWrapper:true,_jsPlumb:E._jsPlumb}]);this._paint=function(H,G){var F=jsPlumb.extend({},G);if(F.outlineColor){F.strokeWidth=F.outlineWidth;F.strokeStyle=jsPlumb.util.convertStyle(F.outlineColor,true)}if(D.node==null){D.node=D.makeNode(H,F);D.svg.appendChild(D.node);D.attachListeners(D.node,D)}x(D.svg,D.node,F,H,D);n(D.node,H)};this.reattachListeners=function(){if(D.node){D.reattachListenersForElement(D.node,D)}}};jsPlumb.Endpoints.svg.Dot=function(){jsPlumb.Endpoints.Dot.apply(this,arguments);y.apply(this,arguments);this.makeNode=function(E,D){return f("circle",{cx:E[2]/2,cy:E[3]/2,r:E[2]/2})}};jsPlumb.Endpoints.svg.Rectangle=function(){jsPlumb.Endpoints.Rectangle.apply(this,arguments);y.apply(this,arguments);this.makeNode=function(E,D){return f("rect",{width:E[2],height:E[3]})}};jsPlumb.Endpoints.svg.Image=jsPlumb.Endpoints.Image;jsPlumb.Endpoints.svg.Blank=jsPlumb.Endpoints.Blank;jsPlumb.Overlays.svg.Label=jsPlumb.Overlays.Label;var p=function(H,F){H.apply(this,F);jsPlumb.jsPlumbUIComponent.apply(this,F);this.isAppendedAtTopLevel=false;var D=this,G=null;this.paint=function(J,M,I,N,K){if(G==null){G=f("path");J.svg.appendChild(G);D.attachListeners(G,J);D.attachListeners(G,D)}var L=F&&(F.length==1)?(F[0].cssClass||""):"";g(G,{d:E(M),"class":L,stroke:N?N:null,fill:K?K:null})};var E=function(I){return"M"+I.hxy.x+","+I.hxy.y+" L"+I.tail[0].x+","+I.tail[0].y+" L"+I.cxy.x+","+I.cxy.y+" L"+I.tail[1].x+","+I.tail[1].y+" L"+I.hxy.x+","+I.hxy.y};this.reattachListeners=function(){if(G){D.reattachListenersForElement(G,D)}}};jsPlumb.Overlays.svg.Arrow=function(){p.apply(this,[jsPlumb.Overlays.Arrow,arguments])};jsPlumb.Overlays.svg.PlainArrow=function(){p.apply(this,[jsPlumb.Overlays.PlainArrow,arguments])};jsPlumb.Overlays.svg.Diamond=function(){p.apply(this,[jsPlumb.Overlays.Diamond,arguments])};jsPlumb.Overlays.svg.GuideLines=function(){var I=null,D=this,H=null,G,F;jsPlumb.Overlays.GuideLines.apply(this,arguments);this.paint=function(K,M,J,N,L){if(I==null){I=f("path");K.svg.appendChild(I);D.attachListeners(I,K);D.attachListeners(I,D);G=f("path");K.svg.appendChild(G);D.attachListeners(G,K);D.attachListeners(G,D);F=f("path");K.svg.appendChild(F);D.attachListeners(F,K);D.attachListeners(F,D)}g(I,{d:E(M[0],M[1]),stroke:"red",fill:null});g(G,{d:E(M[2][0],M[2][1]),stroke:"blue",fill:null});g(F,{d:E(M[3][0],M[3][1]),stroke:"green",fill:null})};var E=function(K,J){return"M "+K.x+","+K.y+" L"+J.x+","+J.y}}})();(function(){var d=null,i=function(p,o){return jsPlumb.CurrentLibrary.hasClass(a(p),o)},a=function(o){return jsPlumb.CurrentLibrary.getElementObject(o)},m=function(o){return jsPlumb.CurrentLibrary.getOffset(a(o))},n=function(o){return jsPlumb.CurrentLibrary.getPageXY(o)},f=function(o){return jsPlumb.CurrentLibrary.getClientXY(o)};var k=function(){var q=this;q.overlayPlacements=[];jsPlumb.jsPlumbUIComponent.apply(this,arguments);jsPlumb.EventGenerator.apply(this,arguments);this._over=function(z){var B=m(a(q.canvas)),D=n(z),u=D[0]-B.left,C=D[1]-B.top;if(u>0&&C>0&&u<q.canvas.width&&C<q.canvas.height){for(var v=0;v<q.overlayPlacements.length;v++){var w=q.overlayPlacements[v];if(w&&(w[0]<=u&&w[1]>=u&&w[2]<=C&&w[3]>=C)){return true}}var A=q.canvas.getContext("2d").getImageData(parseInt(u),parseInt(C),1,1);return A.data[0]!=0||A.data[1]!=0||A.data[2]!=0||A.data[3]!=0}return false};var p=false,o=false,t=null,s=false,r=function(v,u){return v!=null&&i(v,u)};this.mousemove=function(x){var z=n(x),w=f(x),v=document.elementFromPoint(w[0],w[1]),y=r(v,"_jsPlumb_overlay");var u=d==null&&(r(v,"_jsPlumb_endpoint")||r(v,"_jsPlumb_connector"));if(!p&&u&&q._over(x)){p=true;q.fire("mouseenter",q,x);return true}else{if(p&&(!q._over(x)||!u)&&!y){p=false;q.fire("mouseexit",q,x)}}q.fire("mousemove",q,x)};this.click=function(u){if(p&&q._over(u)&&!s){q.fire("click",q,u)}s=false};this.dblclick=function(u){if(p&&q._over(u)&&!s){q.fire("dblclick",q,u)}s=false};this.mousedown=function(u){if(q._over(u)&&!o){o=true;t=m(a(q.canvas));q.fire("mousedown",q,u)}};this.mouseup=function(u){o=false;q.fire("mouseup",q,u)};this.contextmenu=function(u){if(p&&q._over(u)&&!s){q.fire("contextmenu",q,u)}s=false}};var c=function(p){var o=document.createElement("canvas");jsPlumb.appendElement(o,p.parent);o.style.position="absolute";if(p["class"]){o.className=p["class"]}p._jsPlumb.getId(o,p.uuid);if(p.tooltip){o.setAttribute("title",p.tooltip)}return o};var l=function(p){k.apply(this,arguments);var o=[];this.getDisplayElements=function(){return o};this.appendDisplayElement=function(q){o.push(q)}};var h=jsPlumb.CanvasConnector=function(r){l.apply(this,arguments);var o=function(v,t){p.ctx.save();jsPlumb.extend(p.ctx,t);if(t.gradient){var u=p.createGradient(v,p.ctx);for(var s=0;s<t.gradient.stops.length;s++){u.addColorStop(t.gradient.stops[s][0],t.gradient.stops[s][1])}p.ctx.strokeStyle=u}p._paint(v);p.ctx.restore()};var p=this,q=p._jsPlumb.connectorClass+" "+(r.cssClass||"");p.canvas=c({"class":q,_jsPlumb:p._jsPlumb,parent:r.parent,tooltip:r.tooltip});p.ctx=p.canvas.getContext("2d");p.appendDisplayElement(p.canvas);p.paint=function(w,t){if(t!=null){jsPlumb.sizeCanvas(p.canvas,w[0],w[1],w[2],w[3]);if(t.outlineColor!=null){var v=t.outlineWidth||1,s=t.lineWidth+(2*v),u={strokeStyle:t.outlineColor,lineWidth:s};o(w,u)}o(w,t)}}};var b=function(r){var p=this;l.apply(this,arguments);var q=p._jsPlumb.endpointClass+" "+(r.cssClass||""),o={"class":q,_jsPlumb:p._jsPlumb,parent:r.parent,tooltip:p.tooltip};p.canvas=c(o);p.ctx=p.canvas.getContext("2d");p.appendDisplayElement(p.canvas);this.paint=function(x,u,s){jsPlumb.sizeCanvas(p.canvas,x[0],x[1],x[2],x[3]);if(u.outlineColor!=null){var w=u.outlineWidth||1,t=u.lineWidth+(2*w);var v={strokeStyle:u.outlineColor,lineWidth:t}}p._paint.apply(this,arguments)}};jsPlumb.Endpoints.canvas.Dot=function(r){jsPlumb.Endpoints.Dot.apply(this,arguments);b.apply(this,arguments);var q=this,p=function(s){try{return parseInt(s)}catch(t){if(s.substring(s.length-1)=="%"){return parseInt(s.substring(0,s-1))}}},o=function(u){var s=q.defaultOffset,t=q.defaultInnerRadius;u.offset&&(s=p(u.offset));u.innerRadius&&(t=p(u.innerRadius));return[s,t]};this._paint=function(A,t,x){if(t!=null){var B=q.canvas.getContext("2d"),u=x.getOrientation(q);jsPlumb.extend(B,t);if(t.gradient){var v=o(t.gradient),y=u[1]==1?v[0]*-1:v[0],s=u[0]==1?v[0]*-1:v[0],z=B.createRadialGradient(A[4],A[4],A[4],A[4]+s,A[4]+y,v[1]);for(var w=0;w<t.gradient.stops.length;w++){z.addColorStop(t.gradient.stops[w][0],t.gradient.stops[w][1])}B.fillStyle=z}B.beginPath();B.arc(A[4],A[4],A[4],0,Math.PI*2,true);B.closePath();if(t.fillStyle||t.gradient){B.fill()}if(t.strokeStyle){B.stroke()}}}};jsPlumb.Endpoints.canvas.Rectangle=function(p){var o=this;jsPlumb.Endpoints.Rectangle.apply(this,arguments);b.apply(this,arguments);this._paint=function(x,r,v){var A=o.canvas.getContext("2d"),t=v.getOrientation(o);jsPlumb.extend(A,r);if(r.gradient){var z=t[1]==1?x[3]:t[1]==0?x[3]/2:0;var y=t[1]==-1?x[3]:t[1]==0?x[3]/2:0;var s=t[0]==1?x[2]:t[0]==0?x[2]/2:0;var q=t[0]==-1?x[2]:t[0]==0?x[2]/2:0;var w=A.createLinearGradient(s,z,q,y);for(var u=0;u<r.gradient.stops.length;u++){w.addColorStop(r.gradient.stops[u][0],r.gradient.stops[u][1])}A.fillStyle=w}A.beginPath();A.rect(0,0,x[2],x[3]);A.closePath();if(r.fillStyle||r.gradient){A.fill()}if(r.strokeStyle){A.stroke()}}};jsPlumb.Endpoints.canvas.Triangle=function(p){var o=this;jsPlumb.Endpoints.Triangle.apply(this,arguments);b.apply(this,arguments);this._paint=function(z,q,v){var s=z[2],C=z[3],B=z[0],A=z[1],D=o.canvas.getContext("2d"),w=0,u=0,t=0,r=v.getOrientation(o);if(r[0]==1){w=s;u=C;t=180}if(r[1]==-1){w=s;t=90}if(r[1]==1){u=C;t=-90}D.fillStyle=q.fillStyle;D.translate(w,u);D.rotate(t*Math.PI/180);D.beginPath();D.moveTo(0,0);D.lineTo(s/2,C/2);D.lineTo(0,C);D.closePath();if(q.fillStyle||q.gradient){D.fill()}if(q.strokeStyle){D.stroke()}}};jsPlumb.Endpoints.canvas.Image=jsPlumb.Endpoints.Image;jsPlumb.Endpoints.canvas.Blank=jsPlumb.Endpoints.Blank;jsPlumb.Connectors.canvas.Bezier=function(){var o=this;jsPlumb.Connectors.Bezier.apply(this,arguments);h.apply(this,arguments);this._paint=function(p){o.ctx.beginPath();o.ctx.moveTo(p[4],p[5]);o.ctx.bezierCurveTo(p[8],p[9],p[10],p[11],p[6],p[7]);o.ctx.stroke()};this.createGradient=function(r,p,q){return o.ctx.createLinearGradient(r[6],r[7],r[4],r[5])}};jsPlumb.Connectors.canvas.Straight=function(){var o=this;jsPlumb.Connectors.Straight.apply(this,arguments);h.apply(this,arguments);this._paint=function(p){o.ctx.beginPath();o.ctx.moveTo(p[4],p[5]);o.ctx.lineTo(p[6],p[7]);o.ctx.stroke()};this.createGradient=function(q,p){return p.createLinearGradient(q[4],q[5],q[6],q[7])}};jsPlumb.Connectors.canvas.Flowchart=function(){var o=this;jsPlumb.Connectors.Flowchart.apply(this,arguments);h.apply(this,arguments);this._paint=function(q){o.ctx.beginPath();o.ctx.moveTo(q[4],q[5]);for(var p=0;p<q[8];p++){o.ctx.lineTo(q[9+(p*2)],q[10+(p*2)])}o.ctx.lineTo(q[6],q[7]);o.ctx.stroke()};this.createGradient=function(q,p){return p.createLinearGradient(q[4],q[5],q[6],q[7])}};jsPlumb.Overlays.canvas.Label=jsPlumb.Overlays.Label;var g=function(){jsPlumb.jsPlumbUIComponent.apply(this,arguments)};var e=function(p,o){p.apply(this,o);g.apply(this,arguments);this.paint=function(s,u,q,v,t){var r=s.ctx;r.lineWidth=q;r.beginPath();r.moveTo(u.hxy.x,u.hxy.y);r.lineTo(u.tail[0].x,u.tail[0].y);r.lineTo(u.cxy.x,u.cxy.y);r.lineTo(u.tail[1].x,u.tail[1].y);r.lineTo(u.hxy.x,u.hxy.y);r.closePath();if(v){r.strokeStyle=v;r.stroke()}if(t){r.fillStyle=t;r.fill()}}};jsPlumb.Overlays.canvas.Arrow=function(){e.apply(this,[jsPlumb.Overlays.Arrow,arguments])};jsPlumb.Overlays.canvas.PlainArrow=function(){e.apply(this,[jsPlumb.Overlays.PlainArrow,arguments])};jsPlumb.Overlays.canvas.Diamond=function(){e.apply(this,[jsPlumb.Overlays.Diamond,arguments])}})();(function(a){jsPlumb.CurrentLibrary={addClass:function(c,b){c=jsPlumb.CurrentLibrary.getElementObject(c);try{if(c[0].className.constructor==SVGAnimatedString){jsPlumb.util.svg.addClass(c[0],b)}}catch(d){}c.addClass(b)},animate:function(d,c,b){d.animate(c,b)},appendElement:function(c,b){jsPlumb.CurrentLibrary.getElementObject(b).append(c)},bind:function(b,c,d){b=jsPlumb.CurrentLibrary.getElementObject(b);b.bind(c,d)},dragEvents:{start:"start",stop:"stop",drag:"drag",step:"step",over:"over",out:"out",drop:"drop",complete:"complete"},extend:function(c,b){return a.extend(c,b)},getAttribute:function(b,c){return b.attr(c)},getClientXY:function(b){return[b.clientX,b.clientY]},getDocumentElement:function(){return document},getDragObject:function(b){return b[1].draggable},getDragScope:function(b){return b.draggable("option","scope")},getDropScope:function(b){return b.droppable("option","scope")},getDOMElement:function(b){if(typeof(b)=="string"){return document.getElementById(b)}else{if(b.context){return b[0]}else{return b}}},getElementObject:function(b){return typeof(b)=="string"?a("#"+b):a(b)},getOffset:function(b){return b.offset()},getPageXY:function(b){return[b.pageX,b.pageY]},getParent:function(b){return jsPlumb.CurrentLibrary.getElementObject(b).parent()},getScrollLeft:function(b){return b.scrollLeft()},getScrollTop:function(b){return b.scrollTop()},getSelector:function(b){return a(b)},getSize:function(b){return[b.outerWidth(),b.outerHeight()]},getTagName:function(b){var c=jsPlumb.CurrentLibrary.getElementObject(b);return c.length>0?c[0].tagName:null},getUIPosition:function(c){if(c.length==1){ret={left:c[0].pageX,top:c[0].pageY}}else{var d=c[1],b=d.offset;ret=b||d.absolutePosition}return ret},hasClass:function(c,b){return c.hasClass(b)},initDraggable:function(c,b){b=b||{};b.helper=null;b.scope=b.scope||jsPlumb.Defaults.Scope;c.draggable(b)},initDroppable:function(c,b){b.scope=b.scope||jsPlumb.Defaults.Scope;c.droppable(b)},isAlreadyDraggable:function(b){b=jsPlumb.CurrentLibrary.getElementObject(b);return b.hasClass("ui-draggable")},isDragSupported:function(c,b){return c.draggable},isDropSupported:function(c,b){return c.droppable},removeClass:function(c,b){c=jsPlumb.CurrentLibrary.getElementObject(c);try{if(c[0].className.constructor==SVGAnimatedString){jsPlumb.util.svg.removeClass(c[0],b)}}catch(d){}c.removeClass(b)},removeElement:function(b,c){jsPlumb.CurrentLibrary.getElementObject(b).remove()},setAttribute:function(c,d,b){c.attr(d,b)},setDraggable:function(c,b){c.draggable("option","disabled",!b)},setDragScope:function(c,b){c.draggable("option","scope",b)},setOffset:function(b,c){jsPlumb.CurrentLibrary.getElementObject(b).offset(c)},trigger:function(d,e,b){var c=jQuery._data(jsPlumb.CurrentLibrary.getElementObject(d)[0],"handle");c(b)},unbind:function(b,c,d){b=jsPlumb.CurrentLibrary.getElementObject(b);b.unbind(c,d)}};a(document).ready(jsPlumb.init)})(jQuery);(function(){if("undefined"==typeof Math.sgn){Math.sgn=function(l){return 0==l?0:0<l?1:-1}}var d={subtract:function(m,l){return{x:m.x-l.x,y:m.y-l.y}},dotProduct:function(m,l){return m.x*l.x+m.y*l.y},square:function(l){return Math.sqrt(l.x*l.x+l.y*l.y)},scale:function(m,l){return{x:m.x*l,y:m.y*l}}},f=Math.pow(2,-65),h=function(y,x){for(var t=[],v=x.length-1,r=2*v-1,s=[],w=[],p=[],q=[],o=[[1,0.6,0.3,0.1],[0.4,0.6,0.6,0.4],[0.1,0.3,0.6,1]],u=0;u<=v;u++){s[u]=d.subtract(x[u],y)}for(u=0;u<=v-1;u++){w[u]=d.subtract(x[u+1],x[u]),w[u]=d.scale(w[u],3)}for(u=0;u<=v-1;u++){for(var l=0;l<=v;l++){p[u]||(p[u]=[]),p[u][l]=d.dotProduct(w[u],s[l])}}for(u=0;u<=r;u++){q[u]||(q[u]=[]),q[u].y=0,q[u].x=parseFloat(u)/r}r=v-1;for(s=0;s<=v+r;s++){u=Math.max(0,s-r);for(w=Math.min(s,v);u<=w;u++){j=s-u,q[u+j].y+=p[j][u]*o[j][u]}}v=x.length-1;q=b(q,2*v-1,t,0);r=d.subtract(y,x[0]);p=d.square(r);for(u=o=0;u<q;u++){r=d.subtract(y,a(x,v,t[u],null,null)),r=d.square(r),r<p&&(p=r,o=t[u])}r=d.subtract(y,x[v]);r=d.square(r);r<p&&(p=r,o=1);return{location:o,distance:p}},b=function(C,B,x,z){var v=[],w=[],A=[],t=[],u=0,r,y;y=Math.sgn(C[0].y);for(var q=1;q<=B;q++){r=Math.sgn(C[q].y),r!=y&&u++,y=r}switch(u){case 0:return 0;case 1:if(64<=z){return x[0]=(C[0].x+C[B].x)/2,1}var p,u=C[0].y-C[B].y;r=C[B].x-C[0].x;y=C[0].x*C[B].y-C[B].x*C[0].y;q=max_distance_below=0;for(p=1;p<B;p++){var s=u*C[p].x+r*C[p].y+y;s>q?q=s:s<max_distance_below&&(max_distance_below=s)}p=r;q=(1*(y-q)-0*p)*(1/(0*p-1*u));p=r;u=(1*(y-max_distance_below)-0*p)*(1/(0*p-1*u));r=Math.min(q,u);if(Math.max(q,u)-r<f){return A=C[B].x-C[0].x,t=C[B].y-C[0].y,x[0]=0+1*(A*(C[0].y-0)-t*(C[0].x-0))*(1/(0*A-1*t)),1}}a(C,B,0.5,v,w);C=b(v,B,A,z+1);B=b(w,B,t,z+1);for(z=0;z<C;z++){x[z]=A[z]}for(z=0;z<B;z++){x[z+C]=t[z]}return C+B},a=function(m,l,p,q,n){for(var o=[[]],r=0;r<=l;r++){o[0][r]=m[r]}for(m=1;m<=l;m++){for(r=0;r<=l-m;r++){o[m]||(o[m]=[]),o[m][r]||(o[m][r]={}),o[m][r].x=(1-p)*o[m-1][r].x+p*o[m-1][r+1].x,o[m][r].y=(1-p)*o[m-1][r].y+p*o[m-1][r+1].y}}if(null!=q){for(r=0;r<=l;r++){q[r]=o[r][0]}}if(null!=n){for(r=0;r<=l;r++){n[r]=o[l-r][r]}}return o[l][0]},g={},e=function(t){var s=g[t];if(!s){var s=[],p=function(u){return function(){return u}},q=function(){return function(u){return u}},n=function(){return function(u){return 1-u}},o=function(u){return function(v){for(var x=1,w=0;w<u.length;w++){x*=u[w](v)}return x}};s.push(new function(){return function(u){return Math.pow(u,t)}});for(var r=1;r<t;r++){for(var l=[new p(t)],m=0;m<t-r;m++){l.push(new q)}for(m=0;m<r;m++){l.push(new n)}s.push(new o(l))}s.push(new function(){return function(u){return Math.pow(1-u,t)}});g[t]=s}return s},c=function(m,l){for(var p=e(m.length-1),q=0,n=0,o=0;o<m.length;o++){q+=m[o].x*p[o](l),n+=m[o].y*p[o](l)}return{x:q,y:n}},k=function(m,l,p){for(var q=c(m,l),n=0,o=0<p?1:-1,r=null;n<Math.abs(p);){l+=0.005*o,r=c(m,l),n+=Math.sqrt(Math.pow(r.x-q.x,2)+Math.pow(r.y-q.y,2)),q=r}return{point:r,location:l}},i=function(m,l){var o=c(m,l),p=c(m.slice(0,m.length-1),l),n=p.y-o.y,o=p.x-o.x;return 0==n?Infinity:Math.atan(n/o)};window.jsBezier={distanceFromCurve:h,gradientAtPoint:i,gradientAtPointAlongCurveFrom:function(m,l,n){l=k(m,l,n);if(1<l.location){l.location=1}if(0>l.location){l.location=0}return i(m,l.location)},nearestPointOnCurve:function(m,l){var n=h(m,l);return{point:a(l,l.length-1,n.location,null,null),location:n.location}},pointOnCurve:c,pointAlongCurveFrom:function(m,l,n){return k(m,l,n).point},perpendicularToCurveAt:function(m,l,n,o){l=k(m,l,null==o?0:o);m=i(m,l.location);o=Math.atan(-1/m);m=n/2*Math.sin(o);n=n/2*Math.cos(o);return[{x:l.point.x+n,y:l.point.y+m},{x:l.point.x-n,y:l.point.y-m}]}}})();
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/stylesheet/adminconsole.css b/hyracks-control-cc/src/main/resources/static/stylesheet/adminconsole.css
new file mode 100644
index 0000000..f8afc33
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/stylesheet/adminconsole.css
@@ -0,0 +1,6 @@
+div.time-chart {
+ width: 600px;
+ height: 300px;
+ margin-left: auto;
+ margin-right: auto;
+}
\ No newline at end of file
diff --git a/hyracks-control-cc/src/main/resources/static/stylesheet/jquery-ui/themes/base/jquery-ui.css b/hyracks-control-cc/src/main/resources/static/stylesheet/jquery-ui/themes/base/jquery-ui.css
new file mode 100644
index 0000000..efd6e8c
--- /dev/null
+++ b/hyracks-control-cc/src/main/resources/static/stylesheet/jquery-ui/themes/base/jquery-ui.css
@@ -0,0 +1,563 @@
+/*
+ * jQuery UI CSS Framework 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Theming/API
+ */
+
+/* Layout helpers
+----------------------------------*/
+.ui-helper-hidden { display: none; }
+.ui-helper-hidden-accessible { position: absolute !important; clip: rect(1px 1px 1px 1px); clip: rect(1px,1px,1px,1px); }
+.ui-helper-reset { margin: 0; padding: 0; border: 0; outline: 0; line-height: 1.3; text-decoration: none; font-size: 100%; list-style: none; }
+.ui-helper-clearfix:before, .ui-helper-clearfix:after { content: ""; display: table; }
+.ui-helper-clearfix:after { clear: both; }
+.ui-helper-clearfix { zoom: 1; }
+.ui-helper-zfix { width: 100%; height: 100%; top: 0; left: 0; position: absolute; opacity: 0; filter:Alpha(Opacity=0); }
+
+
+/* Interaction Cues
+----------------------------------*/
+.ui-state-disabled { cursor: default !important; }
+
+
+/* Icons
+----------------------------------*/
+
+/* states and images */
+.ui-icon { display: block; text-indent: -99999px; overflow: hidden; background-repeat: no-repeat; }
+
+
+/* Misc visuals
+----------------------------------*/
+
+/* Overlays */
+.ui-widget-overlay { position: absolute; top: 0; left: 0; width: 100%; height: 100%; }
+/*
+ * jQuery UI Accordion 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Accordion#theming
+ */
+/* IE/Win - Fix animation bug - #4615 */
+.ui-accordion { width: 100%; }
+.ui-accordion .ui-accordion-header { cursor: pointer; position: relative; margin-top: 1px; zoom: 1; }
+.ui-accordion .ui-accordion-li-fix { display: inline; }
+.ui-accordion .ui-accordion-header-active { border-bottom: 0 !important; }
+.ui-accordion .ui-accordion-header a { display: block; font-size: 1em; padding: .5em .5em .5em .7em; }
+.ui-accordion-icons .ui-accordion-header a { padding-left: 2.2em; }
+.ui-accordion .ui-accordion-header .ui-icon { position: absolute; left: .5em; top: 50%; margin-top: -8px; }
+.ui-accordion .ui-accordion-content { padding: 1em 2.2em; border-top: 0; margin-top: -2px; position: relative; top: 1px; margin-bottom: 2px; overflow: auto; display: none; zoom: 1; }
+.ui-accordion .ui-accordion-content-active { display: block; }
+/*
+ * jQuery UI Autocomplete 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Autocomplete#theming
+ */
+.ui-autocomplete { position: absolute; cursor: default; }
+
+/* workarounds */
+* html .ui-autocomplete { width:1px; } /* without this, the menu expands to 100% in IE6 */
+
+/*
+ * jQuery UI Menu 1.8.17
+ *
+ * Copyright 2010, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Menu#theming
+ */
+.ui-menu {
+ list-style:none;
+ padding: 2px;
+ margin: 0;
+ display:block;
+ float: left;
+}
+.ui-menu .ui-menu {
+ margin-top: -3px;
+}
+.ui-menu .ui-menu-item {
+ margin:0;
+ padding: 0;
+ zoom: 1;
+ float: left;
+ clear: left;
+ width: 100%;
+}
+.ui-menu .ui-menu-item a {
+ text-decoration:none;
+ display:block;
+ padding:.2em .4em;
+ line-height:1.5;
+ zoom:1;
+}
+.ui-menu .ui-menu-item a.ui-state-hover,
+.ui-menu .ui-menu-item a.ui-state-active {
+ font-weight: normal;
+ margin: -1px;
+}
+/*
+ * jQuery UI Button 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Button#theming
+ */
+.ui-button { display: inline-block; position: relative; padding: 0; margin-right: .1em; text-decoration: none !important; cursor: pointer; text-align: center; zoom: 1; overflow: visible; } /* the overflow property removes extra width in IE */
+.ui-button-icon-only { width: 2.2em; } /* to make room for the icon, a width needs to be set here */
+button.ui-button-icon-only { width: 2.4em; } /* button elements seem to need a little more width */
+.ui-button-icons-only { width: 3.4em; }
+button.ui-button-icons-only { width: 3.7em; }
+
+/*button text element */
+.ui-button .ui-button-text { display: block; line-height: 1.4; }
+.ui-button-text-only .ui-button-text { padding: .4em 1em; }
+.ui-button-icon-only .ui-button-text, .ui-button-icons-only .ui-button-text { padding: .4em; text-indent: -9999999px; }
+.ui-button-text-icon-primary .ui-button-text, .ui-button-text-icons .ui-button-text { padding: .4em 1em .4em 2.1em; }
+.ui-button-text-icon-secondary .ui-button-text, .ui-button-text-icons .ui-button-text { padding: .4em 2.1em .4em 1em; }
+.ui-button-text-icons .ui-button-text { padding-left: 2.1em; padding-right: 2.1em; }
+/* no icon support for input elements, provide padding by default */
+input.ui-button { padding: .4em 1em; }
+
+/*button icon element(s) */
+.ui-button-icon-only .ui-icon, .ui-button-text-icon-primary .ui-icon, .ui-button-text-icon-secondary .ui-icon, .ui-button-text-icons .ui-icon, .ui-button-icons-only .ui-icon { position: absolute; top: 50%; margin-top: -8px; }
+.ui-button-icon-only .ui-icon { left: 50%; margin-left: -8px; }
+.ui-button-text-icon-primary .ui-button-icon-primary, .ui-button-text-icons .ui-button-icon-primary, .ui-button-icons-only .ui-button-icon-primary { left: .5em; }
+.ui-button-text-icon-secondary .ui-button-icon-secondary, .ui-button-text-icons .ui-button-icon-secondary, .ui-button-icons-only .ui-button-icon-secondary { right: .5em; }
+.ui-button-text-icons .ui-button-icon-secondary, .ui-button-icons-only .ui-button-icon-secondary { right: .5em; }
+
+/*button sets*/
+.ui-buttonset { margin-right: 7px; }
+.ui-buttonset .ui-button { margin-left: 0; margin-right: -.3em; }
+
+/* workarounds */
+button.ui-button::-moz-focus-inner { border: 0; padding: 0; } /* reset extra padding in Firefox */
+/*
+ * jQuery UI Datepicker 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Datepicker#theming
+ */
+.ui-datepicker { width: 17em; padding: .2em .2em 0; display: none; }
+.ui-datepicker .ui-datepicker-header { position:relative; padding:.2em 0; }
+.ui-datepicker .ui-datepicker-prev, .ui-datepicker .ui-datepicker-next { position:absolute; top: 2px; width: 1.8em; height: 1.8em; }
+.ui-datepicker .ui-datepicker-prev-hover, .ui-datepicker .ui-datepicker-next-hover { top: 1px; }
+.ui-datepicker .ui-datepicker-prev { left:2px; }
+.ui-datepicker .ui-datepicker-next { right:2px; }
+.ui-datepicker .ui-datepicker-prev-hover { left:1px; }
+.ui-datepicker .ui-datepicker-next-hover { right:1px; }
+.ui-datepicker .ui-datepicker-prev span, .ui-datepicker .ui-datepicker-next span { display: block; position: absolute; left: 50%; margin-left: -8px; top: 50%; margin-top: -8px; }
+.ui-datepicker .ui-datepicker-title { margin: 0 2.3em; line-height: 1.8em; text-align: center; }
+.ui-datepicker .ui-datepicker-title select { font-size:1em; margin:1px 0; }
+.ui-datepicker select.ui-datepicker-month-year {width: 100%;}
+.ui-datepicker select.ui-datepicker-month,
+.ui-datepicker select.ui-datepicker-year { width: 49%;}
+.ui-datepicker table {width: 100%; font-size: .9em; border-collapse: collapse; margin:0 0 .4em; }
+.ui-datepicker th { padding: .7em .3em; text-align: center; font-weight: bold; border: 0; }
+.ui-datepicker td { border: 0; padding: 1px; }
+.ui-datepicker td span, .ui-datepicker td a { display: block; padding: .2em; text-align: right; text-decoration: none; }
+.ui-datepicker .ui-datepicker-buttonpane { background-image: none; margin: .7em 0 0 0; padding:0 .2em; border-left: 0; border-right: 0; border-bottom: 0; }
+.ui-datepicker .ui-datepicker-buttonpane button { float: right; margin: .5em .2em .4em; cursor: pointer; padding: .2em .6em .3em .6em; width:auto; overflow:visible; }
+.ui-datepicker .ui-datepicker-buttonpane button.ui-datepicker-current { float:left; }
+
+/* with multiple calendars */
+.ui-datepicker.ui-datepicker-multi { width:auto; }
+.ui-datepicker-multi .ui-datepicker-group { float:left; }
+.ui-datepicker-multi .ui-datepicker-group table { width:95%; margin:0 auto .4em; }
+.ui-datepicker-multi-2 .ui-datepicker-group { width:50%; }
+.ui-datepicker-multi-3 .ui-datepicker-group { width:33.3%; }
+.ui-datepicker-multi-4 .ui-datepicker-group { width:25%; }
+.ui-datepicker-multi .ui-datepicker-group-last .ui-datepicker-header { border-left-width:0; }
+.ui-datepicker-multi .ui-datepicker-group-middle .ui-datepicker-header { border-left-width:0; }
+.ui-datepicker-multi .ui-datepicker-buttonpane { clear:left; }
+.ui-datepicker-row-break { clear:both; width:100%; font-size:0em; }
+
+/* RTL support */
+.ui-datepicker-rtl { direction: rtl; }
+.ui-datepicker-rtl .ui-datepicker-prev { right: 2px; left: auto; }
+.ui-datepicker-rtl .ui-datepicker-next { left: 2px; right: auto; }
+.ui-datepicker-rtl .ui-datepicker-prev:hover { right: 1px; left: auto; }
+.ui-datepicker-rtl .ui-datepicker-next:hover { left: 1px; right: auto; }
+.ui-datepicker-rtl .ui-datepicker-buttonpane { clear:right; }
+.ui-datepicker-rtl .ui-datepicker-buttonpane button { float: left; }
+.ui-datepicker-rtl .ui-datepicker-buttonpane button.ui-datepicker-current { float:right; }
+.ui-datepicker-rtl .ui-datepicker-group { float:right; }
+.ui-datepicker-rtl .ui-datepicker-group-last .ui-datepicker-header { border-right-width:0; border-left-width:1px; }
+.ui-datepicker-rtl .ui-datepicker-group-middle .ui-datepicker-header { border-right-width:0; border-left-width:1px; }
+
+/* IE6 IFRAME FIX (taken from datepicker 1.5.3 */
+.ui-datepicker-cover {
+ display: none; /*sorry for IE5*/
+ display/**/: block; /*sorry for IE5*/
+ position: absolute; /*must have*/
+ z-index: -1; /*must have*/
+ filter: mask(); /*must have*/
+ top: -4px; /*must have*/
+ left: -4px; /*must have*/
+ width: 200px; /*must have*/
+ height: 200px; /*must have*/
+}/*
+ * jQuery UI Dialog 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Dialog#theming
+ */
+.ui-dialog { position: absolute; padding: .2em; width: 300px; overflow: hidden; }
+.ui-dialog .ui-dialog-titlebar { padding: .4em 1em; position: relative; }
+.ui-dialog .ui-dialog-title { float: left; margin: .1em 16px .1em 0; }
+.ui-dialog .ui-dialog-titlebar-close { position: absolute; right: .3em; top: 50%; width: 19px; margin: -10px 0 0 0; padding: 1px; height: 18px; }
+.ui-dialog .ui-dialog-titlebar-close span { display: block; margin: 1px; }
+.ui-dialog .ui-dialog-titlebar-close:hover, .ui-dialog .ui-dialog-titlebar-close:focus { padding: 0; }
+.ui-dialog .ui-dialog-content { position: relative; border: 0; padding: .5em 1em; background: none; overflow: auto; zoom: 1; }
+.ui-dialog .ui-dialog-buttonpane { text-align: left; border-width: 1px 0 0 0; background-image: none; margin: .5em 0 0 0; padding: .3em 1em .5em .4em; }
+.ui-dialog .ui-dialog-buttonpane .ui-dialog-buttonset { float: right; }
+.ui-dialog .ui-dialog-buttonpane button { margin: .5em .4em .5em 0; cursor: pointer; }
+.ui-dialog .ui-resizable-se { width: 14px; height: 14px; right: 3px; bottom: 3px; }
+.ui-draggable .ui-dialog-titlebar { cursor: move; }
+/*
+ * jQuery UI Progressbar 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Progressbar#theming
+ */
+.ui-progressbar { height:2em; text-align: left; overflow: hidden; }
+.ui-progressbar .ui-progressbar-value {margin: -1px; height:100%; }/*
+ * jQuery UI Resizable 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Resizable#theming
+ */
+.ui-resizable { position: relative;}
+.ui-resizable-handle { position: absolute;font-size: 0.1px;z-index: 99999; display: block; }
+.ui-resizable-disabled .ui-resizable-handle, .ui-resizable-autohide .ui-resizable-handle { display: none; }
+.ui-resizable-n { cursor: n-resize; height: 7px; width: 100%; top: -5px; left: 0; }
+.ui-resizable-s { cursor: s-resize; height: 7px; width: 100%; bottom: -5px; left: 0; }
+.ui-resizable-e { cursor: e-resize; width: 7px; right: -5px; top: 0; height: 100%; }
+.ui-resizable-w { cursor: w-resize; width: 7px; left: -5px; top: 0; height: 100%; }
+.ui-resizable-se { cursor: se-resize; width: 12px; height: 12px; right: 1px; bottom: 1px; }
+.ui-resizable-sw { cursor: sw-resize; width: 9px; height: 9px; left: -5px; bottom: -5px; }
+.ui-resizable-nw { cursor: nw-resize; width: 9px; height: 9px; left: -5px; top: -5px; }
+.ui-resizable-ne { cursor: ne-resize; width: 9px; height: 9px; right: -5px; top: -5px;}/*
+ * jQuery UI Selectable 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Selectable#theming
+ */
+.ui-selectable-helper { position: absolute; z-index: 100; border:1px dotted black; }
+/*
+ * jQuery UI Slider 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Slider#theming
+ */
+.ui-slider { position: relative; text-align: left; }
+.ui-slider .ui-slider-handle { position: absolute; z-index: 2; width: 1.2em; height: 1.2em; cursor: default; }
+.ui-slider .ui-slider-range { position: absolute; z-index: 1; font-size: .7em; display: block; border: 0; background-position: 0 0; }
+
+.ui-slider-horizontal { height: .8em; }
+.ui-slider-horizontal .ui-slider-handle { top: -.3em; margin-left: -.6em; }
+.ui-slider-horizontal .ui-slider-range { top: 0; height: 100%; }
+.ui-slider-horizontal .ui-slider-range-min { left: 0; }
+.ui-slider-horizontal .ui-slider-range-max { right: 0; }
+
+.ui-slider-vertical { width: .8em; height: 100px; }
+.ui-slider-vertical .ui-slider-handle { left: -.3em; margin-left: 0; margin-bottom: -.6em; }
+.ui-slider-vertical .ui-slider-range { left: 0; width: 100%; }
+.ui-slider-vertical .ui-slider-range-min { bottom: 0; }
+.ui-slider-vertical .ui-slider-range-max { top: 0; }/*
+ * jQuery UI Tabs 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Tabs#theming
+ */
+.ui-tabs { position: relative; padding: .2em; zoom: 1; } /* position: relative prevents IE scroll bug (element with position: relative inside container with overflow: auto appear as "fixed") */
+.ui-tabs .ui-tabs-nav { margin: 0; padding: .2em .2em 0; }
+.ui-tabs .ui-tabs-nav li { list-style: none; float: left; position: relative; top: 1px; margin: 0 .2em 1px 0; border-bottom: 0 !important; padding: 0; white-space: nowrap; }
+.ui-tabs .ui-tabs-nav li a { float: left; padding: .5em 1em; text-decoration: none; }
+.ui-tabs .ui-tabs-nav li.ui-tabs-selected { margin-bottom: 0; padding-bottom: 1px; }
+.ui-tabs .ui-tabs-nav li.ui-tabs-selected a, .ui-tabs .ui-tabs-nav li.ui-state-disabled a, .ui-tabs .ui-tabs-nav li.ui-state-processing a { cursor: text; }
+.ui-tabs .ui-tabs-nav li a, .ui-tabs.ui-tabs-collapsible .ui-tabs-nav li.ui-tabs-selected a { cursor: pointer; } /* first selector in group seems obsolete, but required to overcome bug in Opera applying cursor: text overall if defined elsewhere... */
+.ui-tabs .ui-tabs-panel { display: block; border-width: 0; padding: 1em 1.4em; background: none; }
+.ui-tabs .ui-tabs-hide { display: none !important; }
+/*
+ * jQuery UI CSS Framework 1.8.17
+ *
+ * Copyright 2011, AUTHORS.txt (http://jqueryui.com/about)
+ * Dual licensed under the MIT or GPL Version 2 licenses.
+ * http://jquery.org/license
+ *
+ * http://docs.jquery.com/UI/Theming/API
+ *
+ * To view and modify this theme, visit http://jqueryui.com/themeroller/
+ */
+
+
+/* Component containers
+----------------------------------*/
+.ui-widget { font-family: Verdana,Arial,sans-serif/*{ffDefault}*/; font-size: 1.1em/*{fsDefault}*/; }
+.ui-widget .ui-widget { font-size: 1em; }
+.ui-widget input, .ui-widget select, .ui-widget textarea, .ui-widget button { font-family: Verdana,Arial,sans-serif/*{ffDefault}*/; font-size: 1em; }
+.ui-widget-content { border: 1px solid #aaaaaa/*{borderColorContent}*/; background: #ffffff/*{bgColorContent}*/ url(images/ui-bg_flat_75_ffffff_40x100.png)/*{bgImgUrlContent}*/ 50%/*{bgContentXPos}*/ 50%/*{bgContentYPos}*/ repeat-x/*{bgContentRepeat}*/; color: #222222/*{fcContent}*/; }
+.ui-widget-content a { color: #222222/*{fcContent}*/; }
+.ui-widget-header { border: 1px solid #aaaaaa/*{borderColorHeader}*/; background: #cccccc/*{bgColorHeader}*/ url(images/ui-bg_highlight-soft_75_cccccc_1x100.png)/*{bgImgUrlHeader}*/ 50%/*{bgHeaderXPos}*/ 50%/*{bgHeaderYPos}*/ repeat-x/*{bgHeaderRepeat}*/; color: #222222/*{fcHeader}*/; font-weight: bold; }
+.ui-widget-header a { color: #222222/*{fcHeader}*/; }
+
+/* Interaction states
+----------------------------------*/
+.ui-state-default, .ui-widget-content .ui-state-default, .ui-widget-header .ui-state-default { border: 1px solid #d3d3d3/*{borderColorDefault}*/; background: #e6e6e6/*{bgColorDefault}*/ url(images/ui-bg_glass_75_e6e6e6_1x400.png)/*{bgImgUrlDefault}*/ 50%/*{bgDefaultXPos}*/ 50%/*{bgDefaultYPos}*/ repeat-x/*{bgDefaultRepeat}*/; font-weight: normal/*{fwDefault}*/; color: #555555/*{fcDefault}*/; }
+.ui-state-default a, .ui-state-default a:link, .ui-state-default a:visited { color: #555555/*{fcDefault}*/; text-decoration: none; }
+.ui-state-hover, .ui-widget-content .ui-state-hover, .ui-widget-header .ui-state-hover, .ui-state-focus, .ui-widget-content .ui-state-focus, .ui-widget-header .ui-state-focus { border: 1px solid #999999/*{borderColorHover}*/; background: #dadada/*{bgColorHover}*/ url(images/ui-bg_glass_75_dadada_1x400.png)/*{bgImgUrlHover}*/ 50%/*{bgHoverXPos}*/ 50%/*{bgHoverYPos}*/ repeat-x/*{bgHoverRepeat}*/; font-weight: normal/*{fwDefault}*/; color: #212121/*{fcHover}*/; }
+.ui-state-hover a, .ui-state-hover a:hover { color: #212121/*{fcHover}*/; text-decoration: none; }
+.ui-state-active, .ui-widget-content .ui-state-active, .ui-widget-header .ui-state-active { border: 1px solid #aaaaaa/*{borderColorActive}*/; background: #ffffff/*{bgColorActive}*/ url(images/ui-bg_glass_65_ffffff_1x400.png)/*{bgImgUrlActive}*/ 50%/*{bgActiveXPos}*/ 50%/*{bgActiveYPos}*/ repeat-x/*{bgActiveRepeat}*/; font-weight: normal/*{fwDefault}*/; color: #212121/*{fcActive}*/; }
+.ui-state-active a, .ui-state-active a:link, .ui-state-active a:visited { color: #212121/*{fcActive}*/; text-decoration: none; }
+.ui-widget :active { outline: none; }
+
+/* Interaction Cues
+----------------------------------*/
+.ui-state-highlight, .ui-widget-content .ui-state-highlight, .ui-widget-header .ui-state-highlight {border: 1px solid #fcefa1/*{borderColorHighlight}*/; background: #fbf9ee/*{bgColorHighlight}*/ url(images/ui-bg_glass_55_fbf9ee_1x400.png)/*{bgImgUrlHighlight}*/ 50%/*{bgHighlightXPos}*/ 50%/*{bgHighlightYPos}*/ repeat-x/*{bgHighlightRepeat}*/; color: #363636/*{fcHighlight}*/; }
+.ui-state-highlight a, .ui-widget-content .ui-state-highlight a,.ui-widget-header .ui-state-highlight a { color: #363636/*{fcHighlight}*/; }
+.ui-state-error, .ui-widget-content .ui-state-error, .ui-widget-header .ui-state-error {border: 1px solid #cd0a0a/*{borderColorError}*/; background: #fef1ec/*{bgColorError}*/ url(images/ui-bg_glass_95_fef1ec_1x400.png)/*{bgImgUrlError}*/ 50%/*{bgErrorXPos}*/ 50%/*{bgErrorYPos}*/ repeat-x/*{bgErrorRepeat}*/; color: #cd0a0a/*{fcError}*/; }
+.ui-state-error a, .ui-widget-content .ui-state-error a, .ui-widget-header .ui-state-error a { color: #cd0a0a/*{fcError}*/; }
+.ui-state-error-text, .ui-widget-content .ui-state-error-text, .ui-widget-header .ui-state-error-text { color: #cd0a0a/*{fcError}*/; }
+.ui-priority-primary, .ui-widget-content .ui-priority-primary, .ui-widget-header .ui-priority-primary { font-weight: bold; }
+.ui-priority-secondary, .ui-widget-content .ui-priority-secondary, .ui-widget-header .ui-priority-secondary { opacity: .7; filter:Alpha(Opacity=70); font-weight: normal; }
+.ui-state-disabled, .ui-widget-content .ui-state-disabled, .ui-widget-header .ui-state-disabled { opacity: .35; filter:Alpha(Opacity=35); background-image: none; }
+
+/* Icons
+----------------------------------*/
+
+/* states and images */
+.ui-icon { width: 16px; height: 16px; background-image: url(images/ui-icons_222222_256x240.png)/*{iconsContent}*/; }
+.ui-widget-content .ui-icon {background-image: url(images/ui-icons_222222_256x240.png)/*{iconsContent}*/; }
+.ui-widget-header .ui-icon {background-image: url(images/ui-icons_222222_256x240.png)/*{iconsHeader}*/; }
+.ui-state-default .ui-icon { background-image: url(images/ui-icons_888888_256x240.png)/*{iconsDefault}*/; }
+.ui-state-hover .ui-icon, .ui-state-focus .ui-icon {background-image: url(images/ui-icons_454545_256x240.png)/*{iconsHover}*/; }
+.ui-state-active .ui-icon {background-image: url(images/ui-icons_454545_256x240.png)/*{iconsActive}*/; }
+.ui-state-highlight .ui-icon {background-image: url(images/ui-icons_2e83ff_256x240.png)/*{iconsHighlight}*/; }
+.ui-state-error .ui-icon, .ui-state-error-text .ui-icon {background-image: url(images/ui-icons_cd0a0a_256x240.png)/*{iconsError}*/; }
+
+/* positioning */
+.ui-icon-carat-1-n { background-position: 0 0; }
+.ui-icon-carat-1-ne { background-position: -16px 0; }
+.ui-icon-carat-1-e { background-position: -32px 0; }
+.ui-icon-carat-1-se { background-position: -48px 0; }
+.ui-icon-carat-1-s { background-position: -64px 0; }
+.ui-icon-carat-1-sw { background-position: -80px 0; }
+.ui-icon-carat-1-w { background-position: -96px 0; }
+.ui-icon-carat-1-nw { background-position: -112px 0; }
+.ui-icon-carat-2-n-s { background-position: -128px 0; }
+.ui-icon-carat-2-e-w { background-position: -144px 0; }
+.ui-icon-triangle-1-n { background-position: 0 -16px; }
+.ui-icon-triangle-1-ne { background-position: -16px -16px; }
+.ui-icon-triangle-1-e { background-position: -32px -16px; }
+.ui-icon-triangle-1-se { background-position: -48px -16px; }
+.ui-icon-triangle-1-s { background-position: -64px -16px; }
+.ui-icon-triangle-1-sw { background-position: -80px -16px; }
+.ui-icon-triangle-1-w { background-position: -96px -16px; }
+.ui-icon-triangle-1-nw { background-position: -112px -16px; }
+.ui-icon-triangle-2-n-s { background-position: -128px -16px; }
+.ui-icon-triangle-2-e-w { background-position: -144px -16px; }
+.ui-icon-arrow-1-n { background-position: 0 -32px; }
+.ui-icon-arrow-1-ne { background-position: -16px -32px; }
+.ui-icon-arrow-1-e { background-position: -32px -32px; }
+.ui-icon-arrow-1-se { background-position: -48px -32px; }
+.ui-icon-arrow-1-s { background-position: -64px -32px; }
+.ui-icon-arrow-1-sw { background-position: -80px -32px; }
+.ui-icon-arrow-1-w { background-position: -96px -32px; }
+.ui-icon-arrow-1-nw { background-position: -112px -32px; }
+.ui-icon-arrow-2-n-s { background-position: -128px -32px; }
+.ui-icon-arrow-2-ne-sw { background-position: -144px -32px; }
+.ui-icon-arrow-2-e-w { background-position: -160px -32px; }
+.ui-icon-arrow-2-se-nw { background-position: -176px -32px; }
+.ui-icon-arrowstop-1-n { background-position: -192px -32px; }
+.ui-icon-arrowstop-1-e { background-position: -208px -32px; }
+.ui-icon-arrowstop-1-s { background-position: -224px -32px; }
+.ui-icon-arrowstop-1-w { background-position: -240px -32px; }
+.ui-icon-arrowthick-1-n { background-position: 0 -48px; }
+.ui-icon-arrowthick-1-ne { background-position: -16px -48px; }
+.ui-icon-arrowthick-1-e { background-position: -32px -48px; }
+.ui-icon-arrowthick-1-se { background-position: -48px -48px; }
+.ui-icon-arrowthick-1-s { background-position: -64px -48px; }
+.ui-icon-arrowthick-1-sw { background-position: -80px -48px; }
+.ui-icon-arrowthick-1-w { background-position: -96px -48px; }
+.ui-icon-arrowthick-1-nw { background-position: -112px -48px; }
+.ui-icon-arrowthick-2-n-s { background-position: -128px -48px; }
+.ui-icon-arrowthick-2-ne-sw { background-position: -144px -48px; }
+.ui-icon-arrowthick-2-e-w { background-position: -160px -48px; }
+.ui-icon-arrowthick-2-se-nw { background-position: -176px -48px; }
+.ui-icon-arrowthickstop-1-n { background-position: -192px -48px; }
+.ui-icon-arrowthickstop-1-e { background-position: -208px -48px; }
+.ui-icon-arrowthickstop-1-s { background-position: -224px -48px; }
+.ui-icon-arrowthickstop-1-w { background-position: -240px -48px; }
+.ui-icon-arrowreturnthick-1-w { background-position: 0 -64px; }
+.ui-icon-arrowreturnthick-1-n { background-position: -16px -64px; }
+.ui-icon-arrowreturnthick-1-e { background-position: -32px -64px; }
+.ui-icon-arrowreturnthick-1-s { background-position: -48px -64px; }
+.ui-icon-arrowreturn-1-w { background-position: -64px -64px; }
+.ui-icon-arrowreturn-1-n { background-position: -80px -64px; }
+.ui-icon-arrowreturn-1-e { background-position: -96px -64px; }
+.ui-icon-arrowreturn-1-s { background-position: -112px -64px; }
+.ui-icon-arrowrefresh-1-w { background-position: -128px -64px; }
+.ui-icon-arrowrefresh-1-n { background-position: -144px -64px; }
+.ui-icon-arrowrefresh-1-e { background-position: -160px -64px; }
+.ui-icon-arrowrefresh-1-s { background-position: -176px -64px; }
+.ui-icon-arrow-4 { background-position: 0 -80px; }
+.ui-icon-arrow-4-diag { background-position: -16px -80px; }
+.ui-icon-extlink { background-position: -32px -80px; }
+.ui-icon-newwin { background-position: -48px -80px; }
+.ui-icon-refresh { background-position: -64px -80px; }
+.ui-icon-shuffle { background-position: -80px -80px; }
+.ui-icon-transfer-e-w { background-position: -96px -80px; }
+.ui-icon-transferthick-e-w { background-position: -112px -80px; }
+.ui-icon-folder-collapsed { background-position: 0 -96px; }
+.ui-icon-folder-open { background-position: -16px -96px; }
+.ui-icon-document { background-position: -32px -96px; }
+.ui-icon-document-b { background-position: -48px -96px; }
+.ui-icon-note { background-position: -64px -96px; }
+.ui-icon-mail-closed { background-position: -80px -96px; }
+.ui-icon-mail-open { background-position: -96px -96px; }
+.ui-icon-suitcase { background-position: -112px -96px; }
+.ui-icon-comment { background-position: -128px -96px; }
+.ui-icon-person { background-position: -144px -96px; }
+.ui-icon-print { background-position: -160px -96px; }
+.ui-icon-trash { background-position: -176px -96px; }
+.ui-icon-locked { background-position: -192px -96px; }
+.ui-icon-unlocked { background-position: -208px -96px; }
+.ui-icon-bookmark { background-position: -224px -96px; }
+.ui-icon-tag { background-position: -240px -96px; }
+.ui-icon-home { background-position: 0 -112px; }
+.ui-icon-flag { background-position: -16px -112px; }
+.ui-icon-calendar { background-position: -32px -112px; }
+.ui-icon-cart { background-position: -48px -112px; }
+.ui-icon-pencil { background-position: -64px -112px; }
+.ui-icon-clock { background-position: -80px -112px; }
+.ui-icon-disk { background-position: -96px -112px; }
+.ui-icon-calculator { background-position: -112px -112px; }
+.ui-icon-zoomin { background-position: -128px -112px; }
+.ui-icon-zoomout { background-position: -144px -112px; }
+.ui-icon-search { background-position: -160px -112px; }
+.ui-icon-wrench { background-position: -176px -112px; }
+.ui-icon-gear { background-position: -192px -112px; }
+.ui-icon-heart { background-position: -208px -112px; }
+.ui-icon-star { background-position: -224px -112px; }
+.ui-icon-link { background-position: -240px -112px; }
+.ui-icon-cancel { background-position: 0 -128px; }
+.ui-icon-plus { background-position: -16px -128px; }
+.ui-icon-plusthick { background-position: -32px -128px; }
+.ui-icon-minus { background-position: -48px -128px; }
+.ui-icon-minusthick { background-position: -64px -128px; }
+.ui-icon-close { background-position: -80px -128px; }
+.ui-icon-closethick { background-position: -96px -128px; }
+.ui-icon-key { background-position: -112px -128px; }
+.ui-icon-lightbulb { background-position: -128px -128px; }
+.ui-icon-scissors { background-position: -144px -128px; }
+.ui-icon-clipboard { background-position: -160px -128px; }
+.ui-icon-copy { background-position: -176px -128px; }
+.ui-icon-contact { background-position: -192px -128px; }
+.ui-icon-image { background-position: -208px -128px; }
+.ui-icon-video { background-position: -224px -128px; }
+.ui-icon-script { background-position: -240px -128px; }
+.ui-icon-alert { background-position: 0 -144px; }
+.ui-icon-info { background-position: -16px -144px; }
+.ui-icon-notice { background-position: -32px -144px; }
+.ui-icon-help { background-position: -48px -144px; }
+.ui-icon-check { background-position: -64px -144px; }
+.ui-icon-bullet { background-position: -80px -144px; }
+.ui-icon-radio-off { background-position: -96px -144px; }
+.ui-icon-radio-on { background-position: -112px -144px; }
+.ui-icon-pin-w { background-position: -128px -144px; }
+.ui-icon-pin-s { background-position: -144px -144px; }
+.ui-icon-play { background-position: 0 -160px; }
+.ui-icon-pause { background-position: -16px -160px; }
+.ui-icon-seek-next { background-position: -32px -160px; }
+.ui-icon-seek-prev { background-position: -48px -160px; }
+.ui-icon-seek-end { background-position: -64px -160px; }
+.ui-icon-seek-start { background-position: -80px -160px; }
+/* ui-icon-seek-first is deprecated, use ui-icon-seek-start instead */
+.ui-icon-seek-first { background-position: -80px -160px; }
+.ui-icon-stop { background-position: -96px -160px; }
+.ui-icon-eject { background-position: -112px -160px; }
+.ui-icon-volume-off { background-position: -128px -160px; }
+.ui-icon-volume-on { background-position: -144px -160px; }
+.ui-icon-power { background-position: 0 -176px; }
+.ui-icon-signal-diag { background-position: -16px -176px; }
+.ui-icon-signal { background-position: -32px -176px; }
+.ui-icon-battery-0 { background-position: -48px -176px; }
+.ui-icon-battery-1 { background-position: -64px -176px; }
+.ui-icon-battery-2 { background-position: -80px -176px; }
+.ui-icon-battery-3 { background-position: -96px -176px; }
+.ui-icon-circle-plus { background-position: 0 -192px; }
+.ui-icon-circle-minus { background-position: -16px -192px; }
+.ui-icon-circle-close { background-position: -32px -192px; }
+.ui-icon-circle-triangle-e { background-position: -48px -192px; }
+.ui-icon-circle-triangle-s { background-position: -64px -192px; }
+.ui-icon-circle-triangle-w { background-position: -80px -192px; }
+.ui-icon-circle-triangle-n { background-position: -96px -192px; }
+.ui-icon-circle-arrow-e { background-position: -112px -192px; }
+.ui-icon-circle-arrow-s { background-position: -128px -192px; }
+.ui-icon-circle-arrow-w { background-position: -144px -192px; }
+.ui-icon-circle-arrow-n { background-position: -160px -192px; }
+.ui-icon-circle-zoomin { background-position: -176px -192px; }
+.ui-icon-circle-zoomout { background-position: -192px -192px; }
+.ui-icon-circle-check { background-position: -208px -192px; }
+.ui-icon-circlesmall-plus { background-position: 0 -208px; }
+.ui-icon-circlesmall-minus { background-position: -16px -208px; }
+.ui-icon-circlesmall-close { background-position: -32px -208px; }
+.ui-icon-squaresmall-plus { background-position: -48px -208px; }
+.ui-icon-squaresmall-minus { background-position: -64px -208px; }
+.ui-icon-squaresmall-close { background-position: -80px -208px; }
+.ui-icon-grip-dotted-vertical { background-position: 0 -224px; }
+.ui-icon-grip-dotted-horizontal { background-position: -16px -224px; }
+.ui-icon-grip-solid-vertical { background-position: -32px -224px; }
+.ui-icon-grip-solid-horizontal { background-position: -48px -224px; }
+.ui-icon-gripsmall-diagonal-se { background-position: -64px -224px; }
+.ui-icon-grip-diagonal-se { background-position: -80px -224px; }
+
+
+/* Misc visuals
+----------------------------------*/
+
+/* Corner radius */
+.ui-corner-all, .ui-corner-top, .ui-corner-left, .ui-corner-tl { -moz-border-radius-topleft: 4px/*{cornerRadius}*/; -webkit-border-top-left-radius: 4px/*{cornerRadius}*/; -khtml-border-top-left-radius: 4px/*{cornerRadius}*/; border-top-left-radius: 4px/*{cornerRadius}*/; }
+.ui-corner-all, .ui-corner-top, .ui-corner-right, .ui-corner-tr { -moz-border-radius-topright: 4px/*{cornerRadius}*/; -webkit-border-top-right-radius: 4px/*{cornerRadius}*/; -khtml-border-top-right-radius: 4px/*{cornerRadius}*/; border-top-right-radius: 4px/*{cornerRadius}*/; }
+.ui-corner-all, .ui-corner-bottom, .ui-corner-left, .ui-corner-bl { -moz-border-radius-bottomleft: 4px/*{cornerRadius}*/; -webkit-border-bottom-left-radius: 4px/*{cornerRadius}*/; -khtml-border-bottom-left-radius: 4px/*{cornerRadius}*/; border-bottom-left-radius: 4px/*{cornerRadius}*/; }
+.ui-corner-all, .ui-corner-bottom, .ui-corner-right, .ui-corner-br { -moz-border-radius-bottomright: 4px/*{cornerRadius}*/; -webkit-border-bottom-right-radius: 4px/*{cornerRadius}*/; -khtml-border-bottom-right-radius: 4px/*{cornerRadius}*/; border-bottom-right-radius: 4px/*{cornerRadius}*/; }
+
+/* Overlays */
+.ui-widget-overlay { background: #aaaaaa/*{bgColorOverlay}*/ url(images/ui-bg_flat_0_aaaaaa_40x100.png)/*{bgImgUrlOverlay}*/ 50%/*{bgOverlayXPos}*/ 50%/*{bgOverlayYPos}*/ repeat-x/*{bgOverlayRepeat}*/; opacity: .3;filter:Alpha(Opacity=30)/*{opacityOverlay}*/; }
+.ui-widget-shadow { margin: -8px/*{offsetTopShadow}*/ 0 0 -8px/*{offsetLeftShadow}*/; padding: 8px/*{thicknessShadow}*/; background: #aaaaaa/*{bgColorShadow}*/ url(images/ui-bg_flat_0_aaaaaa_40x100.png)/*{bgImgUrlShadow}*/ 50%/*{bgShadowXPos}*/ 50%/*{bgShadowYPos}*/ repeat-x/*{bgShadowRepeat}*/; opacity: .3;filter:Alpha(Opacity=30)/*{opacityShadow}*/; -moz-border-radius: 8px/*{cornerRadiusShadow}*/; -khtml-border-radius: 8px/*{cornerRadiusShadow}*/; -webkit-border-radius: 8px/*{cornerRadiusShadow}*/; border-radius: 8px/*{cornerRadiusShadow}*/; }
\ No newline at end of file
diff --git a/hyracks-control-common/.classpath b/hyracks-control-common/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-control-common/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-control-common/.project b/hyracks-control-common/.project
deleted file mode 100644
index f6c06b0..0000000
--- a/hyracks-control-common/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-control-common</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-control-common/.settings/org.eclipse.jdt.core.prefs b/hyracks-control-common/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 450f5c4..0000000
--- a/hyracks-control-common/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:04 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-control-common/.settings/org.maven.ide.eclipse.prefs b/hyracks-control-common/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 7e87666..0000000
--- a/hyracks-control-common/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Fri Jul 30 07:32:48 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-control-common/pom.xml b/hyracks-control-common/pom.xml
index 3ff12d7..fc7a2d5 100644
--- a/hyracks-control-common/pom.xml
+++ b/hyracks-control-common/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,7 +27,7 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/AbstractRemoteService.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/AbstractRemoteService.java
index cf64554..21e3e9d 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/AbstractRemoteService.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/AbstractRemoteService.java
@@ -14,14 +14,9 @@
*/
package edu.uci.ics.hyracks.control.common;
-import java.rmi.RemoteException;
-import java.rmi.server.UnicastRemoteObject;
-
import edu.uci.ics.hyracks.control.common.service.IService;
-public abstract class AbstractRemoteService extends UnicastRemoteObject implements IService {
- private static final long serialVersionUID = 1L;
-
- public AbstractRemoteService() throws RemoteException {
+public abstract class AbstractRemoteService implements IService {
+ public AbstractRemoteService() {
}
-}
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java
index c02c7ad..9f29fbd 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationContext.java
@@ -59,7 +59,6 @@
this.serverCtx = serverCtx;
this.appName = appName;
this.applicationRootDir = new File(new File(serverCtx.getBaseDir(), APPLICATION_ROOT), appName);
- status = ApplicationStatus.CREATED;
FileUtils.deleteDirectory(applicationRootDir);
applicationRootDir.mkdirs();
}
@@ -87,9 +86,6 @@
}
public void initialize() throws Exception {
- if (status != ApplicationStatus.CREATED) {
- throw new IllegalStateException();
- }
if (deploymentDescriptor != null) {
String bootstrapClass = null;
switch (serverCtx.getServerType()) {
@@ -107,7 +103,6 @@
start();
}
}
- status = ApplicationStatus.INITIALIZED;
}
protected abstract void start() throws Exception;
@@ -168,7 +163,6 @@
}
public void deinitialize() throws Exception {
- status = ApplicationStatus.DEINITIALIZED;
stop();
File expandedFolder = getExpandedFolder();
FileUtils.deleteDirectory(expandedFolder);
@@ -195,7 +189,7 @@
}
@Override
- public Serializable getDestributedState() {
+ public Serializable getDistributedState() {
return distributedState;
}
@@ -203,4 +197,12 @@
public ClassLoader getClassLoader() {
return classLoader;
}
-}
+
+ public void setStatus(ApplicationStatus status) {
+ this.status = status;
+ }
+
+ public ApplicationStatus getStatus() {
+ return status;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationStatus.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationStatus.java
index 8a97ad6..b45c4f1 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationStatus.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/application/ApplicationStatus.java
@@ -16,6 +16,8 @@
public enum ApplicationStatus {
CREATED,
+ IN_INITIALIZATION,
INITIALIZED,
+ IN_DEINITIALIZATION,
DEINITIALIZED
}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java
index ba0bc86..a03f5c9 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/IClusterController.java
@@ -14,12 +14,11 @@
*/
package edu.uci.ics.hyracks.control.common.base;
-import java.rmi.Remote;
import java.util.List;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
@@ -27,15 +26,17 @@
import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
-public interface IClusterController extends Remote {
- public NodeParameters registerNode(NodeRegistration reg) throws Exception;
+public interface IClusterController {
+ public void registerNode(NodeRegistration reg) throws Exception;
- public void unregisterNode(INodeController nodeController) throws Exception;
+ public void unregisterNode(String nodeId) throws Exception;
public void notifyTaskComplete(JobId jobId, TaskAttemptId taskId, String nodeId, TaskProfile statistics)
throws Exception;
- public void notifyTaskFailure(JobId jobId, TaskAttemptId taskId, String nodeId, Exception e) throws Exception;
+ public void notifyTaskFailure(JobId jobId, TaskAttemptId taskId, String nodeId, String details) throws Exception;
+
+ public void notifyJobletCleanup(JobId jobId, String nodeId) throws Exception;
public void nodeHeartbeat(String id, HeartbeatData hbData) throws Exception;
@@ -44,4 +45,6 @@
public void registerPartitionProvider(PartitionDescriptor partitionDescriptor) throws Exception;
public void registerPartitionRequest(PartitionRequest partitionRequest) throws Exception;
+
+ public void notifyApplicationStateChange(String nodeId, String appName, ApplicationStatus status) throws Exception;
}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java
index efaa921..47f4ceb 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/base/INodeController.java
@@ -14,7 +14,6 @@
*/
package edu.uci.ics.hyracks.control.common.base;
-import java.rmi.Remote;
import java.util.List;
import java.util.Map;
@@ -23,23 +22,17 @@
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
-public interface INodeController extends Remote {
- public String getId() throws Exception;
-
- public NCConfig getConfiguration() throws Exception;
-
+public interface INodeController {
public void startTasks(String appName, JobId jobId, byte[] planBytes, List<TaskAttemptDescriptor> taskDescriptors,
- Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies, byte[] ctxVarBytes) throws Exception;
+ Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies) throws Exception;
public void abortTasks(JobId jobId, List<TaskAttemptId> tasks) throws Exception;
- public void cleanUpJob(JobId jobId) throws Exception;
-
- public void notifyRegistration(IClusterController ccs) throws Exception;
+ public void cleanUpJoblet(JobId jobId, JobStatus status) throws Exception;
public void createApplication(String appName, boolean deployHar, byte[] serializedDistributedState)
throws Exception;
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java
index 6afa43e..350f527 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/CCConfig.java
@@ -19,11 +19,20 @@
import org.kohsuke.args4j.Option;
public class CCConfig {
- @Option(name = "-port", usage = "Sets the port to listen for connections from node controllers (default 1099)")
- public int port = 1099;
+ @Option(name = "-client-net-ip-address", usage = "Sets the IP Address to listen for connections from clients", required = true)
+ public String clientNetIpAddress;
- @Option(name = "-http-port", usage = "Sets the http port for the Cluster Controller")
- public int httpPort;
+ @Option(name = "-client-net-port", usage = "Sets the port to listen for connections from clients (default 1098)")
+ public int clientNetPort = 1098;
+
+ @Option(name = "-cluster-net-ip-address", usage = "Sets the IP Address to listen for connections from ", required = true)
+ public String clusterNetIpAddress;
+
+ @Option(name = "-cluster-net-port", usage = "Sets the port to listen for connections from node controllers (default 1099)")
+ public int clusterNetPort = 1099;
+
+ @Option(name = "-http-port", usage = "Sets the http port for the Cluster Controller (default: 19001)")
+ public int httpPort = 16001;
@Option(name = "-heartbeat-period", usage = "Sets the time duration between two heartbeats from each node controller in milliseconds (default: 10000)")
public int heartbeatPeriod = 10000;
@@ -37,9 +46,21 @@
@Option(name = "-default-max-job-attempts", usage = "Sets the default number of job attempts allowed if not specified in the job specification. (default: 5)")
public int defaultMaxJobAttempts = 5;
+ @Option(name = "-job-history-size", usage = "Limits the number of historical jobs remembered by the system to the specified value. (default: 10)")
+ public int jobHistorySize = 10;
+
+ @Option(name = "-cc-root", usage = "Sets the root folder used for file operations. (default: ClusterControllerService)")
+ public String ccRoot = "ClusterControllerService";
+
public void toCommandLine(List<String> cList) {
- cList.add("-port");
- cList.add(String.valueOf(port));
+ cList.add("-client-net-ip-address");
+ cList.add(clientNetIpAddress);
+ cList.add("-client-net-port");
+ cList.add(String.valueOf(clientNetPort));
+ cList.add("-cluster-net-ip-address");
+ cList.add(clusterNetIpAddress);
+ cList.add("-cluster-net-port");
+ cList.add(String.valueOf(clusterNetPort));
cList.add("-http-port");
cList.add(String.valueOf(httpPort));
cList.add("-heartbeat-period");
@@ -50,5 +71,9 @@
cList.add(String.valueOf(profileDumpPeriod));
cList.add("-default-max-job-attempts");
cList.add(String.valueOf(defaultMaxJobAttempts));
+ cList.add("-job-history-size");
+ cList.add(String.valueOf(jobHistorySize));
+ cList.add("-cc-root");
+ cList.add(ccRoot);
}
}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java
index 02ce0f8..574f552 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NCConfig.java
@@ -22,16 +22,19 @@
public class NCConfig implements Serializable {
private static final long serialVersionUID = 1L;
- @Option(name = "-cc-host", usage = "Cluster Controller host name")
+ @Option(name = "-cc-host", usage = "Cluster Controller host name", required = true)
public String ccHost;
@Option(name = "-cc-port", usage = "Cluster Controller port (default: 1099)")
public int ccPort = 1099;
- @Option(name = "-node-id", usage = "Logical name of node controller unique within the cluster")
+ @Option(name = "-cluster-net-ip-address", usage = "IP Address to bind cluster listener", required = true)
+ public String clusterNetIPAddress;
+
+ @Option(name = "-node-id", usage = "Logical name of node controller unique within the cluster", required = true)
public String nodeId;
- @Option(name = "-data-ip-address", usage = "IP Address to bind data listener")
+ @Option(name = "-data-ip-address", usage = "IP Address to bind data listener", required = true)
public String dataIPAddress;
@Option(name = "-frame-size", usage = "Frame Size to use for data communication (default: 32768)")
@@ -49,11 +52,16 @@
@Option(name = "-dcache-client-path", usage = "Sets the path to store the files retrieved from the DCache server (default /tmp/dcache-client)")
public String dcacheClientPath = "/tmp/dcache-client";
+ @Option(name = "-net-thread-count", usage = "Number of threads to use for Network I/O (default: 1)")
+ public int nNetThreads = 1;
+
public void toCommandLine(List<String> cList) {
cList.add("-cc-host");
cList.add(ccHost);
cList.add("-cc-port");
cList.add(String.valueOf(ccPort));
+ cList.add("-cluster-net-ip-address");
+ cList.add(clusterNetIPAddress);
cList.add("-node-id");
cList.add(nodeId);
cList.add("-data-ip-address");
@@ -70,5 +78,7 @@
}
cList.add("-dcache-client-path");
cList.add(dcacheClientPath);
+ cList.add("-net-thread-count");
+ cList.add(String.valueOf(nNetThreads));
}
}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java
index 9b10d27..91cfecf 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/controllers/NodeRegistration.java
@@ -15,14 +15,17 @@
package edu.uci.ics.hyracks.control.common.controllers;
import java.io.Serializable;
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
import edu.uci.ics.hyracks.api.comm.NetworkAddress;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema;
public final class NodeRegistration implements Serializable {
private static final long serialVersionUID = 1L;
- private final INodeController nc;
+ private final InetSocketAddress ncAddress;
private final String nodeId;
@@ -38,9 +41,29 @@
private final int nProcessors;
- public NodeRegistration(INodeController nc, String nodeId, NCConfig ncConfig, NetworkAddress dataPort,
- String osName, String arch, String osVersion, int nProcessors) {
- this.nc = nc;
+ private final String vmName;
+
+ private final String vmVersion;
+
+ private final String vmVendor;
+
+ private final String classpath;
+
+ private final String libraryPath;
+
+ private final String bootClasspath;
+
+ private final List<String> inputArguments;
+
+ private final Map<String, String> systemProperties;
+
+ private final HeartbeatSchema hbSchema;
+
+ public NodeRegistration(InetSocketAddress ncAddress, String nodeId, NCConfig ncConfig, NetworkAddress dataPort,
+ String osName, String arch, String osVersion, int nProcessors, String vmName, String vmVersion,
+ String vmVendor, String classpath, String libraryPath, String bootClasspath, List<String> inputArguments,
+ Map<String, String> systemProperties, HeartbeatSchema hbSchema) {
+ this.ncAddress = ncAddress;
this.nodeId = nodeId;
this.ncConfig = ncConfig;
this.dataPort = dataPort;
@@ -48,10 +71,19 @@
this.arch = arch;
this.osVersion = osVersion;
this.nProcessors = nProcessors;
+ this.vmName = vmName;
+ this.vmVersion = vmVersion;
+ this.vmVendor = vmVendor;
+ this.classpath = classpath;
+ this.libraryPath = libraryPath;
+ this.bootClasspath = bootClasspath;
+ this.inputArguments = inputArguments;
+ this.systemProperties = systemProperties;
+ this.hbSchema = hbSchema;
}
- public INodeController getNodeController() {
- return nc;
+ public InetSocketAddress getNodeControllerAddress() {
+ return ncAddress;
}
public String getNodeId() {
@@ -81,4 +113,40 @@
public int getNProcessors() {
return nProcessors;
}
+
+ public HeartbeatSchema getHeartbeatSchema() {
+ return hbSchema;
+ }
+
+ public String getVmName() {
+ return vmName;
+ }
+
+ public String getVmVersion() {
+ return vmVersion;
+ }
+
+ public String getVmVendor() {
+ return vmVendor;
+ }
+
+ public String getClasspath() {
+ return classpath;
+ }
+
+ public String getLibraryPath() {
+ return libraryPath;
+ }
+
+ public String getBootClasspath() {
+ return bootClasspath;
+ }
+
+ public List<String> getInputArguments() {
+ return inputArguments;
+ }
+
+ public Map<String, String> getSystemProperties() {
+ return systemProperties;
+ }
}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java
index 1816cd5..1dba3bc 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatData.java
@@ -31,4 +31,14 @@
public int peakThreadCount;
public long totalStartedThreadCount;
public double systemLoadAverage;
+ public long[] gcCollectionCounts;
+ public long[] gcCollectionTimes;
+ public long netPayloadBytesRead;
+ public long netPayloadBytesWritten;
+ public long netSignalingBytesRead;
+ public long netSignalingBytesWritten;
+ public long ipcMessagesSent;
+ public long ipcMessageBytesSent;
+ public long ipcMessagesReceived;
+ public long ipcMessageBytesReceived;
}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatSchema.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatSchema.java
new file mode 100644
index 0000000..e53e9a8
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/heartbeat/HeartbeatSchema.java
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.heartbeat;
+
+import java.io.Serializable;
+
+public class HeartbeatSchema implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final GarbageCollectorInfo[] gcInfos;
+
+ public HeartbeatSchema(GarbageCollectorInfo[] gcInfos) {
+ this.gcInfos = gcInfos;
+ }
+
+ public GarbageCollectorInfo[] getGarbageCollectorInfos() {
+ return gcInfos;
+ }
+
+ public static class GarbageCollectorInfo implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final String name;
+
+ public GarbageCollectorInfo(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/CCNCFunctions.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/CCNCFunctions.java
new file mode 100644
index 0000000..9ea4636
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/CCNCFunctions.java
@@ -0,0 +1,792 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.ipc;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
+import edu.uci.ics.hyracks.api.dataflow.ActivityId;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
+import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
+import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
+import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
+import edu.uci.ics.hyracks.control.common.job.PartitionState;
+import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
+import edu.uci.ics.hyracks.ipc.api.IPayloadSerializerDeserializer;
+import edu.uci.ics.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer;
+
+public class CCNCFunctions {
+ private static final int FID_CODE_SIZE = 1;
+
+ public enum FunctionId {
+ REGISTER_NODE,
+ UNREGISTER_NODE,
+ NOTIFY_JOBLET_CLEANUP,
+ NOTIFY_TASK_COMPLETE,
+ NOTIFY_TASK_FAILURE,
+ NODE_HEARTBEAT,
+ REPORT_PROFILE,
+ REGISTER_PARTITION_PROVIDER,
+ REGISTER_PARTITION_REQUEST,
+ APPLICATION_STATE_CHANGE_RESPONSE,
+
+ NODE_REGISTRATION_RESULT,
+ START_TASKS,
+ ABORT_TASKS,
+ CLEANUP_JOBLET,
+ CREATE_APPLICATION,
+ DESTROY_APPLICATION,
+ REPORT_PARTITION_AVAILABILITY,
+
+ OTHER
+ }
+
+ public static abstract class Function implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ public abstract FunctionId getFunctionId();
+ }
+
+ public static class RegisterNodeFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final NodeRegistration reg;
+
+ public RegisterNodeFunction(NodeRegistration reg) {
+ this.reg = reg;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.REGISTER_NODE;
+ }
+
+ public NodeRegistration getNodeRegistration() {
+ return reg;
+ }
+ }
+
+ public static class UnregisterNodeFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String nodeId;
+
+ public UnregisterNodeFunction(String nodeId) {
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.UNREGISTER_NODE;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+ }
+
+ public static class NotifyTaskCompleteFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final JobId jobId;
+ private final TaskAttemptId taskId;
+ private final String nodeId;
+ private final TaskProfile statistics;
+
+ public NotifyTaskCompleteFunction(JobId jobId, TaskAttemptId taskId, String nodeId, TaskProfile statistics) {
+ this.jobId = jobId;
+ this.taskId = taskId;
+ this.nodeId = nodeId;
+ this.statistics = statistics;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.NOTIFY_TASK_COMPLETE;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+
+ public TaskAttemptId getTaskId() {
+ return taskId;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public TaskProfile getStatistics() {
+ return statistics;
+ }
+ }
+
+ public static class NotifyTaskFailureFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final JobId jobId;
+ private final TaskAttemptId taskId;
+ private final String nodeId;
+ private final String details;
+
+ public NotifyTaskFailureFunction(JobId jobId, TaskAttemptId taskId, String nodeId, String details) {
+ this.jobId = jobId;
+ this.taskId = taskId;
+ this.nodeId = nodeId;
+ this.details = details;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.NOTIFY_TASK_FAILURE;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+
+ public TaskAttemptId getTaskId() {
+ return taskId;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public String getDetails() {
+ return details;
+ }
+ }
+
+ public static class NotifyJobletCleanupFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final JobId jobId;
+ private final String nodeId;
+
+ public NotifyJobletCleanupFunction(JobId jobId, String nodeId) {
+ this.jobId = jobId;
+ this.nodeId = nodeId;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.NOTIFY_JOBLET_CLEANUP;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+ }
+
+ public static class NodeHeartbeatFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String nodeId;
+ private final HeartbeatData hbData;
+
+ public NodeHeartbeatFunction(String nodeId, HeartbeatData hbData) {
+ this.nodeId = nodeId;
+ this.hbData = hbData;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.NODE_HEARTBEAT;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public HeartbeatData getHeartbeatData() {
+ return hbData;
+ }
+ }
+
+ public static class ReportProfileFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String nodeId;
+ private final List<JobProfile> profiles;
+
+ public ReportProfileFunction(String nodeId, List<JobProfile> profiles) {
+ this.nodeId = nodeId;
+ this.profiles = profiles;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.REPORT_PROFILE;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public List<JobProfile> getProfiles() {
+ return profiles;
+ }
+ }
+
+ public static class RegisterPartitionProviderFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final PartitionDescriptor partitionDescriptor;
+
+ public RegisterPartitionProviderFunction(PartitionDescriptor partitionDescriptor) {
+ this.partitionDescriptor = partitionDescriptor;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.REGISTER_PARTITION_PROVIDER;
+ }
+
+ public PartitionDescriptor getPartitionDescriptor() {
+ return partitionDescriptor;
+ }
+
+ public static Object deserialize(ByteBuffer buffer, int length) throws Exception {
+ ByteArrayInputStream bais = new ByteArrayInputStream(buffer.array(), buffer.position(), length);
+ DataInputStream dis = new DataInputStream(bais);
+
+ // Read PartitionId
+ PartitionId pid = readPartitionId(dis);
+
+ // Read nodeId
+ String nodeId = dis.readUTF();
+
+ // Read TaskAttemptId
+ TaskAttemptId taId = readTaskAttemptId(dis);
+
+ // Read reusable flag
+ boolean reusable = dis.readBoolean();
+
+ // Read Partition State
+ PartitionState state = readPartitionState(dis);
+
+ PartitionDescriptor pd = new PartitionDescriptor(pid, nodeId, taId, reusable);
+ pd.setState(state);
+ return new RegisterPartitionProviderFunction(pd);
+ }
+
+ public static void serialize(OutputStream out, Object object) throws Exception {
+ RegisterPartitionProviderFunction fn = (RegisterPartitionProviderFunction) object;
+
+ DataOutputStream dos = new DataOutputStream(out);
+
+ PartitionDescriptor pd = fn.getPartitionDescriptor();
+
+ // Write PartitionId
+ writePartitionId(dos, pd.getPartitionId());
+
+ // Write nodeId
+ dos.writeUTF(pd.getNodeId());
+
+ // Write TaskAttemptId
+ writeTaskAttemptId(dos, pd.getProducingTaskAttemptId());
+
+ // Write reusable flag
+ dos.writeBoolean(pd.isReusable());
+
+ // Write Partition State
+ writePartitionState(dos, pd.getState());
+ }
+ }
+
+ public static class RegisterPartitionRequestFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final PartitionRequest partitionRequest;
+
+ public RegisterPartitionRequestFunction(PartitionRequest partitionRequest) {
+ this.partitionRequest = partitionRequest;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.REGISTER_PARTITION_REQUEST;
+ }
+
+ public PartitionRequest getPartitionRequest() {
+ return partitionRequest;
+ }
+
+ public static Object deserialize(ByteBuffer buffer, int length) throws Exception {
+ ByteArrayInputStream bais = new ByteArrayInputStream(buffer.array(), buffer.position(), length);
+ DataInputStream dis = new DataInputStream(bais);
+
+ // Read PartitionId
+ PartitionId pid = readPartitionId(dis);
+
+ // Read nodeId
+ String nodeId = dis.readUTF();
+
+ // Read TaskAttemptId
+ TaskAttemptId taId = readTaskAttemptId(dis);
+
+ // Read Partition State
+ PartitionState state = readPartitionState(dis);
+
+ PartitionRequest pr = new PartitionRequest(pid, nodeId, taId, state);
+ return new RegisterPartitionRequestFunction(pr);
+ }
+
+ public static void serialize(OutputStream out, Object object) throws Exception {
+ RegisterPartitionRequestFunction fn = (RegisterPartitionRequestFunction) object;
+
+ DataOutputStream dos = new DataOutputStream(out);
+
+ PartitionRequest pr = fn.getPartitionRequest();
+
+ // Write PartitionId
+ writePartitionId(dos, pr.getPartitionId());
+
+ // Write nodeId
+ dos.writeUTF(pr.getNodeId());
+
+ // Write TaskAttemptId
+ writeTaskAttemptId(dos, pr.getRequestingTaskAttemptId());
+
+ // Write Partition State
+ writePartitionState(dos, pr.getMinimumState());
+ }
+ }
+
+ public static class ApplicationStateChangeResponseFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String nodeId;
+ private final String appName;
+ private final ApplicationStatus status;
+
+ public ApplicationStateChangeResponseFunction(String nodeId, String appName, ApplicationStatus status) {
+ this.nodeId = nodeId;
+ this.appName = appName;
+ this.status = status;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.APPLICATION_STATE_CHANGE_RESPONSE;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ public String getApplicationName() {
+ return appName;
+ }
+
+ public ApplicationStatus getStatus() {
+ return status;
+ }
+ }
+
+ public static class NodeRegistrationResult extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final NodeParameters params;
+
+ private final Exception exception;
+
+ public NodeRegistrationResult(NodeParameters params, Exception exception) {
+ this.params = params;
+ this.exception = exception;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.NODE_REGISTRATION_RESULT;
+ }
+
+ public NodeParameters getNodeParameters() {
+ return params;
+ }
+
+ public Exception getException() {
+ return exception;
+ }
+ }
+
+ public static class StartTasksFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String appName;
+ private final JobId jobId;
+ private final byte[] planBytes;
+ private final List<TaskAttemptDescriptor> taskDescriptors;
+ private final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies;
+
+ public StartTasksFunction(String appName, JobId jobId, byte[] planBytes,
+ List<TaskAttemptDescriptor> taskDescriptors,
+ Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies) {
+ this.appName = appName;
+ this.jobId = jobId;
+ this.planBytes = planBytes;
+ this.taskDescriptors = taskDescriptors;
+ this.connectorPolicies = connectorPolicies;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.START_TASKS;
+ }
+
+ public String getAppName() {
+ return appName;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+
+ public byte[] getPlanBytes() {
+ return planBytes;
+ }
+
+ public List<TaskAttemptDescriptor> getTaskDescriptors() {
+ return taskDescriptors;
+ }
+
+ public Map<ConnectorDescriptorId, IConnectorPolicy> getConnectorPolicies() {
+ return connectorPolicies;
+ }
+ }
+
+ public static class AbortTasksFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final JobId jobId;
+ private final List<TaskAttemptId> tasks;
+
+ public AbortTasksFunction(JobId jobId, List<TaskAttemptId> tasks) {
+ this.jobId = jobId;
+ this.tasks = tasks;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.ABORT_TASKS;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+
+ public List<TaskAttemptId> getTasks() {
+ return tasks;
+ }
+ }
+
+ public static class CleanupJobletFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final JobId jobId;
+ private final JobStatus status;
+
+ public CleanupJobletFunction(JobId jobId, JobStatus status) {
+ this.jobId = jobId;
+ this.status = status;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.CLEANUP_JOBLET;
+ }
+
+ public JobId getJobId() {
+ return jobId;
+ }
+
+ public JobStatus getStatus() {
+ return status;
+ }
+ }
+
+ public static class CreateApplicationFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String appName;
+ private final boolean deployHar;
+ private final byte[] serializedDistributedState;
+
+ public CreateApplicationFunction(String appName, boolean deployHar, byte[] serializedDistributedState) {
+ this.appName = appName;
+ this.deployHar = deployHar;
+ this.serializedDistributedState = serializedDistributedState;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.CREATE_APPLICATION;
+ }
+
+ public String getAppName() {
+ return appName;
+ }
+
+ public boolean isDeployHar() {
+ return deployHar;
+ }
+
+ public byte[] getSerializedDistributedState() {
+ return serializedDistributedState;
+ }
+ }
+
+ public static class DestroyApplicationFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final String appName;
+
+ public DestroyApplicationFunction(String appName) {
+ this.appName = appName;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.DESTROY_APPLICATION;
+ }
+
+ public String getAppName() {
+ return appName;
+ }
+ }
+
+ public static class ReportPartitionAvailabilityFunction extends Function {
+ private static final long serialVersionUID = 1L;
+
+ private final PartitionId pid;
+ private final NetworkAddress networkAddress;
+
+ public ReportPartitionAvailabilityFunction(PartitionId pid, NetworkAddress networkAddress) {
+ this.pid = pid;
+ this.networkAddress = networkAddress;
+ }
+
+ @Override
+ public FunctionId getFunctionId() {
+ return FunctionId.REPORT_PARTITION_AVAILABILITY;
+ }
+
+ public PartitionId getPartitionId() {
+ return pid;
+ }
+
+ public NetworkAddress getNetworkAddress() {
+ return networkAddress;
+ }
+
+ public static Object deserialize(ByteBuffer buffer, int length) throws Exception {
+ ByteArrayInputStream bais = new ByteArrayInputStream(buffer.array(), buffer.position(), length);
+ DataInputStream dis = new DataInputStream(bais);
+
+ // Read PartitionId
+ PartitionId pid = readPartitionId(dis);
+
+ // Read NetworkAddress
+ NetworkAddress networkAddress = readNetworkAddress(dis);
+
+ return new ReportPartitionAvailabilityFunction(pid, networkAddress);
+ }
+
+ public static void serialize(OutputStream out, Object object) throws Exception {
+ ReportPartitionAvailabilityFunction fn = (ReportPartitionAvailabilityFunction) object;
+
+ DataOutputStream dos = new DataOutputStream(out);
+
+ // Write PartitionId
+ writePartitionId(dos, fn.getPartitionId());
+
+ // Write NetworkAddress
+ writeNetworkAddress(dos, fn.getNetworkAddress());
+ }
+ }
+
+ public static class SerializerDeserializer implements IPayloadSerializerDeserializer {
+ private final JavaSerializationBasedPayloadSerializerDeserializer javaSerde;
+
+ public SerializerDeserializer() {
+ javaSerde = new JavaSerializationBasedPayloadSerializerDeserializer();
+ }
+
+ @Override
+ public Object deserializeObject(ByteBuffer buffer, int length) throws Exception {
+ if (length < FID_CODE_SIZE) {
+ throw new IllegalStateException("Message size too small: " + length);
+ }
+ byte fid = buffer.get();
+ return deserialize(fid, buffer, length - FID_CODE_SIZE);
+ }
+
+ @Override
+ public Exception deserializeException(ByteBuffer buffer, int length) throws Exception {
+ if (length < FID_CODE_SIZE) {
+ throw new IllegalStateException("Message size too small: " + length);
+ }
+ byte fid = buffer.get();
+ if (fid != FunctionId.OTHER.ordinal()) {
+ throw new IllegalStateException("Expected FID for OTHER, found: " + fid);
+ }
+ return (Exception) deserialize(fid, buffer, length - FID_CODE_SIZE);
+ }
+
+ @Override
+ public byte[] serializeObject(Object object) throws Exception {
+ if (object instanceof Function) {
+ Function fn = (Function) object;
+ return serialize(object, (byte) fn.getFunctionId().ordinal());
+ } else {
+ return serialize(object, (byte) FunctionId.OTHER.ordinal());
+ }
+ }
+
+ @Override
+ public byte[] serializeException(Exception object) throws Exception {
+ return serialize(object, (byte) FunctionId.OTHER.ordinal());
+ }
+
+ private byte[] serialize(Object object, byte fid) throws Exception {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ baos.write(fid);
+ serialize(baos, object, fid);
+ JavaSerializationBasedPayloadSerializerDeserializer.serialize(baos, object);
+ baos.close();
+ return baos.toByteArray();
+ }
+
+ private void serialize(OutputStream out, Object object, byte fid) throws Exception {
+ switch (FunctionId.values()[fid]) {
+ case REGISTER_PARTITION_PROVIDER:
+ RegisterPartitionProviderFunction.serialize(out, object);
+ return;
+
+ case REGISTER_PARTITION_REQUEST:
+ RegisterPartitionRequestFunction.serialize(out, object);
+ return;
+
+ case REPORT_PARTITION_AVAILABILITY:
+ ReportPartitionAvailabilityFunction.serialize(out, object);
+ return;
+ }
+ JavaSerializationBasedPayloadSerializerDeserializer.serialize(out, object);
+ }
+
+ private Object deserialize(byte fid, ByteBuffer buffer, int length) throws Exception {
+ switch (FunctionId.values()[fid]) {
+ case REGISTER_PARTITION_PROVIDER:
+ return RegisterPartitionProviderFunction.deserialize(buffer, length);
+
+ case REGISTER_PARTITION_REQUEST:
+ return RegisterPartitionRequestFunction.deserialize(buffer, length);
+
+ case REPORT_PARTITION_AVAILABILITY:
+ return ReportPartitionAvailabilityFunction.deserialize(buffer, length);
+ }
+
+ return javaSerde.deserializeObject(buffer, length);
+ }
+ }
+
+ private static PartitionId readPartitionId(DataInputStream dis) throws IOException {
+ long jobId = dis.readLong();
+ int cdid = dis.readInt();
+ int senderIndex = dis.readInt();
+ int receiverIndex = dis.readInt();
+ PartitionId pid = new PartitionId(new JobId(jobId), new ConnectorDescriptorId(cdid), senderIndex, receiverIndex);
+ return pid;
+ }
+
+ private static void writePartitionId(DataOutputStream dos, PartitionId pid) throws IOException {
+ dos.writeLong(pid.getJobId().getId());
+ dos.writeInt(pid.getConnectorDescriptorId().getId());
+ dos.writeInt(pid.getSenderIndex());
+ dos.writeInt(pid.getReceiverIndex());
+ }
+
+ private static TaskAttemptId readTaskAttemptId(DataInputStream dis) throws IOException {
+ int odid = dis.readInt();
+ int aid = dis.readInt();
+ int partition = dis.readInt();
+ int attempt = dis.readInt();
+ TaskAttemptId taId = new TaskAttemptId(new TaskId(new ActivityId(new OperatorDescriptorId(odid), aid),
+ partition), attempt);
+ return taId;
+ }
+
+ private static void writeTaskAttemptId(DataOutputStream dos, TaskAttemptId taId) throws IOException {
+ TaskId tid = taId.getTaskId();
+ ActivityId aid = tid.getActivityId();
+ OperatorDescriptorId odId = aid.getOperatorDescriptorId();
+ dos.writeInt(odId.getId());
+ dos.writeInt(aid.getLocalId());
+ dos.writeInt(tid.getPartition());
+ dos.writeInt(taId.getAttempt());
+ }
+
+ private static PartitionState readPartitionState(DataInputStream dis) throws IOException {
+ PartitionState state = PartitionState.values()[dis.readInt()];
+ return state;
+ }
+
+ private static void writePartitionState(DataOutputStream dos, PartitionState state) throws IOException {
+ dos.writeInt(state.ordinal());
+ }
+
+ private static NetworkAddress readNetworkAddress(DataInputStream dis) throws IOException {
+ int bLen = dis.readInt();
+ byte[] ipAddress = new byte[bLen];
+ dis.read(ipAddress);
+ int port = dis.readInt();
+ NetworkAddress networkAddress = new NetworkAddress(ipAddress, port);
+ return networkAddress;
+ }
+
+ private static void writeNetworkAddress(DataOutputStream dos, NetworkAddress networkAddress) throws IOException {
+ byte[] ipAddress = networkAddress.getIpAddress();
+ dos.writeInt(ipAddress.length);
+ dos.write(ipAddress);
+ dos.writeInt(networkAddress.getPort());
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
new file mode 100644
index 0000000..a0dabdd
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/ClusterControllerRemoteProxy.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.ipc;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.base.IClusterController;
+import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
+import edu.uci.ics.hyracks.control.common.job.PartitionDescriptor;
+import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+
+public class ClusterControllerRemoteProxy implements IClusterController {
+ private final IIPCHandle ipcHandle;
+
+ public ClusterControllerRemoteProxy(IIPCHandle ipcHandle) {
+ this.ipcHandle = ipcHandle;
+ }
+
+ @Override
+ public void registerNode(NodeRegistration reg) throws Exception {
+ CCNCFunctions.RegisterNodeFunction fn = new CCNCFunctions.RegisterNodeFunction(reg);
+ ipcHandle.send(-1, fn, null);
+ }
+
+ @Override
+ public void unregisterNode(String nodeId) throws Exception {
+ CCNCFunctions.UnregisterNodeFunction fn = new CCNCFunctions.UnregisterNodeFunction(
+ nodeId);
+ ipcHandle.send(-1, fn, null);
+ }
+
+ @Override
+ public void notifyTaskComplete(JobId jobId, TaskAttemptId taskId, String nodeId, TaskProfile statistics)
+ throws Exception {
+ CCNCFunctions.NotifyTaskCompleteFunction fn = new CCNCFunctions.NotifyTaskCompleteFunction(
+ jobId, taskId, nodeId, statistics);
+ ipcHandle.send(-1, fn, null);
+ }
+
+ @Override
+ public void notifyTaskFailure(JobId jobId, TaskAttemptId taskId, String nodeId, String details) throws Exception {
+ CCNCFunctions.NotifyTaskFailureFunction fn = new CCNCFunctions.NotifyTaskFailureFunction(
+ jobId, taskId, nodeId, details);
+ ipcHandle.send(-1, fn, null);
+ }
+
+ @Override
+ public void notifyJobletCleanup(JobId jobId, String nodeId) throws Exception {
+ CCNCFunctions.NotifyJobletCleanupFunction fn = new CCNCFunctions.NotifyJobletCleanupFunction(
+ jobId, nodeId);
+ ipcHandle.send(-1, fn, null);
+ }
+
+ @Override
+ public void nodeHeartbeat(String id, HeartbeatData hbData) throws Exception {
+ CCNCFunctions.NodeHeartbeatFunction fn = new CCNCFunctions.NodeHeartbeatFunction(id,
+ hbData);
+ ipcHandle.send(-1, fn, null);
+ }
+
+ @Override
+ public void reportProfile(String id, List<JobProfile> profiles) throws Exception {
+ CCNCFunctions.ReportProfileFunction fn = new CCNCFunctions.ReportProfileFunction(id,
+ profiles);
+ ipcHandle.send(-1, fn, null);
+ }
+
+ @Override
+ public void registerPartitionProvider(PartitionDescriptor partitionDescriptor) throws Exception {
+ CCNCFunctions.RegisterPartitionProviderFunction fn = new CCNCFunctions.RegisterPartitionProviderFunction(
+ partitionDescriptor);
+ ipcHandle.send(-1, fn, null);
+ }
+
+ @Override
+ public void registerPartitionRequest(PartitionRequest partitionRequest) throws Exception {
+ CCNCFunctions.RegisterPartitionRequestFunction fn = new CCNCFunctions.RegisterPartitionRequestFunction(
+ partitionRequest);
+ ipcHandle.send(-1, fn, null);
+ }
+
+ @Override
+ public void notifyApplicationStateChange(String nodeId, String appName, ApplicationStatus status) throws Exception {
+ CCNCFunctions.ApplicationStateChangeResponseFunction fn = new CCNCFunctions.ApplicationStateChangeResponseFunction(
+ nodeId, appName, status);
+ ipcHandle.send(-1, fn, null);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
new file mode 100644
index 0000000..8176cb7
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/ipc/NodeControllerRemoteProxy.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.ipc;
+
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.base.INodeController;
+import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+
+public class NodeControllerRemoteProxy implements INodeController {
+ private final IIPCHandle ipcHandle;
+
+ public NodeControllerRemoteProxy(IIPCHandle ipcHandle) {
+ this.ipcHandle = ipcHandle;
+ }
+
+ @Override
+ public void startTasks(String appName, JobId jobId, byte[] planBytes, List<TaskAttemptDescriptor> taskDescriptors,
+ Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies) throws Exception {
+ CCNCFunctions.StartTasksFunction stf = new CCNCFunctions.StartTasksFunction(appName, jobId, planBytes,
+ taskDescriptors, connectorPolicies);
+ ipcHandle.send(-1, stf, null);
+ }
+
+ @Override
+ public void abortTasks(JobId jobId, List<TaskAttemptId> tasks) throws Exception {
+ CCNCFunctions.AbortTasksFunction atf = new CCNCFunctions.AbortTasksFunction(jobId, tasks);
+ ipcHandle.send(-1, atf, null);
+ }
+
+ @Override
+ public void cleanUpJoblet(JobId jobId, JobStatus status) throws Exception {
+ CCNCFunctions.CleanupJobletFunction cjf = new CCNCFunctions.CleanupJobletFunction(jobId, status);
+ ipcHandle.send(-1, cjf, null);
+ }
+
+ @Override
+ public void createApplication(String appName, boolean deployHar, byte[] serializedDistributedState)
+ throws Exception {
+ CCNCFunctions.CreateApplicationFunction caf = new CCNCFunctions.CreateApplicationFunction(appName, deployHar,
+ serializedDistributedState);
+ ipcHandle.send(-1, caf, null);
+ }
+
+ @Override
+ public void destroyApplication(String appName) throws Exception {
+ CCNCFunctions.DestroyApplicationFunction daf = new CCNCFunctions.DestroyApplicationFunction(appName);
+ ipcHandle.send(-1, daf, null);
+ }
+
+ @Override
+ public void reportPartitionAvailability(PartitionId pid, NetworkAddress networkAddress) throws Exception {
+ CCNCFunctions.ReportPartitionAvailabilityFunction rpaf = new CCNCFunctions.ReportPartitionAvailabilityFunction(
+ pid, networkAddress);
+ ipcHandle.send(-1, rpaf, null);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/TaskAttemptDescriptor.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/TaskAttemptDescriptor.java
index 15ba75c..f6d1f78 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/TaskAttemptDescriptor.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/TaskAttemptDescriptor.java
@@ -15,7 +15,9 @@
package edu.uci.ics.hyracks.control.common.job;
import java.io.Serializable;
+import java.util.Arrays;
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
public class TaskAttemptDescriptor implements Serializable {
@@ -29,6 +31,8 @@
private final int[] nOutputPartitions;
+ private NetworkAddress[][] inputPartitionLocations;
+
public TaskAttemptDescriptor(TaskAttemptId taId, int nPartitions, int[] nInputPartitions, int[] nOutputPartitions) {
this.taId = taId;
this.nPartitions = nPartitions;
@@ -51,4 +55,19 @@
public int[] getOutputPartitionCounts() {
return nOutputPartitions;
}
+
+ public void setInputPartitionLocations(NetworkAddress[][] inputPartitionLocations) {
+ this.inputPartitionLocations = inputPartitionLocations;
+ }
+
+ public NetworkAddress[][] getInputPartitionLocations() {
+ return inputPartitionLocations;
+ }
+
+ @Override
+ public String toString() {
+ return "TaskAttemptDescriptor[taId = " + taId + ", nPartitions = " + nPartitions + ", nInputPartitions = "
+ + Arrays.toString(nInputPartitions) + ", nOutputPartitions = " + Arrays.toString(nOutputPartitions)
+ + "]";
+ }
}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/counters/MultiResolutionEventProfiler.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/counters/MultiResolutionEventProfiler.java
new file mode 100644
index 0000000..f8fba7a
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/counters/MultiResolutionEventProfiler.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.job.profiling.counters;
+
+import java.io.Serializable;
+
+public class MultiResolutionEventProfiler implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final int[] times;
+
+ private long offset;
+
+ private int ptr;
+
+ private int resolution;
+
+ private int eventCounter;
+
+ public MultiResolutionEventProfiler(int nSamples) {
+ times = new int[nSamples];
+ offset = -1;
+ ptr = 0;
+ resolution = 1;
+ eventCounter = 0;
+ }
+
+ public void reportEvent() {
+ ++eventCounter;
+ if (eventCounter % resolution != 0) {
+ return;
+ }
+ if (ptr >= times.length) {
+ compact();
+ return;
+ }
+ eventCounter = 0;
+ long time = System.currentTimeMillis();
+ if (offset < 0) {
+ offset = time;
+ }
+ int value = (int) (time - offset);
+ times[ptr++] = value;
+ }
+
+ private void compact() {
+ for (int i = 1; i < ptr / 2; ++i) {
+ times[i] = times[i * 2];
+ }
+ resolution <<= 1;
+ ptr >>= 1;
+ }
+
+ public int getResolution() {
+ return resolution;
+ }
+
+ public int getCount() {
+ return ptr;
+ }
+
+ public int[] getSamples() {
+ return times;
+ }
+
+ public long getOffset() {
+ return offset;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/AbstractProfile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/AbstractProfile.java
index 74d6000..b4e3619 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/AbstractProfile.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/AbstractProfile.java
@@ -18,6 +18,7 @@
import java.util.HashMap;
import java.util.Map;
+import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -37,12 +38,14 @@
public abstract JSONObject toJSON() throws JSONException;
protected void populateCounters(JSONObject jo) throws JSONException {
+ JSONArray countersObj = new JSONArray();
for (Map.Entry<String, Long> e : counters.entrySet()) {
JSONObject jpe = new JSONObject();
jpe.put("name", e.getKey());
jpe.put("value", e.getValue());
- jo.accumulate("counters", jpe);
+ countersObj.put(jpe);
}
+ jo.put("counters", countersObj);
}
protected void merge(AbstractProfile profile) {
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobProfile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobProfile.java
index 5457ac5..46a964a 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobProfile.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobProfile.java
@@ -3,6 +3,7 @@
import java.util.HashMap;
import java.util.Map;
+import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -32,12 +33,13 @@
public JSONObject toJSON() throws JSONException {
JSONObject json = new JSONObject();
- json.put("type", "job-profile");
json.put("job-id", jobId.toString());
populateCounters(json);
+ JSONArray jobletsArray = new JSONArray();
for (JobletProfile p : jobletProfiles.values()) {
- json.accumulate("joblets", p.toJSON());
+ jobletsArray.put(p.toJSON());
}
+ json.put("joblets", jobletsArray);
return json;
}
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobletProfile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobletProfile.java
index 5069fc3..0c60006 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobletProfile.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/JobletProfile.java
@@ -17,6 +17,7 @@
import java.util.HashMap;
import java.util.Map;
+import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@@ -46,12 +47,13 @@
public JSONObject toJSON() throws JSONException {
JSONObject json = new JSONObject();
- json.put("type", "joblet-profile");
json.put("node-id", nodeId.toString());
populateCounters(json);
+ JSONArray tasks = new JSONArray();
for (TaskProfile p : taskProfiles.values()) {
- json.accumulate("tasks", p.toJSON());
+ tasks.put(p.toJSON());
}
+ json.put("tasks", tasks);
return json;
}
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/PartitionProfile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/PartitionProfile.java
new file mode 100644
index 0000000..ef61796
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/PartitionProfile.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.job.profiling.om;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.job.profiling.counters.MultiResolutionEventProfiler;
+
+public class PartitionProfile implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private final PartitionId pid;
+
+ private final long openTime;
+
+ private final long closeTime;
+
+ private final MultiResolutionEventProfiler mrep;
+
+ public PartitionProfile(PartitionId pid, long openTime, long closeTime, MultiResolutionEventProfiler mrep) {
+ this.pid = pid;
+ this.openTime = openTime;
+ this.closeTime = closeTime;
+ this.mrep = mrep;
+ }
+
+ public PartitionId getPartitionId() {
+ return pid;
+ }
+
+ public long getOpenTime() {
+ return openTime;
+ }
+
+ public long getCloseTime() {
+ return closeTime;
+ }
+
+ public MultiResolutionEventProfiler getSamples() {
+ return mrep;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java
index 3ab1090..2116f61 100644
--- a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/job/profiling/om/TaskProfile.java
@@ -14,32 +14,71 @@
*/
package edu.uci.ics.hyracks.control.common.job.profiling.om;
+import java.util.Map;
+
+import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.job.profiling.counters.MultiResolutionEventProfiler;
public class TaskProfile extends AbstractProfile {
private static final long serialVersionUID = 1L;
private final TaskAttemptId taskAttemptId;
- public TaskProfile(TaskAttemptId taskAttemptId) {
+ private final Map<PartitionId, PartitionProfile> partitionSendProfile;
+
+ public TaskProfile(TaskAttemptId taskAttemptId, Map<PartitionId, PartitionProfile> partitionSendProfile) {
this.taskAttemptId = taskAttemptId;
+ this.partitionSendProfile = partitionSendProfile;
}
public TaskAttemptId getTaskId() {
return taskAttemptId;
}
+ public Map<PartitionId, PartitionProfile> getPartitionSendProfile() {
+ return partitionSendProfile;
+ }
+
@Override
public JSONObject toJSON() throws JSONException {
JSONObject json = new JSONObject();
- json.put("type", "task-profile");
json.put("activity-id", taskAttemptId.getTaskId().getActivityId().toString());
json.put("partition", taskAttemptId.getTaskId().getPartition());
json.put("attempt", taskAttemptId.getAttempt());
+ if (partitionSendProfile != null) {
+ JSONArray pspArray = new JSONArray();
+ for (PartitionProfile pp : partitionSendProfile.values()) {
+ JSONObject ppObj = new JSONObject();
+ PartitionId pid = pp.getPartitionId();
+ JSONObject pidObj = new JSONObject();
+ pidObj.put("job-id", pid.getJobId());
+ pidObj.put("connector-id", pid.getConnectorDescriptorId());
+ pidObj.put("sender-index", pid.getSenderIndex());
+ pidObj.put("receiver-index", pid.getReceiverIndex());
+ ppObj.put("partition-id", pidObj);
+ ppObj.put("open-time", pp.getOpenTime());
+ ppObj.put("close-time", pp.getCloseTime());
+ MultiResolutionEventProfiler samples = pp.getSamples();
+ ppObj.put("offset", samples.getOffset());
+ int resolution = samples.getResolution();
+ int sampleCount = samples.getCount();
+ JSONArray ftA = new JSONArray();
+ int[] ft = samples.getSamples();
+ for (int i = 0; i < sampleCount; ++i) {
+ ftA.put(ft[i]);
+ }
+ ppObj.put("frame-times", ftA);
+ ppObj.put("resolution", resolution);
+ pspArray.put(ppObj);
+ }
+ json.put("partition-send-profile", pspArray);
+ }
populateCounters(json);
return json;
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/logs/LogFile.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/logs/LogFile.java
new file mode 100644
index 0000000..1521aa0
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/logs/LogFile.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.logs;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintWriter;
+
+import org.json.JSONObject;
+
+public class LogFile {
+ private final File root;
+
+ private PrintWriter out;
+
+ public LogFile(File root) {
+ this.root = root;
+ }
+
+ public void open() throws Exception {
+ root.mkdirs();
+ out = new PrintWriter(new FileOutputStream(new File(root, String.valueOf(System.currentTimeMillis()) + ".log"),
+ true));
+ }
+
+ public void log(JSONObject object) throws Exception {
+ out.println(object.toString(1));
+ out.flush();
+ }
+
+ public void close() {
+ out.flush();
+ out.close();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/AbstractWork.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/AbstractWork.java
new file mode 100644
index 0000000..4ead100
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/AbstractWork.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.work;
+
+import java.util.logging.Level;
+
+public abstract class AbstractWork implements Runnable {
+ public Level logLevel() {
+ return Level.INFO;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/FutureValue.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/FutureValue.java
new file mode 100644
index 0000000..7eb4ff6
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/FutureValue.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.work;
+
+public class FutureValue<T> implements IResultCallback<T> {
+ private boolean done;
+
+ private T value;
+
+ private Exception e;
+
+ public FutureValue() {
+ done = false;
+ value = null;
+ e = null;
+ }
+
+ @Override
+ public synchronized void setValue(T value) {
+ done = true;
+ this.value = value;
+ e = null;
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void setException(Exception e) {
+ done = true;
+ this.e = e;
+ value = null;
+ notifyAll();
+ }
+
+ public synchronized void reset() {
+ done = false;
+ value = null;
+ e = null;
+ notifyAll();
+ }
+
+ public synchronized T get() throws Exception {
+ while (!done) {
+ wait();
+ }
+ if (e != null) {
+ throw e;
+ }
+ return value;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IPCResponder.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IPCResponder.java
new file mode 100644
index 0000000..dcea864
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IPCResponder.java
@@ -0,0 +1,33 @@
+package edu.uci.ics.hyracks.control.common.work;
+
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+
+public class IPCResponder<T> implements IResultCallback<T> {
+ private final IIPCHandle handle;
+
+ private final long rmid;
+
+ public IPCResponder(IIPCHandle handle, long rmid) {
+ this.handle = handle;
+ this.rmid = rmid;
+ }
+
+ @Override
+ public void setValue(T result) {
+ try {
+ handle.send(rmid, result, null);
+ } catch (IPCException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void setException(Exception e) {
+ try {
+ handle.send(rmid, null, e);
+ } catch (IPCException e1) {
+ e1.printStackTrace();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IResultCallback.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IResultCallback.java
new file mode 100644
index 0000000..80c3d76
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/IResultCallback.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.work;
+
+public interface IResultCallback<T> {
+ public void setValue(T result);
+
+ public void setException(Exception e);
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/SynchronizableWork.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/SynchronizableWork.java
new file mode 100644
index 0000000..e94086f
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/SynchronizableWork.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.work;
+
+public abstract class SynchronizableWork extends AbstractWork {
+ private boolean done;
+
+ private Exception e;
+
+ protected abstract void doRun() throws Exception;
+
+ public void init() {
+ done = false;
+ e = null;
+ }
+
+ @Override
+ public final void run() {
+ try {
+ doRun();
+ } catch (Exception e) {
+ this.e = e;
+ } finally {
+ synchronized (this) {
+ done = true;
+ notifyAll();
+ }
+ }
+ }
+
+ public final synchronized void sync() throws Exception {
+ while (!done) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ throw e;
+ }
+ }
+ if (e != null) {
+ throw e;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java
new file mode 100644
index 0000000..36f0c49
--- /dev/null
+++ b/hyracks-control-common/src/main/java/edu/uci/ics/hyracks/control/common/work/WorkQueue.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.common.work;
+
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+
+public class WorkQueue {
+ private static final Logger LOGGER = Logger.getLogger(WorkQueue.class.getName());
+
+ private final LinkedBlockingQueue<AbstractWork> queue;
+ private final WorkerThread thread;
+ private final Semaphore stopSemaphore;
+ private boolean stopped;
+ private final AtomicInteger enqueueCount;
+ private final AtomicInteger dequeueCount;
+
+ public WorkQueue() {
+ queue = new LinkedBlockingQueue<AbstractWork>();
+ thread = new WorkerThread();
+ stopSemaphore = new Semaphore(1);
+ enqueueCount = new AtomicInteger();
+ dequeueCount = new AtomicInteger();
+ }
+
+ public void start() throws HyracksException {
+ stopped = false;
+ enqueueCount.set(0);
+ dequeueCount.set(0);
+ try {
+ stopSemaphore.acquire();
+ } catch (InterruptedException e) {
+ throw new HyracksException(e);
+ }
+ thread.start();
+ }
+
+ public void stop() throws HyracksException {
+ synchronized (this) {
+ stopped = true;
+ }
+ schedule(new AbstractWork() {
+ @Override
+ public void run() {
+ }
+ });
+ try {
+ stopSemaphore.acquire();
+ } catch (InterruptedException e) {
+ throw new HyracksException(e);
+ }
+ }
+
+ public void schedule(AbstractWork event) {
+ enqueueCount.incrementAndGet();
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Enqueue: " + enqueueCount);
+ }
+ if (LOGGER.isLoggable(event.logLevel())) {
+ LOGGER.log(event.logLevel(), "Scheduling: " + event);
+ }
+ queue.offer(event);
+ }
+
+ public void scheduleAndSync(SynchronizableWork sRunnable) throws Exception {
+ schedule(sRunnable);
+ sRunnable.sync();
+ }
+
+ private class WorkerThread extends Thread {
+ WorkerThread() {
+ setDaemon(true);
+ }
+
+ @Override
+ public void run() {
+ try {
+ AbstractWork r;
+ while (true) {
+ synchronized (WorkQueue.this) {
+ if (stopped) {
+ return;
+ }
+ }
+ try {
+ r = queue.take();
+ } catch (InterruptedException e) {
+ continue;
+ }
+ dequeueCount.incrementAndGet();
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Dequeue: " + dequeueCount + "/" + enqueueCount);
+ }
+ try {
+ if (LOGGER.isLoggable(r.logLevel())) {
+ LOGGER.log(r.logLevel(), "Executing: " + r);
+ }
+ r.run();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ } finally {
+ stopSemaphore.release();
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/.classpath b/hyracks-control-nc/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-control-nc/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-control-nc/.project b/hyracks-control-nc/.project
deleted file mode 100644
index 83d4a09..0000000
--- a/hyracks-control-nc/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-control-nc</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-control-nc/.settings/org.eclipse.jdt.core.prefs b/hyracks-control-nc/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index dfac000..0000000
--- a/hyracks-control-nc/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:05 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-control-nc/.settings/org.maven.ide.eclipse.prefs b/hyracks-control-nc/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 787d153..0000000
--- a/hyracks-control-nc/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Sep 28 12:04:30 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-control-nc/pom.xml b/hyracks-control-nc/pom.xml
index 3c98365..8c56958 100644
--- a/hyracks-control-nc/pom.xml
+++ b/hyracks-control-nc/pom.xml
@@ -1,13 +1,10 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -33,10 +30,15 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-net</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
</dependencies>
<reporting>
<plugins>
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java
index 2176dda..8f7bc4a 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Joblet.java
@@ -18,14 +18,13 @@
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Hashtable;
import java.util.Map;
-import java.util.concurrent.Executor;
import edu.uci.ics.hyracks.api.application.INCApplicationContext;
import edu.uci.ics.hyracks.api.comm.IPartitionCollector;
import edu.uci.ics.hyracks.api.comm.PartitionChannel;
import edu.uci.ics.hyracks.api.context.IHyracksJobletContext;
-import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
import edu.uci.ics.hyracks.api.dataflow.TaskId;
import edu.uci.ics.hyracks.api.dataflow.state.ITaskState;
@@ -34,34 +33,37 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.io.IWorkspaceFileFactory;
+import edu.uci.ics.hyracks.api.job.IJobletEventListener;
import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.job.JobActivityGraph;
import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
import edu.uci.ics.hyracks.api.job.profiling.counters.ICounter;
import edu.uci.ics.hyracks.api.job.profiling.counters.ICounterContext;
-import edu.uci.ics.hyracks.api.naming.MultipartName;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
import edu.uci.ics.hyracks.api.resources.IDeallocatable;
import edu.uci.ics.hyracks.control.common.job.PartitionRequest;
import edu.uci.ics.hyracks.control.common.job.PartitionState;
import edu.uci.ics.hyracks.control.common.job.profiling.counters.Counter;
import edu.uci.ics.hyracks.control.common.job.profiling.om.JobletProfile;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.PartitionProfile;
import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.control.nc.io.WorkspaceFileFactory;
import edu.uci.ics.hyracks.control.nc.resources.DefaultDeallocatableRegistry;
public class Joblet implements IHyracksJobletContext, ICounterContext {
- private static final long serialVersionUID = 1L;
-
private final NodeControllerService nodeController;
private final INCApplicationContext appCtx;
private final JobId jobId;
+ private final JobActivityGraph jag;
+
private final Map<PartitionId, IPartitionCollector> partitionRequestMap;
- private final Map<OperatorDescriptorId, Map<Integer, IOperatorEnvironment>> envMap;
+ private final IOperatorEnvironment env;
private final Map<TaskId, ITaskState> taskStateMap;
@@ -69,24 +71,29 @@
private final Map<String, Counter> counterMap;
- private final Map<MultipartName, Object> localVariableMap;
-
private final DefaultDeallocatableRegistry deallocatableRegistry;
private final IWorkspaceFileFactory fileFactory;
- public Joblet(NodeControllerService nodeController, JobId jobId, INCApplicationContext appCtx) {
+ private IJobletEventListener jobletEventListener;
+
+ private JobStatus cleanupStatus;
+
+ private boolean cleanupPending;
+
+ public Joblet(NodeControllerService nodeController, JobId jobId, INCApplicationContext appCtx, JobActivityGraph jag) {
this.nodeController = nodeController;
this.appCtx = appCtx;
this.jobId = jobId;
+ this.jag = jag;
partitionRequestMap = new HashMap<PartitionId, IPartitionCollector>();
- envMap = new HashMap<OperatorDescriptorId, Map<Integer, IOperatorEnvironment>>();
+ env = new OperatorEnvironmentImpl(nodeController.getId());
taskStateMap = new HashMap<TaskId, ITaskState>();
taskMap = new HashMap<TaskAttemptId, Task>();
counterMap = new HashMap<String, Counter>();
- localVariableMap = new HashMap<MultipartName, Object>();
deallocatableRegistry = new DefaultDeallocatableRegistry();
fileFactory = new WorkspaceFileFactory(this, (IOManager) appCtx.getRootContext().getIOManager());
+ cleanupPending = false;
}
@Override
@@ -94,36 +101,29 @@
return jobId;
}
- public synchronized IOperatorEnvironment getEnvironment(OperatorDescriptorId opId, int partition) {
- if (!envMap.containsKey(opId)) {
- envMap.put(opId, new HashMap<Integer, IOperatorEnvironment>());
- }
- Map<Integer, IOperatorEnvironment> opEnvMap = envMap.get(opId);
- if (!opEnvMap.containsKey(partition)) {
- opEnvMap.put(partition, new OperatorEnvironmentImpl(nodeController.getId()));
- }
- return opEnvMap.get(partition);
+ public JobActivityGraph getJobActivityGraph() {
+ return jag;
+ }
+
+ public IOperatorEnvironment getEnvironment() {
+ return env;
}
public void addTask(Task task) {
taskMap.put(task.getTaskAttemptId(), task);
}
+ public void removeTask(Task task) {
+ taskMap.remove(task.getTaskAttemptId());
+ if (cleanupPending && taskMap.isEmpty()) {
+ performCleanup();
+ }
+ }
+
public Map<TaskAttemptId, Task> getTaskMap() {
return taskMap;
}
- public synchronized Object lookupLocalVariable(MultipartName name) throws HyracksDataException {
- if (!localVariableMap.containsKey(name)) {
- throw new HyracksDataException("Unknown variable: " + name);
- }
- return localVariableMap.get(name);
- }
-
- public synchronized void setLocalVariable(MultipartName name, Object value) {
- localVariableMap.put(name, value);
- }
-
private final class OperatorEnvironmentImpl implements IOperatorEnvironment {
private final String nodeId;
@@ -146,33 +146,18 @@
}
}
- public Executor getExecutor() {
- return nodeController.getExecutor();
- }
-
- public synchronized void notifyTaskComplete(Task task) throws Exception {
- taskMap.remove(task);
- TaskProfile taskProfile = new TaskProfile(task.getTaskAttemptId());
- task.dumpProfile(taskProfile);
- nodeController.notifyTaskComplete(jobId, task.getTaskAttemptId(), taskProfile);
- }
-
- public synchronized void notifyTaskFailed(Task task, Exception exception) {
- taskMap.remove(task);
- nodeController.notifyTaskFailed(jobId, task.getTaskAttemptId(), exception);
- }
-
public NodeControllerService getNodeController() {
return nodeController;
}
- public synchronized void dumpProfile(JobletProfile jProfile) {
+ public void dumpProfile(JobletProfile jProfile) {
Map<String, Long> counters = jProfile.getCounters();
for (Map.Entry<String, Counter> e : counterMap.entrySet()) {
counters.put(e.getKey(), e.getValue().get());
}
for (Task task : taskMap.values()) {
- TaskProfile taskProfile = new TaskProfile(task.getTaskAttemptId());
+ TaskProfile taskProfile = new TaskProfile(task.getTaskAttemptId(),
+ new Hashtable<PartitionId, PartitionProfile>(task.getPartitionSendProfile()));
task.dumpProfile(taskProfile);
jProfile.getTaskProfiles().put(task.getTaskAttemptId(), taskProfile);
}
@@ -194,7 +179,12 @@
}
public void close() {
- deallocatableRegistry.close();
+ nodeController.getExecutor().execute(new Runnable() {
+ @Override
+ public void run() {
+ deallocatableRegistry.close();
+ }
+ });
}
@Override
@@ -247,4 +237,35 @@
collector.addPartitions(Collections.singleton(channel));
}
}
+
+ public IJobletEventListener getJobletEventListener() {
+ return jobletEventListener;
+ }
+
+ public void setJobletEventListener(IJobletEventListener jobletEventListener) {
+ this.jobletEventListener = jobletEventListener;
+ }
+
+ public void cleanup(JobStatus status) {
+ cleanupStatus = status;
+ cleanupPending = true;
+ if (taskMap.isEmpty()) {
+ performCleanup();
+ }
+ }
+
+ private void performCleanup() {
+ nodeController.getJobletMap().remove(jobId);
+ IJobletEventListener listener = getJobletEventListener();
+ if (listener != null) {
+ listener.jobletFinish(cleanupStatus);
+ }
+ close();
+ cleanupPending = false;
+ try {
+ nodeController.getClusterController().notifyJobletCleanup(jobId, nodeController.getId());
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
index 28cb8be..f3d00ac 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/NodeControllerService.java
@@ -15,9 +15,7 @@
package edu.uci.ics.hyracks.control.nc;
import java.io.File;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Serializable;
+import java.lang.management.GarbageCollectorMXBean;
import java.lang.management.ManagementFactory;
import java.lang.management.MemoryMXBean;
import java.lang.management.MemoryUsage;
@@ -26,11 +24,8 @@
import java.lang.management.ThreadMXBean;
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.rmi.registry.LocateRegistry;
-import java.rmi.registry.Registry;
import java.text.MessageFormat;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.Hashtable;
import java.util.List;
import java.util.Map;
@@ -44,77 +39,63 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.io.IOUtils;
-import org.apache.http.HttpResponse;
-import org.apache.http.client.HttpClient;
-import org.apache.http.client.methods.HttpGet;
-import org.apache.http.impl.client.DefaultHttpClient;
-
-import edu.uci.ics.hyracks.api.application.INCApplicationContext;
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.comm.IPartitionCollector;
-import edu.uci.ics.hyracks.api.comm.IPartitionWriterFactory;
-import edu.uci.ics.hyracks.api.comm.NetworkAddress;
-import edu.uci.ics.hyracks.api.comm.PartitionChannel;
import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
-import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
-import edu.uci.ics.hyracks.api.dataflow.IActivity;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
-import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
-import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
-import edu.uci.ics.hyracks.api.dataflow.TaskId;
-import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
-import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.io.IODeviceHandle;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
-import edu.uci.ics.hyracks.api.job.JobActivityGraph;
import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.naming.MultipartName;
-import edu.uci.ics.hyracks.api.partitions.PartitionId;
import edu.uci.ics.hyracks.control.common.AbstractRemoteService;
-import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
import edu.uci.ics.hyracks.control.common.base.IClusterController;
-import edu.uci.ics.hyracks.control.common.base.INodeController;
import edu.uci.ics.hyracks.control.common.context.ServerContext;
import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
import edu.uci.ics.hyracks.control.common.controllers.NodeRegistration;
import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatData;
-import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
+import edu.uci.ics.hyracks.control.common.heartbeat.HeartbeatSchema;
+import edu.uci.ics.hyracks.control.common.ipc.CCNCFunctions;
+import edu.uci.ics.hyracks.control.common.ipc.ClusterControllerRemoteProxy;
import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
-import edu.uci.ics.hyracks.control.common.job.profiling.om.JobletProfile;
-import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
+import edu.uci.ics.hyracks.control.common.work.FutureValue;
+import edu.uci.ics.hyracks.control.common.work.WorkQueue;
import edu.uci.ics.hyracks.control.nc.application.NCApplicationContext;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
-import edu.uci.ics.hyracks.control.nc.net.ConnectionManager;
-import edu.uci.ics.hyracks.control.nc.net.NetworkInputChannel;
-import edu.uci.ics.hyracks.control.nc.partitions.MaterializedPartitionWriter;
+import edu.uci.ics.hyracks.control.nc.net.NetworkManager;
import edu.uci.ics.hyracks.control.nc.partitions.PartitionManager;
-import edu.uci.ics.hyracks.control.nc.partitions.PipelinedPartition;
-import edu.uci.ics.hyracks.control.nc.partitions.ReceiveSideMaterializingCollector;
import edu.uci.ics.hyracks.control.nc.runtime.RootHyracksContext;
+import edu.uci.ics.hyracks.control.nc.work.AbortTasksWork;
+import edu.uci.ics.hyracks.control.nc.work.BuildJobProfilesWork;
+import edu.uci.ics.hyracks.control.nc.work.CleanupJobletWork;
+import edu.uci.ics.hyracks.control.nc.work.CreateApplicationWork;
+import edu.uci.ics.hyracks.control.nc.work.DestroyApplicationWork;
+import edu.uci.ics.hyracks.control.nc.work.ReportPartitionAvailabilityWork;
+import edu.uci.ics.hyracks.control.nc.work.StartTasksWork;
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.IIPCI;
+import edu.uci.ics.hyracks.ipc.api.IPCPerformanceCounters;
+import edu.uci.ics.hyracks.ipc.impl.IPCSystem;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MuxDemuxPerformanceCounters;
-public class NodeControllerService extends AbstractRemoteService implements INodeController {
+public class NodeControllerService extends AbstractRemoteService {
private static Logger LOGGER = Logger.getLogger(NodeControllerService.class.getName());
- private static final long serialVersionUID = 1L;
-
private NCConfig ncConfig;
private final String id;
private final IHyracksRootContext ctx;
+ private final IPCSystem ipc;
+
private final PartitionManager partitionManager;
- private final ConnectionManager connectionManager;
+ private final NetworkManager netManager;
+
+ private final WorkQueue queue;
private final Timer timer;
+ private boolean registrationPending;
+
+ private Exception registrationException;
+
private IClusterController ccs;
private final Map<JobId, Joblet> jobletMap;
@@ -123,12 +104,16 @@
private NodeParameters nodeParameters;
+ private HeartbeatTask heartbeatTask;
+
private final ServerContext serverCtx;
private final Map<String, NCApplicationContext> applications;
private final MemoryMXBean memoryMXBean;
+ private final List<GarbageCollectorMXBean> gcMXBeans;
+
private final ThreadMXBean threadMXBean;
private final RuntimeMXBean runtimeMXBean;
@@ -139,23 +124,28 @@
this.ncConfig = ncConfig;
id = ncConfig.nodeId;
executor = Executors.newCachedThreadPool();
+ NodeControllerIPCI ipci = new NodeControllerIPCI();
+ ipc = new IPCSystem(new InetSocketAddress(ncConfig.clusterNetIPAddress, 0), ipci,
+ new CCNCFunctions.SerializerDeserializer());
this.ctx = new RootHyracksContext(ncConfig.frameSize, new IOManager(getDevices(ncConfig.ioDevices), executor));
if (id == null) {
throw new Exception("id not set");
}
- connectionManager = new ConnectionManager(ctx, getIpAddress(ncConfig));
partitionManager = new PartitionManager(this);
- connectionManager.setPartitionRequestListener(partitionManager);
+ netManager = new NetworkManager(ctx, getIpAddress(ncConfig), partitionManager, ncConfig.nNetThreads);
+ queue = new WorkQueue();
jobletMap = new Hashtable<JobId, Joblet>();
timer = new Timer(true);
serverCtx = new ServerContext(ServerContext.ServerType.NODE_CONTROLLER, new File(new File(
NodeControllerService.class.getName()), id));
applications = new Hashtable<String, NCApplicationContext>();
memoryMXBean = ManagementFactory.getMemoryMXBean();
+ gcMXBeans = ManagementFactory.getGarbageCollectorMXBeans();
threadMXBean = ManagementFactory.getThreadMXBean();
runtimeMXBean = ManagementFactory.getRuntimeMXBean();
osMXBean = ManagementFactory.getOperatingSystemMXBean();
+ registrationPending = true;
}
public IHyracksRootContext getRootContext() {
@@ -172,22 +162,50 @@
return devices;
}
+ private synchronized void setNodeRegistrationResult(NodeParameters parameters, Exception exception) {
+ this.nodeParameters = parameters;
+ this.registrationException = exception;
+ this.registrationPending = false;
+ notifyAll();
+ }
+
@Override
public void start() throws Exception {
LOGGER.log(Level.INFO, "Starting NodeControllerService");
- connectionManager.start();
- Registry registry = LocateRegistry.getRegistry(ncConfig.ccHost, ncConfig.ccPort);
- IClusterController cc = (IClusterController) registry.lookup(IClusterController.class.getName());
- this.nodeParameters = cc.registerNode(new NodeRegistration(this, id, ncConfig, connectionManager
- .getNetworkAddress(), osMXBean.getName(), osMXBean.getArch(), osMXBean.getVersion(), osMXBean
- .getAvailableProcessors()));
+ ipc.start();
+ netManager.start();
+ IIPCHandle ccIPCHandle = ipc.getHandle(new InetSocketAddress(ncConfig.ccHost, ncConfig.ccPort));
+ this.ccs = new ClusterControllerRemoteProxy(ccIPCHandle);
+ HeartbeatSchema.GarbageCollectorInfo[] gcInfos = new HeartbeatSchema.GarbageCollectorInfo[gcMXBeans.size()];
+ for (int i = 0; i < gcInfos.length; ++i) {
+ gcInfos[i] = new HeartbeatSchema.GarbageCollectorInfo(gcMXBeans.get(i).getName());
+ }
+ HeartbeatSchema hbSchema = new HeartbeatSchema(gcInfos);
+ ccs.registerNode(new NodeRegistration(ipc.getSocketAddress(), id, ncConfig, netManager.getNetworkAddress(),
+ osMXBean.getName(), osMXBean.getArch(), osMXBean.getVersion(), osMXBean.getAvailableProcessors(),
+ runtimeMXBean.getVmName(), runtimeMXBean.getVmVersion(), runtimeMXBean.getVmVendor(), runtimeMXBean
+ .getClassPath(), runtimeMXBean.getLibraryPath(), runtimeMXBean.getBootClassPath(),
+ runtimeMXBean.getInputArguments(), runtimeMXBean.getSystemProperties(), hbSchema));
+
+ synchronized (this) {
+ while (registrationPending) {
+ wait();
+ }
+ }
+ if (registrationException != null) {
+ throw registrationException;
+ }
+
+ queue.start();
+
+ heartbeatTask = new HeartbeatTask(ccs);
// Schedule heartbeat generator.
- timer.schedule(new HeartbeatTask(cc), 0, nodeParameters.getHeartbeatPeriod());
+ timer.schedule(heartbeatTask, 0, nodeParameters.getHeartbeatPeriod());
if (nodeParameters.getProfileDumpPeriod() > 0) {
// Schedule profile dump generator.
- timer.schedule(new ProfileDumpTask(cc), 0, nodeParameters.getProfileDumpPeriod());
+ timer.schedule(new ProfileDumpTask(ccs), 0, nodeParameters.getProfileDumpPeriod());
}
LOGGER.log(Level.INFO, "Started NodeControllerService");
@@ -197,17 +215,30 @@
public void stop() throws Exception {
LOGGER.log(Level.INFO, "Stopping NodeControllerService");
partitionManager.close();
- connectionManager.stop();
+ heartbeatTask.cancel();
+ netManager.stop();
+ queue.stop();
LOGGER.log(Level.INFO, "Stopped NodeControllerService");
}
- @Override
public String getId() {
return id;
}
- public ConnectionManager getConnectionManager() {
- return connectionManager;
+ public ServerContext getServerContext() {
+ return serverCtx;
+ }
+
+ public Map<String, NCApplicationContext> getApplications() {
+ return applications;
+ }
+
+ public Map<JobId, Joblet> getJobletMap() {
+ return jobletMap;
+ }
+
+ public NetworkManager getNetworkManager() {
+ return netManager;
}
public PartitionManager getPartitionManager() {
@@ -218,6 +249,22 @@
return ccs;
}
+ public NodeParameters getNodeParameters() {
+ return nodeParameters;
+ }
+
+ public Executor getExecutor() {
+ return executor;
+ }
+
+ public NCConfig getConfiguration() throws Exception {
+ return ncConfig;
+ }
+
+ public WorkQueue getWorkQueue() {
+ return queue;
+ }
+
private static InetAddress getIpAddress(NCConfig ncConfig) throws Exception {
String ipaddrStr = ncConfig.dataIPAddress;
ipaddrStr = ipaddrStr.trim();
@@ -235,184 +282,6 @@
return InetAddress.getByAddress(ipBytes);
}
- @Override
- public void startTasks(String appName, final JobId jobId, byte[] jagBytes,
- List<TaskAttemptDescriptor> taskDescriptors,
- Map<ConnectorDescriptorId, IConnectorPolicy> connectorPoliciesMap, byte[] ctxVarBytes) throws Exception {
- try {
- NCApplicationContext appCtx = applications.get(appName);
- final JobActivityGraph plan = (JobActivityGraph) appCtx.deserialize(jagBytes);
- Map<MultipartName, Object> ctxVarMap = (Map<MultipartName, Object>) appCtx.deserialize(ctxVarBytes);
-
- IRecordDescriptorProvider rdp = new IRecordDescriptorProvider() {
- @Override
- public RecordDescriptor getOutputRecordDescriptor(OperatorDescriptorId opId, int outputIndex) {
- return plan.getJobSpecification().getOperatorOutputRecordDescriptor(opId, outputIndex);
- }
-
- @Override
- public RecordDescriptor getInputRecordDescriptor(OperatorDescriptorId opId, int inputIndex) {
- return plan.getJobSpecification().getOperatorInputRecordDescriptor(opId, inputIndex);
- }
- };
-
- final Joblet joblet = getOrCreateLocalJoblet(jobId, appCtx);
-
- for (TaskAttemptDescriptor td : taskDescriptors) {
- TaskAttemptId taId = td.getTaskAttemptId();
- TaskId tid = taId.getTaskId();
- IActivity han = plan.getActivityNodeMap().get(tid.getActivityId());
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Initializing " + taId + " -> " + han);
- }
- final int partition = tid.getPartition();
- Map<MultipartName, Object> inputGlobalVariables = createInputGlobalVariables(ctxVarMap, han);
- Task task = new Task(joblet, taId, han.getClass().getName(), executor);
- IOperatorEnvironment env = joblet.getEnvironment(tid.getActivityId().getOperatorDescriptorId(),
- tid.getPartition());
- IOperatorNodePushable operator = han.createPushRuntime(task, env, rdp, partition,
- td.getPartitionCount());
-
- List<IPartitionCollector> collectors = new ArrayList<IPartitionCollector>();
-
- List<IConnectorDescriptor> inputs = plan.getActivityInputConnectorDescriptors(tid.getActivityId());
- if (inputs != null) {
- for (int i = 0; i < inputs.size(); ++i) {
- IConnectorDescriptor conn = inputs.get(i);
- IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId());
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("input: " + i + ": " + conn.getConnectorId());
- }
- RecordDescriptor recordDesc = plan.getJobSpecification().getConnectorRecordDescriptor(conn);
- IPartitionCollector collector = createPartitionCollector(td, partition, task, i, conn,
- recordDesc, cPolicy);
- collectors.add(collector);
- }
- }
- List<IConnectorDescriptor> outputs = plan.getActivityOutputConnectorDescriptors(tid.getActivityId());
- if (outputs != null) {
- for (int i = 0; i < outputs.size(); ++i) {
- final IConnectorDescriptor conn = outputs.get(i);
- RecordDescriptor recordDesc = plan.getJobSpecification().getConnectorRecordDescriptor(conn);
- IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId());
-
- IPartitionWriterFactory pwFactory = createPartitionWriterFactory(cPolicy, jobId, conn,
- partition, taId);
-
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("output: " + i + ": " + conn.getConnectorId());
- }
- IFrameWriter writer = conn.createPartitioner(task, recordDesc, pwFactory, partition,
- td.getPartitionCount(), td.getOutputPartitionCounts()[i]);
- operator.setOutputFrameWriter(i, writer, recordDesc);
- }
- }
-
- task.setTaskRuntime(collectors.toArray(new IPartitionCollector[collectors.size()]), operator);
- joblet.addTask(task);
-
- task.start();
- }
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- private Map<MultipartName, Object> createInputGlobalVariables(Map<MultipartName, Object> ctxVarMap, IActivity han) {
- Map<MultipartName, Object> gVars = new HashMap<MultipartName, Object>();
- // for (MultipartName inVar : han.getInputVariables()) {
- // gVars.put(inVar, ctxVarMap.get(inVar));
- // }
- return gVars;
- }
-
- private IPartitionCollector createPartitionCollector(TaskAttemptDescriptor td, final int partition, Task task,
- int i, IConnectorDescriptor conn, RecordDescriptor recordDesc, IConnectorPolicy cPolicy)
- throws HyracksDataException {
- IPartitionCollector collector = conn.createPartitionCollector(task, recordDesc, partition,
- td.getInputPartitionCounts()[i], td.getPartitionCount());
- if (cPolicy.materializeOnReceiveSide()) {
- return new ReceiveSideMaterializingCollector(ctx, partitionManager, collector, task.getTaskAttemptId(),
- executor);
- } else {
- return collector;
- }
- }
-
- private IPartitionWriterFactory createPartitionWriterFactory(IConnectorPolicy cPolicy, final JobId jobId,
- final IConnectorDescriptor conn, final int senderIndex, final TaskAttemptId taId) {
- if (cPolicy.materializeOnSendSide()) {
- return new IPartitionWriterFactory() {
- @Override
- public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException {
- return new MaterializedPartitionWriter(ctx, partitionManager, new PartitionId(jobId,
- conn.getConnectorId(), senderIndex, receiverIndex), taId, executor);
- }
- };
- } else {
- return new IPartitionWriterFactory() {
- @Override
- public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException {
- return new PipelinedPartition(partitionManager, new PartitionId(jobId, conn.getConnectorId(),
- senderIndex, receiverIndex), taId);
- }
- };
- }
- }
-
- private synchronized Joblet getOrCreateLocalJoblet(JobId jobId, INCApplicationContext appCtx) throws Exception {
- Joblet ji = jobletMap.get(jobId);
- if (ji == null) {
- ji = new Joblet(this, jobId, appCtx);
- jobletMap.put(jobId, ji);
- }
- return ji;
- }
-
- public Executor getExecutor() {
- return executor;
- }
-
- @Override
- public void cleanUpJob(JobId jobId) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Cleaning up after job: " + jobId);
- }
- Joblet joblet = jobletMap.remove(jobId);
- if (joblet != null) {
- partitionManager.unregisterPartitions(jobId);
- joblet.close();
- }
- }
-
- public void notifyTaskComplete(JobId jobId, TaskAttemptId taskId, TaskProfile taskProfile) throws Exception {
- try {
- ccs.notifyTaskComplete(jobId, taskId, id, taskProfile);
- } catch (Exception e) {
- e.printStackTrace();
- throw e;
- }
- }
-
- public void notifyTaskFailed(JobId jobId, TaskAttemptId taskId, Exception exception) {
- try {
- ccs.notifyTaskFailure(jobId, taskId, id, exception);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public void notifyRegistration(IClusterController ccs) throws Exception {
- this.ccs = ccs;
- }
-
- @Override
- public NCConfig getConfiguration() throws Exception {
- return ncConfig;
- }
-
private class HeartbeatTask extends TimerTask {
private IClusterController cc;
@@ -421,6 +290,8 @@
public HeartbeatTask(IClusterController cc) {
this.cc = cc;
hbData = new HeartbeatData();
+ hbData.gcCollectionCounts = new long[gcMXBeans.size()];
+ hbData.gcCollectionTimes = new long[gcMXBeans.size()];
}
@Override
@@ -439,6 +310,25 @@
hbData.peakThreadCount = threadMXBean.getPeakThreadCount();
hbData.totalStartedThreadCount = threadMXBean.getTotalStartedThreadCount();
hbData.systemLoadAverage = osMXBean.getSystemLoadAverage();
+ int gcN = gcMXBeans.size();
+ for (int i = 0; i < gcN; ++i) {
+ GarbageCollectorMXBean gcMXBean = gcMXBeans.get(i);
+ hbData.gcCollectionCounts[i] = gcMXBean.getCollectionCount();
+ hbData.gcCollectionTimes[i] = gcMXBean.getCollectionTime();
+ }
+
+ MuxDemuxPerformanceCounters netPC = netManager.getPerformanceCounters();
+ hbData.netPayloadBytesRead = netPC.getPayloadBytesRead();
+ hbData.netPayloadBytesWritten = netPC.getPayloadBytesWritten();
+ hbData.netSignalingBytesRead = netPC.getSignalingBytesRead();
+ hbData.netSignalingBytesWritten = netPC.getSignalingBytesWritten();
+
+ IPCPerformanceCounters ipcPC = ipc.getPerformanceCounters();
+ hbData.ipcMessagesSent = ipcPC.getMessageSentCount();
+ hbData.ipcMessageBytesSent = ipcPC.getMessageBytesSent();
+ hbData.ipcMessagesReceived = ipcPC.getMessageReceivedCount();
+ hbData.ipcMessageBytesReceived = ipcPC.getMessageBytesReceived();
+
try {
cc.nodeHeartbeat(id, hbData);
} catch (Exception e) {
@@ -457,24 +347,10 @@
@Override
public void run() {
try {
- List<JobProfile> profiles;
- synchronized (NodeControllerService.this) {
- profiles = new ArrayList<JobProfile>();
- for (Joblet ji : jobletMap.values()) {
- profiles.add(new JobProfile(ji.getJobId()));
- }
- }
- for (JobProfile jProfile : profiles) {
- Joblet ji;
- JobletProfile jobletProfile = new JobletProfile(id);
- synchronized (NodeControllerService.this) {
- ji = jobletMap.get(jProfile.getJobId());
- }
- if (ji != null) {
- ji.dumpProfile(jobletProfile);
- jProfile.getJobletProfiles().put(id, jobletProfile);
- }
- }
+ FutureValue<List<JobProfile>> fv = new FutureValue<List<JobProfile>>();
+ BuildJobProfilesWork bjpw = new BuildJobProfilesWork(NodeControllerService.this, fv);
+ queue.scheduleAndSync(bjpw);
+ List<JobProfile> profiles = fv.get();
if (!profiles.isEmpty()) {
cc.reportProfile(id, profiles);
}
@@ -484,69 +360,58 @@
}
}
- @Override
- public synchronized void abortTasks(JobId jobId, List<TaskAttemptId> tasks) throws Exception {
- if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("Aborting Tasks: " + jobId + ":" + tasks);
- }
- Joblet ji = jobletMap.get(jobId);
- if (ji != null) {
- Map<TaskAttemptId, Task> taskMap = ji.getTaskMap();
- for (TaskAttemptId taId : tasks) {
- Task task = taskMap.get(taId);
- if (task != null) {
- task.abort();
+ private final class NodeControllerIPCI implements IIPCI {
+ @Override
+ public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
+ CCNCFunctions.Function fn = (CCNCFunctions.Function) payload;
+ switch (fn.getFunctionId()) {
+ case START_TASKS: {
+ CCNCFunctions.StartTasksFunction stf = (CCNCFunctions.StartTasksFunction) fn;
+ queue.schedule(new StartTasksWork(NodeControllerService.this, stf.getAppName(), stf.getJobId(), stf
+ .getPlanBytes(), stf.getTaskDescriptors(), stf.getConnectorPolicies()));
+ return;
+ }
+
+ case ABORT_TASKS: {
+ CCNCFunctions.AbortTasksFunction atf = (CCNCFunctions.AbortTasksFunction) fn;
+ queue.schedule(new AbortTasksWork(NodeControllerService.this, atf.getJobId(), atf.getTasks()));
+ return;
+ }
+
+ case CLEANUP_JOBLET: {
+ CCNCFunctions.CleanupJobletFunction cjf = (CCNCFunctions.CleanupJobletFunction) fn;
+ queue.schedule(new CleanupJobletWork(NodeControllerService.this, cjf.getJobId(), cjf.getStatus()));
+ return;
+ }
+
+ case CREATE_APPLICATION: {
+ CCNCFunctions.CreateApplicationFunction caf = (CCNCFunctions.CreateApplicationFunction) fn;
+ queue.schedule(new CreateApplicationWork(NodeControllerService.this, caf.getAppName(), caf
+ .isDeployHar(), caf.getSerializedDistributedState()));
+ return;
+ }
+
+ case DESTROY_APPLICATION: {
+ CCNCFunctions.DestroyApplicationFunction daf = (CCNCFunctions.DestroyApplicationFunction) fn;
+ queue.schedule(new DestroyApplicationWork(NodeControllerService.this, daf.getAppName()));
+ return;
+ }
+
+ case REPORT_PARTITION_AVAILABILITY: {
+ CCNCFunctions.ReportPartitionAvailabilityFunction rpaf = (CCNCFunctions.ReportPartitionAvailabilityFunction) fn;
+ queue.schedule(new ReportPartitionAvailabilityWork(NodeControllerService.this, rpaf
+ .getPartitionId(), rpaf.getNetworkAddress()));
+ return;
+ }
+
+ case NODE_REGISTRATION_RESULT: {
+ CCNCFunctions.NodeRegistrationResult nrrf = (CCNCFunctions.NodeRegistrationResult) fn;
+ setNodeRegistrationResult(nrrf.getNodeParameters(), nrrf.getException());
+ return;
}
}
- ji.close();
- }
- }
+ throw new IllegalArgumentException("Unknown function: " + fn.getFunctionId());
- @Override
- public void createApplication(String appName, boolean deployHar, byte[] serializedDistributedState)
- throws Exception {
- NCApplicationContext appCtx;
- synchronized (applications) {
- if (applications.containsKey(appName)) {
- throw new HyracksException("Duplicate application with name: " + appName + " being created.");
- }
- appCtx = new NCApplicationContext(serverCtx, ctx, appName, id);
- applications.put(appName, appCtx);
- }
- if (deployHar) {
- HttpClient hc = new DefaultHttpClient();
- HttpGet get = new HttpGet("http://" + ncConfig.ccHost + ":"
- + nodeParameters.getClusterControllerInfo().getWebPort() + "/applications/" + appName);
- HttpResponse response = hc.execute(get);
- InputStream is = response.getEntity().getContent();
- OutputStream os = appCtx.getHarOutputStream();
- try {
- IOUtils.copyLarge(is, os);
- } finally {
- os.close();
- is.close();
- }
- }
- appCtx.initializeClassPath();
- appCtx.setDistributedState((Serializable) appCtx.deserialize(serializedDistributedState));
- appCtx.initialize();
- }
-
- @Override
- public void destroyApplication(String appName) throws Exception {
- ApplicationContext appCtx = applications.remove(appName);
- if (appCtx != null) {
- appCtx.deinitialize();
- }
- }
-
- @Override
- public void reportPartitionAvailability(PartitionId pid, NetworkAddress networkAddress) throws Exception {
- Joblet ji = jobletMap.get(pid.getJobId());
- if (ji != null) {
- PartitionChannel channel = new PartitionChannel(pid, new NetworkInputChannel(ctx, connectionManager,
- new InetSocketAddress(networkAddress.getIpAddress(), networkAddress.getPort()), pid, 1));
- ji.reportPartitionAvailability(channel);
}
}
}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Task.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Task.java
index 6a73902..1bb0b2f 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Task.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/Task.java
@@ -14,10 +14,15 @@
*/
package edu.uci.ics.hyracks.control.nc;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintWriter;
+import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
-import java.util.Collections;
import java.util.HashMap;
+import java.util.Hashtable;
+import java.util.LinkedHashSet;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Executor;
import java.util.concurrent.Semaphore;
@@ -26,25 +31,29 @@
import edu.uci.ics.hyracks.api.comm.IPartitionCollector;
import edu.uci.ics.hyracks.api.context.IHyracksJobletContext;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
+import edu.uci.ics.hyracks.api.dataflow.state.ITaskState;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.io.IIOManager;
import edu.uci.ics.hyracks.api.io.IWorkspaceFileFactory;
+import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.profiling.counters.ICounter;
import edu.uci.ics.hyracks.api.job.profiling.counters.ICounterContext;
-import edu.uci.ics.hyracks.api.naming.MultipartName;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
import edu.uci.ics.hyracks.api.resources.IDeallocatable;
-import edu.uci.ics.hyracks.api.workflow.variables.WorkflowVariableDescriptor;
import edu.uci.ics.hyracks.control.common.job.PartitionState;
import edu.uci.ics.hyracks.control.common.job.profiling.counters.Counter;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.PartitionProfile;
import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
import edu.uci.ics.hyracks.control.nc.io.IOManager;
import edu.uci.ics.hyracks.control.nc.io.WorkspaceFileFactory;
import edu.uci.ics.hyracks.control.nc.resources.DefaultDeallocatableRegistry;
+import edu.uci.ics.hyracks.control.nc.work.NotifyTaskCompleteWork;
+import edu.uci.ics.hyracks.control.nc.work.NotifyTaskFailureWork;
public class Task implements IHyracksTaskContext, ICounterContext, Runnable {
private final Joblet joblet;
@@ -61,16 +70,22 @@
private final Map<String, Counter> counterMap;
- private final Map<MultipartName, Object> inputGlobalVariables;
+ private final IOperatorEnvironment opEnv;
- private final Map<MultipartName, Object> outputVariables;
+ private final Map<PartitionId, PartitionProfile> partitionSendProfile;
- private final Map<MultipartName, WorkflowVariableDescriptor> outputVariableDescriptorMap;
+ private final Set<Thread> pendingThreads;
private IPartitionCollector[] collectors;
private IOperatorNodePushable operator;
+ private volatile boolean failed;
+
+ private ByteArrayOutputStream errorBaos;
+
+ private PrintWriter errorWriter;
+
private volatile boolean aborted;
public Task(Joblet joblet, TaskAttemptId taskId, String displayName, Executor executor) {
@@ -81,13 +96,12 @@
fileFactory = new WorkspaceFileFactory(this, (IOManager) joblet.getIOManager());
deallocatableRegistry = new DefaultDeallocatableRegistry();
counterMap = new HashMap<String, Counter>();
- // this.inputGlobalVariables = inputGlobalVariables;
- inputGlobalVariables = Collections.emptyMap();
- outputVariables = new HashMap<MultipartName, Object>();
- outputVariableDescriptorMap = new HashMap<MultipartName, WorkflowVariableDescriptor>();
- // for (WorkflowVariableDescriptor wvd : outputVariableDescriptors) {
- // outputVariableDescriptorMap.put(wvd.getName(), wvd);
- // }
+ opEnv = joblet.getEnvironment();
+ partitionSendProfile = new Hashtable<PartitionId, PartitionProfile>();
+ pendingThreads = new LinkedHashSet<Thread>();
+ failed = false;
+ errorBaos = new ByteArrayOutputStream();
+ errorWriter = new PrintWriter(errorBaos, true);
}
public void setTaskRuntime(IPartitionCollector[] collectors, IOperatorNodePushable operator) {
@@ -125,28 +139,6 @@
deallocatableRegistry.registerDeallocatable(deallocatable);
}
- @Override
- public Object lookupGlobalVariable(ActivityId producerActivity, int partition, String varName)
- throws HyracksDataException {
- MultipartName var = new MultipartName(producerActivity, partition, varName);
- if (!inputGlobalVariables.containsKey(var)) {
- throw new HyracksDataException("Unknown Variable: " + var);
- }
- return inputGlobalVariables.get(var);
- }
-
- @Override
- public Object lookupLocalVariable(ActivityId producerActivity, int partition, String varName)
- throws HyracksDataException {
- return joblet.lookupLocalVariable(new MultipartName(producerActivity, partition, varName));
- }
-
- @Override
- public void setVariable(String name, Object value) {
- outputVariables.put(new MultipartName(taskAttemptId.getTaskId().getActivityId(), taskAttemptId.getTaskId()
- .getPartition(), name), value);
- }
-
public void close() {
deallocatableRegistry.close();
}
@@ -176,6 +168,14 @@
return this;
}
+ public Joblet getJoblet() {
+ return joblet;
+ }
+
+ public Map<PartitionId, PartitionProfile> getPartitionSendProfile() {
+ return partitionSendProfile;
+ }
+
public synchronized void dumpProfile(TaskProfile tProfile) {
Map<String, Long> dumpMap = tProfile.getCounters();
for (Counter c : counterMap.values()) {
@@ -183,24 +183,49 @@
}
}
- public void start() throws HyracksException {
- aborted = false;
- joblet.getExecutor().execute(this);
+ public void setPartitionSendProfile(PartitionProfile profile) {
+ partitionSendProfile.put(profile.getPartitionId(), profile);
}
- public void abort() {
+ public void start() throws HyracksException {
+ aborted = false;
+ executor.execute(this);
+ }
+
+ public synchronized void abort() {
aborted = true;
for (IPartitionCollector c : collectors) {
c.abort();
}
+ for (Thread t : pendingThreads) {
+ t.interrupt();
+ }
+ }
+
+ private synchronized void addPendingThread(Thread t) {
+ pendingThreads.add(t);
+ }
+
+ private synchronized void removePendingThread(Thread t) {
+ pendingThreads.remove(t);
+ if (pendingThreads.isEmpty()) {
+ notifyAll();
+ }
+ }
+
+ public synchronized void waitForCompletion() throws InterruptedException {
+ while (!pendingThreads.isEmpty()) {
+ wait();
+ }
}
@Override
public void run() {
Thread ct = Thread.currentThread();
String threadName = ct.getName();
+ addPendingThread(ct);
try {
- ct.setName(displayName + ": " + taskAttemptId);
+ ct.setName(displayName + ":" + taskAttemptId + ":" + 0);
operator.initialize();
try {
if (collectors.length > 0) {
@@ -209,13 +234,29 @@
final IPartitionCollector collector = collectors[i];
final IFrameWriter writer = operator.getInputFrameWriter(i);
sem.acquire();
+ final int cIdx = i;
executor.execute(new Runnable() {
public void run() {
+ if (aborted) {
+ return;
+ }
+ Thread thread = Thread.currentThread();
+ addPendingThread(thread);
+ String oldName = thread.getName();
+ thread.setName(displayName + ":" + taskAttemptId + ":" + cIdx);
try {
pushFrames(collector, writer);
} catch (HyracksDataException e) {
+ synchronized (Task.this) {
+ failed = true;
+ errorWriter.println("Exception caught by thread: " + thread.getName());
+ e.printStackTrace(errorWriter);
+ errorWriter.println();
+ }
} finally {
+ thread.setName(oldName);
sem.release();
+ removePendingThread(thread);
}
}
});
@@ -229,30 +270,26 @@
} finally {
operator.deinitialize();
}
- Map<MultipartName, Object> outputGlobalVariables = new HashMap<MultipartName, Object>();
- for (Map.Entry<MultipartName, Object> e : outputVariables.entrySet()) {
- MultipartName varName = e.getKey();
- WorkflowVariableDescriptor wvd = outputVariableDescriptorMap.get(varName);
- if (wvd == null) {
- throw new HyracksDataException("Unknown variable found: " + varName);
- }
- switch (wvd.getScope()) {
- case LOCAL:
- joblet.setLocalVariable(varName, e.getValue());
- break;
-
- case GLOBAL:
- outputGlobalVariables.put(varName, e.getValue());
- break;
- }
- }
- joblet.notifyTaskComplete(this);
+ NodeControllerService ncs = joblet.getNodeController();
+ ncs.getWorkQueue().schedule(new NotifyTaskCompleteWork(ncs, this));
} catch (Exception e) {
- e.printStackTrace();
- joblet.notifyTaskFailed(this, e);
+ failed = true;
+ errorWriter.println("Exception caught by thread: " + ct.getName());
+ e.printStackTrace(errorWriter);
+ errorWriter.println();
} finally {
ct.setName(threadName);
close();
+ removePendingThread(ct);
+ }
+ if (failed) {
+ errorWriter.close();
+ NodeControllerService ncs = joblet.getNodeController();
+ try {
+ ncs.getWorkQueue().schedule(new NotifyTaskFailureWork(ncs, this, errorBaos.toString("UTF-8")));
+ } catch (UnsupportedEncodingException e) {
+ e.printStackTrace();
+ }
}
}
@@ -294,4 +331,14 @@
throw new HyracksDataException(e);
}
}
+
+ @Override
+ public void setTaskState(ITaskState taskState) {
+ opEnv.setTaskState(taskState);
+ }
+
+ @Override
+ public ITaskState getTaskState(TaskId taskId) {
+ return opEnv.getTaskState(taskId);
+ }
}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/application/NCApplicationContext.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/application/NCApplicationContext.java
index 06f4212..1121c6c 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/application/NCApplicationContext.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/application/NCApplicationContext.java
@@ -26,7 +26,6 @@
return nodeId;
}
- @Override
public void setDistributedState(Serializable state) {
distributedState = state;
}
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
index 2275d00..c3105ab 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/io/IOManager.java
@@ -83,6 +83,7 @@
throw new HyracksDataException("Error writing to file: " + fHandle.getFileReference().toString());
}
remaining -= len;
+ offset += len;
n += len;
}
return n;
@@ -104,6 +105,7 @@
return -1;
}
remaining -= len;
+ offset += len;
n += len;
}
return n;
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/ConnectionManager.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/ConnectionManager.java
deleted file mode 100644
index b9eaf1f..0000000
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/ConnectionManager.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.control.nc.net;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.ServerSocket;
-import java.nio.ByteBuffer;
-import java.nio.channels.AsynchronousCloseException;
-import java.nio.channels.SelectionKey;
-import java.nio.channels.Selector;
-import java.nio.channels.ServerSocketChannel;
-import java.nio.channels.SocketChannel;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import edu.uci.ics.hyracks.api.comm.NetworkAddress;
-import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
-import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.control.nc.partitions.IPartitionRequestListener;
-
-public class ConnectionManager {
- private static final Logger LOGGER = Logger.getLogger(ConnectionManager.class.getName());
-
- static final int INITIAL_MESSAGE_SIZE = 20;
-
- private final IHyracksRootContext ctx;
-
- private IPartitionRequestListener partitionRequestListener;
-
- private final ServerSocketChannel serverChannel;
-
- private volatile boolean stopped;
-
- private final ConnectionListenerThread connectionListener;
-
- private final DataListenerThread dataListener;
-
- private final NetworkAddress networkAddress;
-
- public ConnectionManager(IHyracksRootContext ctx, InetAddress inetAddress) throws IOException {
- this.ctx = ctx;
- serverChannel = ServerSocketChannel.open();
- ServerSocket serverSocket = serverChannel.socket();
- serverSocket.bind(new InetSocketAddress(inetAddress, 0));
- stopped = false;
- connectionListener = new ConnectionListenerThread();
- dataListener = new DataListenerThread();
- networkAddress = new NetworkAddress(serverSocket.getInetAddress(), serverSocket.getLocalPort());
-
- }
-
- public void setPartitionRequestListener(IPartitionRequestListener partitionRequestListener) {
- this.partitionRequestListener = partitionRequestListener;
- }
-
- public void start() {
- connectionListener.start();
- dataListener.start();
- }
-
- public void stop() {
- try {
- stopped = true;
- serverChannel.close();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public void connect(INetworkChannel channel) throws IOException {
- dataListener.addOutgoingConnection(channel);
- }
-
- private final class ConnectionListenerThread extends Thread {
- public ConnectionListenerThread() {
- super("Hyracks NC Connection Listener");
- setDaemon(true);
- }
-
- @Override
- public void run() {
- while (!stopped) {
- try {
- SocketChannel sc = serverChannel.accept();
- dataListener.addIncomingConnection(sc);
- } catch (AsynchronousCloseException e) {
- // do nothing
- if (!stopped) {
- e.printStackTrace();
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- }
-
- private final class DataListenerThread extends Thread {
- private Selector selector;
-
- private final List<SocketChannel> pendingIncomingConnections;
- private final Set<SocketChannel> pendingNegotiations;
- private final List<INetworkChannel> pendingOutgoingConnections;
-
- public DataListenerThread() {
- super("Hyracks Data Listener Thread");
- setDaemon(true);
- try {
- selector = Selector.open();
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- pendingIncomingConnections = new ArrayList<SocketChannel>();
- pendingNegotiations = new HashSet<SocketChannel>();
- pendingOutgoingConnections = new ArrayList<INetworkChannel>();
- }
-
- synchronized void addIncomingConnection(SocketChannel sc) throws IOException {
- pendingIncomingConnections.add(sc);
- selector.wakeup();
- }
-
- synchronized void addOutgoingConnection(INetworkChannel channel) throws IOException {
- pendingOutgoingConnections.add(channel);
- selector.wakeup();
- }
-
- @Override
- public void run() {
- while (!stopped) {
- try {
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine("Starting Select");
- }
- int n = selector.select();
- synchronized (this) {
- if (!pendingIncomingConnections.isEmpty()) {
- for (SocketChannel sc : pendingIncomingConnections) {
- sc.configureBlocking(false);
- SelectionKey scKey = sc.register(selector, SelectionKey.OP_READ);
- ByteBuffer buffer = ByteBuffer.allocate(INITIAL_MESSAGE_SIZE);
- scKey.attach(buffer);
- pendingNegotiations.add(sc);
- }
- pendingIncomingConnections.clear();
- }
- if (!pendingOutgoingConnections.isEmpty()) {
- for (INetworkChannel nc : pendingOutgoingConnections) {
- SocketChannel sc = SocketChannel.open();
- sc.configureBlocking(false);
- SelectionKey scKey = sc.register(selector, 0);
- scKey.attach(nc);
- nc.setSelectionKey(scKey);
- nc.notifyConnectionManagerRegistration();
- }
- pendingOutgoingConnections.clear();
- }
- if (LOGGER.isLoggable(Level.FINE)) {
- LOGGER.fine("Selector: " + n);
- }
- if (n > 0) {
- for (Iterator<SelectionKey> i = selector.selectedKeys().iterator(); i.hasNext();) {
- SelectionKey key = i.next();
- i.remove();
- SocketChannel sc = (SocketChannel) key.channel();
- if (pendingNegotiations.contains(sc)) {
- if (key.isReadable()) {
- ByteBuffer buffer = (ByteBuffer) key.attachment();
- sc.read(buffer);
- buffer.flip();
- if (buffer.remaining() >= INITIAL_MESSAGE_SIZE) {
- PartitionId pid = readInitialMessage(buffer);
- pendingNegotiations.remove(sc);
- key.interestOps(0);
- NetworkOutputChannel channel = new NetworkOutputChannel(ctx, 5);
- channel.setSelectionKey(key);
- key.attach(channel);
- try {
- partitionRequestListener.registerPartitionRequest(pid, channel);
- } catch (HyracksException e) {
- key.cancel();
- sc.close();
- }
- } else {
- buffer.compact();
- }
- }
- } else {
- INetworkChannel channel = (INetworkChannel) key.attachment();
- boolean close = false;
- try {
- close = channel.dispatchNetworkEvent();
- } catch (IOException e) {
- e.printStackTrace();
- close = true;
- }
- if (close) {
- key.cancel();
- sc.close();
- }
- }
- }
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
-
- private PartitionId readInitialMessage(ByteBuffer buffer) {
- JobId jobId = new JobId(buffer.getLong());
- ConnectorDescriptorId cdid = new ConnectorDescriptorId(buffer.getInt());
- int senderIndex = buffer.getInt();
- int receiverIndex = buffer.getInt();
- return new PartitionId(jobId, cdid, senderIndex, receiverIndex);
- }
- }
-
- public NetworkAddress getNetworkAddress() {
- return networkAddress;
- }
-}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/INetworkChannel.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/INetworkChannel.java
deleted file mode 100644
index 61cd91f..0000000
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/INetworkChannel.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package edu.uci.ics.hyracks.control.nc.net;
-
-import java.io.IOException;
-import java.net.SocketAddress;
-import java.nio.channels.SelectionKey;
-
-public interface INetworkChannel {
- public boolean dispatchNetworkEvent() throws IOException;
-
- public void setSelectionKey(SelectionKey key);
-
- public SelectionKey getSelectionKey();
-
- public SocketAddress getRemoteAddress();
-
- public void abort();
-
- public void notifyConnectionManagerRegistration() throws IOException;
-}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java
index 58f7088..609d7f0 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkInputChannel.java
@@ -14,11 +14,8 @@
*/
package edu.uci.ics.hyracks.control.nc.net;
-import java.io.IOException;
import java.net.SocketAddress;
import java.nio.ByteBuffer;
-import java.nio.channels.SelectionKey;
-import java.nio.channels.SocketChannel;
import java.util.ArrayDeque;
import java.util.Queue;
import java.util.logging.Level;
@@ -26,57 +23,47 @@
import edu.uci.ics.hyracks.api.channels.IInputChannel;
import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
-import edu.uci.ics.hyracks.api.comm.FrameHelper;
import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
-public class NetworkInputChannel implements IInputChannel, INetworkChannel {
+public class NetworkInputChannel implements IInputChannel {
private static final Logger LOGGER = Logger.getLogger(NetworkInputChannel.class.getName());
- private final ConnectionManager connectionManager;
+ private IHyracksRootContext ctx;
+
+ private final NetworkManager netManager;
private final SocketAddress remoteAddress;
private final PartitionId partitionId;
- private final Queue<ByteBuffer> emptyQueue;
-
private final Queue<ByteBuffer> fullQueue;
- private SocketChannel socketChannel;
+ private final int nBuffers;
- private SelectionKey key;
-
- private ByteBuffer currentBuffer;
-
- private boolean eos;
-
- private boolean aborted;
+ private ChannelControlBlock ccb;
private IInputChannelMonitor monitor;
private Object attachment;
- private ByteBuffer writeBuffer;
-
- public NetworkInputChannel(IHyracksRootContext ctx, ConnectionManager connectionManager,
- SocketAddress remoteAddress, PartitionId partitionId, int nBuffers) {
- this.connectionManager = connectionManager;
+ public NetworkInputChannel(IHyracksRootContext ctx, NetworkManager netManager, SocketAddress remoteAddress,
+ PartitionId partitionId, int nBuffers) {
+ this.ctx = ctx;
+ this.netManager = netManager;
this.remoteAddress = remoteAddress;
this.partitionId = partitionId;
- this.emptyQueue = new ArrayDeque<ByteBuffer>(nBuffers);
- for (int i = 0; i < nBuffers; ++i) {
- emptyQueue.add(ctx.allocateFrame());
- }
fullQueue = new ArrayDeque<ByteBuffer>(nBuffers);
- aborted = false;
- eos = false;
+ this.nBuffers = nBuffers;
}
@Override
- public void registerMonitor(IInputChannelMonitor monitor) throws HyracksException {
+ public void registerMonitor(IInputChannelMonitor monitor) {
this.monitor = monitor;
}
@@ -96,29 +83,34 @@
}
@Override
- public synchronized void recycleBuffer(ByteBuffer buffer) {
+ public void recycleBuffer(ByteBuffer buffer) {
buffer.clear();
- emptyQueue.add(buffer);
- if (!eos && !aborted) {
- int ops = key.interestOps();
- if ((ops & SelectionKey.OP_READ) == 0) {
- key.interestOps(ops | SelectionKey.OP_READ);
- key.selector().wakeup();
- if (currentBuffer == null) {
- currentBuffer = emptyQueue.poll();
- }
- }
- }
+ ccb.getReadInterface().getEmptyBufferAcceptor().accept(buffer);
}
@Override
public void open() throws HyracksDataException {
- currentBuffer = emptyQueue.poll();
try {
- connectionManager.connect(this);
- } catch (IOException e) {
+ ccb = netManager.connect(remoteAddress);
+ } catch (Exception e) {
throw new HyracksDataException(e);
}
+ ccb.getReadInterface().setFullBufferAcceptor(new ReadFullBufferAcceptor());
+ ccb.getWriteInterface().setEmptyBufferAcceptor(new WriteEmptyBufferAcceptor());
+ for (int i = 0; i < nBuffers; ++i) {
+ ccb.getReadInterface().getEmptyBufferAcceptor().accept(ctx.allocateFrame());
+ }
+ ByteBuffer writeBuffer = ByteBuffer.allocate(NetworkManager.INITIAL_MESSAGE_SIZE);
+ writeBuffer.putLong(partitionId.getJobId().getId());
+ writeBuffer.putInt(partitionId.getConnectorDescriptorId().getId());
+ writeBuffer.putInt(partitionId.getSenderIndex());
+ writeBuffer.putInt(partitionId.getReceiverIndex());
+ writeBuffer.flip();
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Sending partition request: " + partitionId + " on channel: " + ccb);
+ }
+ ccb.getWriteInterface().getFullBufferAcceptor().accept(writeBuffer);
+ ccb.getWriteInterface().getFullBufferAcceptor().close();
}
@Override
@@ -126,110 +118,28 @@
}
- @Override
- public synchronized boolean dispatchNetworkEvent() throws IOException {
- if (aborted) {
- eos = true;
- monitor.notifyEndOfStream(this);
- return true;
+ private class ReadFullBufferAcceptor implements ICloseableBufferAcceptor {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ fullQueue.add(buffer);
+ monitor.notifyDataAvailability(NetworkInputChannel.this, 1);
}
- if (key.isConnectable()) {
- if (socketChannel.finishConnect()) {
- key.interestOps(key.interestOps() & ~SelectionKey.OP_CONNECT);
- prepareForWrite();
- }
- } else if (key.isWritable()) {
- socketChannel.write(writeBuffer);
- if (writeBuffer.remaining() == 0) {
- key.interestOps(SelectionKey.OP_READ);
- }
- } else if (key.isReadable()) {
- if (LOGGER.isLoggable(Level.FINER)) {
- LOGGER.finer("Before read: " + currentBuffer.position() + " " + currentBuffer.limit());
- }
- int bytesRead = socketChannel.read(currentBuffer);
- if (bytesRead < 0) {
- eos = true;
- monitor.notifyEndOfStream(this);
- return true;
- }
- if (LOGGER.isLoggable(Level.FINER)) {
- LOGGER.finer("After read: " + currentBuffer.position() + " " + currentBuffer.limit());
- }
- currentBuffer.flip();
- int dataLen = currentBuffer.remaining();
- if (dataLen >= currentBuffer.capacity() || aborted()) {
- if (LOGGER.isLoggable(Level.FINEST)) {
- LOGGER.finest("NetworkInputChannel: frame received: sender = " + partitionId.getSenderIndex());
- }
- if (currentBuffer.getInt(FrameHelper.getTupleCountOffset(currentBuffer.capacity())) == 0) {
- eos = true;
- monitor.notifyEndOfStream(this);
- return true;
- }
- fullQueue.add(currentBuffer);
- currentBuffer = emptyQueue.poll();
- if (currentBuffer == null && key.isValid()) {
- int ops = key.interestOps();
- key.interestOps(ops & ~SelectionKey.OP_READ);
- }
- monitor.notifyDataAvailability(this, 1);
- return false;
- }
- currentBuffer.compact();
+
+ @Override
+ public void close() {
+ monitor.notifyEndOfStream(NetworkInputChannel.this);
}
- return false;
+
+ @Override
+ public void error(int ecode) {
+ monitor.notifyFailure(NetworkInputChannel.this);
+ }
}
- private void prepareForConnect() {
- key.interestOps(SelectionKey.OP_CONNECT);
- }
-
- private void prepareForWrite() {
- writeBuffer = ByteBuffer.allocate(ConnectionManager.INITIAL_MESSAGE_SIZE);
- writeBuffer.putLong(partitionId.getJobId().getId());
- writeBuffer.putInt(partitionId.getConnectorDescriptorId().getId());
- writeBuffer.putInt(partitionId.getSenderIndex());
- writeBuffer.putInt(partitionId.getReceiverIndex());
- writeBuffer.flip();
-
- key.interestOps(SelectionKey.OP_WRITE);
- }
-
- @Override
- public void setSelectionKey(SelectionKey key) {
- this.key = key;
- socketChannel = (SocketChannel) key.channel();
- }
-
- @Override
- public SocketAddress getRemoteAddress() {
- return remoteAddress;
- }
-
- @Override
- public SelectionKey getSelectionKey() {
- return key;
- }
-
- public PartitionId getPartitionId() {
- return partitionId;
- }
-
- public void abort() {
- aborted = true;
- }
-
- public boolean aborted() {
- return aborted;
- }
-
- @Override
- public void notifyConnectionManagerRegistration() throws IOException {
- if (socketChannel.connect(remoteAddress)) {
- prepareForWrite();
- } else {
- prepareForConnect();
+ private class WriteEmptyBufferAcceptor implements IBufferAcceptor {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ // do nothing
}
}
}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java
new file mode 100644
index 0000000..68e3120
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkManager.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.net;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.SocketAddress;
+import java.nio.ByteBuffer;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
+import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.nc.partitions.IPartitionRequestListener;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.IChannelOpenListener;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MultiplexedConnection;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MuxDemux;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MuxDemuxPerformanceCounters;
+
+public class NetworkManager {
+ private static final Logger LOGGER = Logger.getLogger(NetworkManager.class.getName());
+
+ static final int INITIAL_MESSAGE_SIZE = 20;
+
+ private final IHyracksRootContext ctx;
+
+ private final IPartitionRequestListener partitionRequestListener;
+
+ private final MuxDemux md;
+
+ private NetworkAddress networkAddress;
+
+ public NetworkManager(IHyracksRootContext ctx, InetAddress inetAddress,
+ IPartitionRequestListener partitionRequestListener, int nThreads) throws IOException {
+ this.ctx = ctx;
+ this.partitionRequestListener = partitionRequestListener;
+ md = new MuxDemux(new InetSocketAddress(inetAddress, 0), new ChannelOpenListener(), nThreads);
+ }
+
+ public void start() throws IOException {
+ md.start();
+ InetSocketAddress sockAddr = md.getLocalAddress();
+ networkAddress = new NetworkAddress(sockAddr.getAddress().getAddress(), sockAddr.getPort());
+ }
+
+ public NetworkAddress getNetworkAddress() {
+ return networkAddress;
+ }
+
+ public void stop() {
+
+ }
+
+ public ChannelControlBlock connect(SocketAddress remoteAddress) throws InterruptedException, NetException {
+ MultiplexedConnection mConn = md.connect((InetSocketAddress) remoteAddress);
+ return mConn.openChannel();
+ }
+
+ private class ChannelOpenListener implements IChannelOpenListener {
+ @Override
+ public void channelOpened(ChannelControlBlock channel) {
+ channel.getReadInterface().setFullBufferAcceptor(new InitialBufferAcceptor(channel));
+ channel.getReadInterface().getEmptyBufferAcceptor().accept(ByteBuffer.allocate(INITIAL_MESSAGE_SIZE));
+ }
+ }
+
+ private class InitialBufferAcceptor implements ICloseableBufferAcceptor {
+ private final ChannelControlBlock ccb;
+
+ private NetworkOutputChannel noc;
+
+ public InitialBufferAcceptor(ChannelControlBlock ccb) {
+ this.ccb = ccb;
+ }
+
+ @Override
+ public void accept(ByteBuffer buffer) {
+ PartitionId pid = readInitialMessage(buffer);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Received initial partition request: " + pid + " on channel: " + ccb);
+ }
+ noc = new NetworkOutputChannel(ctx, ccb, 5);
+ try {
+ partitionRequestListener.registerPartitionRequest(pid, noc);
+ } catch (HyracksException e) {
+ noc.abort();
+ }
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public void error(int ecode) {
+ if (noc != null) {
+ noc.abort();
+ }
+ }
+ }
+
+ private static PartitionId readInitialMessage(ByteBuffer buffer) {
+ JobId jobId = new JobId(buffer.getLong());
+ ConnectorDescriptorId cdid = new ConnectorDescriptorId(buffer.getInt());
+ int senderIndex = buffer.getInt();
+ int receiverIndex = buffer.getInt();
+ return new PartitionId(jobId, cdid, senderIndex, receiverIndex);
+ }
+
+ public MuxDemuxPerformanceCounters getPerformanceCounters() {
+ return md.getPerformanceCounters();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java
index 31ce924..185768e 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/net/NetworkOutputChannel.java
@@ -14,125 +14,45 @@
*/
package edu.uci.ics.hyracks.control.nc.net;
-import java.io.IOException;
-import java.net.SocketAddress;
import java.nio.ByteBuffer;
-import java.nio.channels.SelectionKey;
-import java.nio.channels.SocketChannel;
import java.util.ArrayDeque;
-import java.util.Queue;
+import java.util.Deque;
-import edu.uci.ics.hyracks.api.comm.FrameHelper;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
-public class NetworkOutputChannel implements INetworkChannel, IFrameWriter {
- private final IHyracksRootContext ctx;
+public class NetworkOutputChannel implements IFrameWriter {
+ private final ChannelControlBlock ccb;
- private final Queue<ByteBuffer> emptyQueue;
-
- private final Queue<ByteBuffer> fullQueue;
-
- private SelectionKey key;
+ private final Deque<ByteBuffer> emptyStack;
private boolean aborted;
- private boolean eos;
-
- private boolean eosSent;
-
- private boolean failed;
-
- private ByteBuffer currentBuffer;
-
- public NetworkOutputChannel(IHyracksRootContext ctx, int nBuffers) {
- this.ctx = ctx;
- emptyQueue = new ArrayDeque<ByteBuffer>(nBuffers);
+ public NetworkOutputChannel(IHyracksRootContext ctx, ChannelControlBlock ccb, int nBuffers) {
+ this.ccb = ccb;
+ emptyStack = new ArrayDeque<ByteBuffer>(nBuffers);
for (int i = 0; i < nBuffers; ++i) {
- emptyQueue.add(ctx.allocateFrame());
+ emptyStack.push(ByteBuffer.allocateDirect(ctx.getFrameSize()));
}
- fullQueue = new ArrayDeque<ByteBuffer>(nBuffers);
- }
-
- @Override
- public synchronized boolean dispatchNetworkEvent() throws IOException {
- if (failed || aborted) {
- eos = true;
- return true;
- } else if (key.isWritable()) {
- while (true) {
- if (currentBuffer == null) {
- if (eosSent) {
- return true;
- }
- currentBuffer = fullQueue.poll();
- if (currentBuffer == null) {
- if (eos) {
- currentBuffer = emptyQueue.poll();
- currentBuffer.clear();
- currentBuffer.putInt(FrameHelper.getTupleCountOffset(ctx.getFrameSize()), 0);
- eosSent = true;
- } else {
- key.interestOps(key.interestOps() & ~SelectionKey.OP_WRITE);
- return false;
- }
- }
- }
- int bytesWritten = ((SocketChannel) key.channel()).write(currentBuffer);
- if (bytesWritten < 0) {
- eos = true;
- return true;
- }
- if (currentBuffer.remaining() == 0) {
- emptyQueue.add(currentBuffer);
- notifyAll();
- currentBuffer = null;
- if (eosSent) {
- return true;
- }
- } else {
- return false;
- }
- }
- }
- return false;
- }
-
- @Override
- public void setSelectionKey(SelectionKey key) {
- this.key = key;
- }
-
- @Override
- public SelectionKey getSelectionKey() {
- return key;
- }
-
- @Override
- public SocketAddress getRemoteAddress() {
- return ((SocketChannel) key.channel()).socket().getRemoteSocketAddress();
- }
-
- @Override
- public synchronized void abort() {
- aborted = true;
+ ccb.getWriteInterface().setEmptyBufferAcceptor(new WriteEmptyBufferAcceptor());
}
@Override
public void open() throws HyracksDataException {
- currentBuffer = null;
}
@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
ByteBuffer destBuffer = null;
synchronized (this) {
- if (aborted) {
- throw new HyracksDataException("Connection has been aborted");
- }
while (true) {
- destBuffer = emptyQueue.poll();
+ if (aborted) {
+ throw new HyracksDataException("Connection has been aborted");
+ }
+ destBuffer = emptyStack.poll();
if (destBuffer != null) {
break;
}
@@ -148,26 +68,34 @@
destBuffer.clear();
destBuffer.put(buffer);
destBuffer.flip();
- synchronized (this) {
- fullQueue.add(destBuffer);
- }
- key.interestOps(SelectionKey.OP_WRITE);
- key.selector().wakeup();
+ ccb.getWriteInterface().getFullBufferAcceptor().accept(destBuffer);
}
@Override
public void fail() throws HyracksDataException {
- failed = true;
+ ccb.getWriteInterface().getFullBufferAcceptor().error(1);
}
@Override
- public synchronized void close() throws HyracksDataException {
- eos = true;
- key.interestOps(SelectionKey.OP_WRITE);
- key.selector().wakeup();
+ public void close() throws HyracksDataException {
+ ccb.getWriteInterface().getFullBufferAcceptor().close();
}
- @Override
- public void notifyConnectionManagerRegistration() throws IOException {
+ void abort() {
+ ccb.getWriteInterface().getFullBufferAcceptor().error(1);
+ synchronized (NetworkOutputChannel.this) {
+ aborted = true;
+ NetworkOutputChannel.this.notifyAll();
+ }
+ }
+
+ private class WriteEmptyBufferAcceptor implements IBufferAcceptor {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ synchronized (NetworkOutputChannel.this) {
+ emptyStack.push(buffer);
+ NetworkOutputChannel.this.notifyAll();
+ }
+ }
}
}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionInputChannel.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionInputChannel.java
index e579da7..0ea3124 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionInputChannel.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionInputChannel.java
@@ -23,7 +23,7 @@
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.partitions.IPartition;
import edu.uci.ics.hyracks.api.partitions.PartitionId;
public class MaterializedPartitionInputChannel implements IInputChannel {
@@ -54,7 +54,7 @@
}
@Override
- public void registerMonitor(IInputChannelMonitor monitor) throws HyracksException {
+ public void registerMonitor(IInputChannelMonitor monitor) {
this.monitor = monitor;
}
@@ -84,7 +84,7 @@
@Override
public void open() throws HyracksDataException {
- MaterializedPartition partition = (MaterializedPartition) manager.getPartition(pid);
+ IPartition partition = manager.getPartition(pid);
partition.writeTo(writer);
}
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionWriter.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionWriter.java
index b989805..1a1b15f 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionWriter.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializedPartitionWriter.java
@@ -33,15 +33,15 @@
public class MaterializedPartitionWriter implements IFrameWriter {
private static final Logger LOGGER = Logger.getLogger(MaterializedPartitionWriter.class.getName());
- protected final IHyracksRootContext ctx;
+ private final IHyracksRootContext ctx;
- protected final PartitionManager manager;
+ private final PartitionManager manager;
- protected final PartitionId pid;
+ private final PartitionId pid;
- protected final TaskAttemptId taId;
+ private final TaskAttemptId taId;
- protected final Executor executor;
+ private final Executor executor;
private FileReference fRef;
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
new file mode 100644
index 0000000..143fd2c
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/MaterializingPipelinedPartition.java
@@ -0,0 +1,175 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.partitions;
+
+import java.nio.ByteBuffer;
+import java.util.concurrent.Executor;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksRootContext;
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileHandle;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IIOManager;
+import edu.uci.ics.hyracks.api.partitions.IPartition;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.job.PartitionState;
+import edu.uci.ics.hyracks.control.nc.io.IOManager;
+
+public class MaterializingPipelinedPartition implements IFrameWriter, IPartition {
+ private static final Logger LOGGER = Logger.getLogger(MaterializingPipelinedPartition.class.getName());
+
+ private final IHyracksRootContext ctx;
+
+ private final Executor executor;
+
+ private final IOManager ioManager;
+
+ private final PartitionManager manager;
+
+ private final PartitionId pid;
+
+ private final TaskAttemptId taId;
+
+ private FileReference fRef;
+
+ private FileHandle handle;
+
+ private long size;
+
+ private boolean eos;
+
+ private boolean failed;
+
+ public MaterializingPipelinedPartition(IHyracksRootContext ctx, PartitionManager manager, PartitionId pid,
+ TaskAttemptId taId, Executor executor) {
+ this.ctx = ctx;
+ this.executor = executor;
+ this.ioManager = (IOManager) ctx.getIOManager();
+ this.manager = manager;
+ this.pid = pid;
+ this.taId = taId;
+ }
+
+ @Override
+ public void deallocate() {
+ fRef.delete();
+ }
+
+ @Override
+ public void writeTo(final IFrameWriter writer) {
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ FileHandle fh = ioManager.open(fRef, IIOManager.FileReadWriteMode.READ_ONLY,
+ IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
+ try {
+ writer.open();
+ try {
+ long offset = 0;
+ ByteBuffer buffer = ctx.allocateFrame();
+ boolean fail = false;
+ boolean done = false;
+ while (!fail && !done) {
+ synchronized (MaterializingPipelinedPartition.this) {
+ while (offset >= size && !eos && !failed) {
+ try {
+ MaterializingPipelinedPartition.this.wait();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ fail = failed;
+ done = eos && offset >= size;
+ }
+ if (fail) {
+ writer.fail();
+ } else if (!done) {
+ buffer.clear();
+ long readLen = ioManager.syncRead(fh, offset, buffer);
+ if (readLen < buffer.capacity()) {
+ throw new HyracksDataException("Premature end of file");
+ }
+ offset += readLen;
+ buffer.flip();
+ writer.nextFrame(buffer);
+ }
+ }
+ } finally {
+ writer.close();
+ }
+ } finally {
+ ioManager.close(fh);
+ }
+ } catch (HyracksDataException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+
+ @Override
+ public boolean isReusable() {
+ return true;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("open(" + pid + " by " + taId);
+ }
+ fRef = manager.getFileFactory().createUnmanagedWorkspaceFile(pid.toString());
+ handle = ctx.getIOManager().open(fRef, IIOManager.FileReadWriteMode.READ_WRITE,
+ IIOManager.FileSyncMode.METADATA_ASYNC_DATA_ASYNC);
+ size = 0;
+ eos = false;
+ failed = false;
+ manager.registerPartition(pid, taId, this, PartitionState.STARTED);
+ }
+
+ @Override
+ public synchronized void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ size += ctx.getIOManager().syncWrite(handle, size, buffer);
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void fail() throws HyracksDataException {
+ failed = true;
+ notifyAll();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("close(" + pid + " by " + taId);
+ }
+ boolean commit = false;
+ synchronized (this) {
+ eos = true;
+ ctx.getIOManager().close(handle);
+ handle = null;
+ commit = !failed;
+ notifyAll();
+ }
+ if (commit) {
+ manager.updatePartitionState(pid, taId, this, PartitionState.COMMITTED);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PartitionManager.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PartitionManager.java
index baa50a1..ce40fe6 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PartitionManager.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PartitionManager.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.control.nc.partitions;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -79,13 +80,13 @@
return partitionMap.get(pid).get(0);
}
- public synchronized void unregisterPartitions(JobId jobId) {
+ public synchronized void unregisterPartitions(JobId jobId, Collection<IPartition> unregisteredPartitions) {
for (Iterator<Map.Entry<PartitionId, List<IPartition>>> i = partitionMap.entrySet().iterator(); i.hasNext();) {
Map.Entry<PartitionId, List<IPartition>> e = i.next();
PartitionId pid = e.getKey();
if (jobId.equals(pid.getJobId())) {
for (IPartition p : e.getValue()) {
- p.deallocate();
+ unregisteredPartitions.add(p);
}
i.remove();
}
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java
index 9403736..92dc0b2 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/PipelinedPartition.java
@@ -32,6 +32,8 @@
private IFrameWriter delegate;
+ private boolean pendingConnection;
+
private boolean failed;
public PipelinedPartition(PartitionManager manager, PartitionId pid, TaskAttemptId taId) {
@@ -57,32 +59,46 @@
}
@Override
- public synchronized void open() throws HyracksDataException {
+ public void open() throws HyracksDataException {
manager.registerPartition(pid, taId, this, PartitionState.STARTED);
failed = false;
- while (delegate == null) {
- try {
- wait();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
- }
- }
- delegate.open();
+ pendingConnection = true;
}
@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ ensureConnected();
delegate.nextFrame(buffer);
}
+ private void ensureConnected() throws HyracksDataException {
+ if (pendingConnection) {
+ synchronized (this) {
+ while (delegate == null) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+ delegate.open();
+ }
+ pendingConnection = false;
+ }
+
@Override
public void fail() throws HyracksDataException {
+ ensureConnected();
failed = true;
- delegate.fail();
+ if (delegate != null) {
+ delegate.fail();
+ }
}
@Override
public void close() throws HyracksDataException {
+ ensureConnected();
if (!failed) {
manager.updatePartitionState(pid, taId, this, PartitionState.COMMITTED);
}
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java
index df76604..f93f874 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/partitions/ReceiveSideMaterializingCollector.java
@@ -136,7 +136,7 @@
try {
wait();
} catch (InterruptedException e) {
- e.printStackTrace();
+ throw new HyracksDataException(e);
}
}
}
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ConnectorSenderProfilingFrameWriter.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ConnectorSenderProfilingFrameWriter.java
index 5d592be..be7b319 100644
--- a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ConnectorSenderProfilingFrameWriter.java
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ConnectorSenderProfilingFrameWriter.java
@@ -31,12 +31,13 @@
public ConnectorSenderProfilingFrameWriter(IHyracksTaskContext ctx, IFrameWriter writer,
ConnectorDescriptorId cdId, int senderIndex, int receiverIndex) {
this.writer = writer;
+ int attempt = ctx.getTaskAttemptId().getAttempt();
this.openCounter = ctx.getCounterContext().getCounter(
- cdId + ".sender." + senderIndex + "." + receiverIndex + ".open", true);
+ cdId + ".sender." + attempt + "." + senderIndex + "." + receiverIndex + ".open", true);
this.closeCounter = ctx.getCounterContext().getCounter(
- cdId + ".sender." + senderIndex + "." + receiverIndex + ".close", true);
+ cdId + ".sender." + attempt + "." + senderIndex + "." + receiverIndex + ".close", true);
this.frameCounter = ctx.getCounterContext().getCounter(
- cdId + ".sender." + senderIndex + "." + receiverIndex + ".nextFrame", true);
+ cdId + ".sender." + attempt + "." + senderIndex + "." + receiverIndex + ".nextFrame", true);
}
@Override
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ProfilingPartitionWriterFactory.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ProfilingPartitionWriterFactory.java
new file mode 100644
index 0000000..affa01c
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/profiling/ProfilingPartitionWriterFactory.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.profiling;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.comm.IPartitionWriterFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.job.profiling.counters.MultiResolutionEventProfiler;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.PartitionProfile;
+import edu.uci.ics.hyracks.control.nc.Task;
+
+public class ProfilingPartitionWriterFactory implements IPartitionWriterFactory {
+ private static final int N_SAMPLES = 64;
+
+ private final IHyracksTaskContext ctx;
+
+ private final IConnectorDescriptor cd;
+
+ private final int senderIndex;
+
+ private final IPartitionWriterFactory delegate;
+
+ public ProfilingPartitionWriterFactory(IHyracksTaskContext ctx, IConnectorDescriptor cd, int senderIndex,
+ IPartitionWriterFactory delegate) {
+ this.ctx = ctx;
+ this.cd = cd;
+ this.senderIndex = senderIndex;
+ this.delegate = delegate;
+ }
+
+ @Override
+ public IFrameWriter createFrameWriter(final int receiverIndex) throws HyracksDataException {
+ final IFrameWriter writer = new ConnectorSenderProfilingFrameWriter(ctx,
+ delegate.createFrameWriter(receiverIndex), cd.getConnectorId(), senderIndex, receiverIndex);
+ return new IFrameWriter() {
+ private long openTime;
+
+ private long closeTime;
+
+ MultiResolutionEventProfiler mrep = new MultiResolutionEventProfiler(N_SAMPLES);
+
+ @Override
+ public void open() throws HyracksDataException {
+ openTime = System.currentTimeMillis();
+ writer.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ mrep.reportEvent();
+ writer.nextFrame(buffer);
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ closeTime = System.currentTimeMillis();
+ ((Task) ctx).setPartitionSendProfile(new PartitionProfile(new PartitionId(ctx.getJobletContext()
+ .getJobId(), cd.getConnectorId(), senderIndex, receiverIndex), openTime, closeTime, mrep));
+ writer.close();
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/AbortTasksWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/AbortTasksWork.java
new file mode 100644
index 0000000..e8c4052
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/AbortTasksWork.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.nc.Joblet;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+import edu.uci.ics.hyracks.control.nc.Task;
+
+public class AbortTasksWork extends SynchronizableWork {
+ private static final Logger LOGGER = Logger.getLogger(AbortTasksWork.class.getName());
+
+ private final NodeControllerService ncs;
+
+ private final JobId jobId;
+
+ private final List<TaskAttemptId> tasks;
+
+ public AbortTasksWork(NodeControllerService ncs, JobId jobId, List<TaskAttemptId> tasks) {
+ this.ncs = ncs;
+ this.jobId = jobId;
+ this.tasks = tasks;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Aborting Tasks: " + jobId + ":" + tasks);
+ }
+ Map<JobId, Joblet> jobletMap = ncs.getJobletMap();
+ Joblet ji = jobletMap.get(jobId);
+ if (ji != null) {
+ Map<TaskAttemptId, Task> taskMap = ji.getTaskMap();
+ for (TaskAttemptId taId : tasks) {
+ Task task = taskMap.get(taId);
+ if (task != null) {
+ task.abort();
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/BuildJobProfilesWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/BuildJobProfilesWork.java
new file mode 100644
index 0000000..574bc6d
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/BuildJobProfilesWork.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.JobProfile;
+import edu.uci.ics.hyracks.control.common.job.profiling.om.JobletProfile;
+import edu.uci.ics.hyracks.control.common.work.FutureValue;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.nc.Joblet;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+
+public class BuildJobProfilesWork extends SynchronizableWork {
+ private final NodeControllerService ncs;
+
+ private final FutureValue<List<JobProfile>> fv;
+
+ public BuildJobProfilesWork(NodeControllerService ncs, FutureValue<List<JobProfile>> fv) {
+ this.ncs = ncs;
+ this.fv = fv;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ List<JobProfile> profiles = new ArrayList<JobProfile>();
+ Map<JobId, Joblet> jobletMap = ncs.getJobletMap();
+ for (Joblet ji : jobletMap.values()) {
+ profiles.add(new JobProfile(ji.getJobId()));
+ }
+ for (JobProfile jProfile : profiles) {
+ Joblet ji;
+ JobletProfile jobletProfile = new JobletProfile(ncs.getId());
+ ji = jobletMap.get(jProfile.getJobId());
+ if (ji != null) {
+ ji.dumpProfile(jobletProfile);
+ jProfile.getJobletProfiles().put(ncs.getId(), jobletProfile);
+ }
+ }
+ fv.setValue(profiles);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java
new file mode 100644
index 0000000..b75a1fc
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CleanupJobletWork.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobStatus;
+import edu.uci.ics.hyracks.api.partitions.IPartition;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.nc.Joblet;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+
+public class CleanupJobletWork extends SynchronizableWork {
+ private static final Logger LOGGER = Logger.getLogger(CleanupJobletWork.class.getName());
+
+ private final NodeControllerService ncs;
+
+ private final JobId jobId;
+
+ private JobStatus status;
+
+ public CleanupJobletWork(NodeControllerService ncs, JobId jobId, JobStatus status) {
+ this.ncs = ncs;
+ this.jobId = jobId;
+ this.status = status;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Cleaning up after job: " + jobId);
+ }
+ final List<IPartition> unregisteredPartitions = new ArrayList<IPartition>();
+ ncs.getPartitionManager().unregisterPartitions(jobId, unregisteredPartitions);
+ ncs.getExecutor().execute(new Runnable() {
+ @Override
+ public void run() {
+ for (IPartition p : unregisteredPartitions) {
+ p.deallocate();
+ }
+ }
+ });
+ Map<JobId, Joblet> jobletMap = ncs.getJobletMap();
+ Joblet joblet = jobletMap.remove(jobId);
+ if (joblet != null) {
+ joblet.cleanup(status);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java
new file mode 100644
index 0000000..eb982df
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/CreateApplicationWork.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Serializable;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.client.HttpClient;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.impl.client.DefaultHttpClient;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
+import edu.uci.ics.hyracks.control.common.controllers.NodeParameters;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+import edu.uci.ics.hyracks.control.nc.application.NCApplicationContext;
+
+public class CreateApplicationWork extends SynchronizableWork {
+ private static final Logger LOGGER = Logger.getLogger(CreateApplicationWork.class.getName());
+
+ private final NodeControllerService ncs;
+
+ private final String appName;
+
+ private final boolean deployHar;
+
+ private final byte[] serializedDistributedState;
+
+ public CreateApplicationWork(NodeControllerService ncs, String appName, boolean deployHar,
+ byte[] serializedDistributedState) {
+ this.ncs = ncs;
+ this.appName = appName;
+ this.deployHar = deployHar;
+ this.serializedDistributedState = serializedDistributedState;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ try {
+ NCApplicationContext appCtx;
+ Map<String, NCApplicationContext> applications = ncs.getApplications();
+ if (applications.containsKey(appName)) {
+ throw new HyracksException("Duplicate application with name: " + appName + " being created.");
+ }
+ appCtx = new NCApplicationContext(ncs.getServerContext(), ncs.getRootContext(), appName, ncs.getId());
+ applications.put(appName, appCtx);
+ if (deployHar) {
+ NCConfig ncConfig = ncs.getConfiguration();
+ NodeParameters nodeParameters = ncs.getNodeParameters();
+ HttpClient hc = new DefaultHttpClient();
+ HttpGet get = new HttpGet("http://" + ncConfig.ccHost + ":"
+ + nodeParameters.getClusterControllerInfo().getWebPort() + "/applications/" + appName);
+ HttpResponse response = hc.execute(get);
+ InputStream is = response.getEntity().getContent();
+ OutputStream os = appCtx.getHarOutputStream();
+ try {
+ IOUtils.copyLarge(is, os);
+ } finally {
+ os.close();
+ is.close();
+ }
+ }
+ appCtx.initializeClassPath();
+ appCtx.setDistributedState((Serializable) appCtx.deserialize(serializedDistributedState));
+ appCtx.initialize();
+ ncs.getClusterController()
+ .notifyApplicationStateChange(ncs.getId(), appName, ApplicationStatus.INITIALIZED);
+ } catch (Exception e) {
+ LOGGER.warning("Error creating application: " + e.getMessage());
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java
new file mode 100644
index 0000000..cfe00f6
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/DestroyApplicationWork.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import java.util.Map;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.control.common.application.ApplicationContext;
+import edu.uci.ics.hyracks.control.common.application.ApplicationStatus;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+import edu.uci.ics.hyracks.control.nc.application.NCApplicationContext;
+
+public class DestroyApplicationWork extends SynchronizableWork {
+ private static final Logger LOGGER = Logger.getLogger(DestroyApplicationWork.class.getName());
+
+ private final NodeControllerService ncs;
+
+ private final String appName;
+
+ public DestroyApplicationWork(NodeControllerService ncs, String appName) {
+ this.ncs = ncs;
+ this.appName = appName;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ try {
+ Map<String, NCApplicationContext> applications = ncs.getApplications();
+ ApplicationContext appCtx = applications.remove(appName);
+ if (appCtx != null) {
+ appCtx.deinitialize();
+ }
+ } catch (Exception e) {
+ LOGGER.warning("Error destroying application: " + e.getMessage());
+ }
+ ncs.getClusterController().notifyApplicationStateChange(ncs.getId(), appName, ApplicationStatus.DEINITIALIZED);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/NotifyTaskCompleteWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/NotifyTaskCompleteWork.java
new file mode 100644
index 0000000..022c92f
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/NotifyTaskCompleteWork.java
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import edu.uci.ics.hyracks.control.common.job.profiling.om.TaskProfile;
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+import edu.uci.ics.hyracks.control.nc.Task;
+
+public class NotifyTaskCompleteWork extends AbstractWork {
+ private final NodeControllerService ncs;
+ private final Task task;
+
+ public NotifyTaskCompleteWork(NodeControllerService ncs, Task task) {
+ this.ncs = ncs;
+ this.task = task;
+ }
+
+ @Override
+ public void run() {
+ TaskProfile taskProfile = new TaskProfile(task.getTaskAttemptId(), task.getPartitionSendProfile());
+ task.dumpProfile(taskProfile);
+ try {
+ ncs.getClusterController().notifyTaskComplete(task.getJobletContext().getJobId(), task.getTaskAttemptId(),
+ ncs.getId(), taskProfile);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ task.getJoblet().removeTask(task);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/NotifyTaskFailureWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/NotifyTaskFailureWork.java
new file mode 100644
index 0000000..3957934
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/NotifyTaskFailureWork.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import edu.uci.ics.hyracks.control.common.work.AbstractWork;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+import edu.uci.ics.hyracks.control.nc.Task;
+
+public class NotifyTaskFailureWork extends AbstractWork {
+ private final NodeControllerService ncs;
+ private final Task task;
+ private final String details;
+
+ public NotifyTaskFailureWork(NodeControllerService ncs, Task task, String details) {
+ this.ncs = ncs;
+ this.task = task;
+ this.details = details;
+ }
+
+ @Override
+ public void run() {
+ try {
+ ncs.getClusterController().notifyTaskFailure(task.getJobletContext().getJobId(), task.getTaskAttemptId(),
+ ncs.getId(), details);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ task.getJoblet().removeTask(task);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java
new file mode 100644
index 0000000..9734567
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/ReportPartitionAvailabilityWork.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.util.Map;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.comm.NetworkAddress;
+import edu.uci.ics.hyracks.api.comm.PartitionChannel;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.nc.Joblet;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+import edu.uci.ics.hyracks.control.nc.net.NetworkInputChannel;
+
+public class ReportPartitionAvailabilityWork extends SynchronizableWork {
+ private static final Logger LOGGER = Logger.getLogger(ReportPartitionAvailabilityWork.class.getName());
+
+ private final NodeControllerService ncs;
+
+ private final PartitionId pid;
+
+ private final NetworkAddress networkAddress;
+
+ public ReportPartitionAvailabilityWork(NodeControllerService ncs, PartitionId pid, NetworkAddress networkAddress) {
+ this.ncs = ncs;
+ this.pid = pid;
+ this.networkAddress = networkAddress;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ Map<JobId, Joblet> jobletMap = ncs.getJobletMap();
+ Joblet ji = jobletMap.get(pid.getJobId());
+ if (ji != null) {
+ PartitionChannel channel = new PartitionChannel(pid, new NetworkInputChannel(ncs.getRootContext(),
+ ncs.getNetworkManager(), new InetSocketAddress(InetAddress.getByAddress(networkAddress
+ .getIpAddress()), networkAddress.getPort()), pid, 5));
+ ji.reportPartitionAvailability(channel);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/StartTasksWork.java b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/StartTasksWork.java
new file mode 100644
index 0000000..a101612
--- /dev/null
+++ b/hyracks-control-nc/src/main/java/edu/uci/ics/hyracks/control/nc/work/StartTasksWork.java
@@ -0,0 +1,233 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.control.nc.work;
+
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.application.INCApplicationContext;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.comm.IPartitionCollector;
+import edu.uci.ics.hyracks.api.comm.IPartitionWriterFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.IActivity;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.OperatorDescriptorId;
+import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
+import edu.uci.ics.hyracks.api.dataflow.connectors.IConnectorPolicy;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.IJobletEventListener;
+import edu.uci.ics.hyracks.api.job.IJobletEventListenerFactory;
+import edu.uci.ics.hyracks.api.job.JobActivityGraph;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+import edu.uci.ics.hyracks.control.common.job.TaskAttemptDescriptor;
+import edu.uci.ics.hyracks.control.common.work.SynchronizableWork;
+import edu.uci.ics.hyracks.control.nc.Joblet;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+import edu.uci.ics.hyracks.control.nc.Task;
+import edu.uci.ics.hyracks.control.nc.application.NCApplicationContext;
+import edu.uci.ics.hyracks.control.nc.partitions.MaterializedPartitionWriter;
+import edu.uci.ics.hyracks.control.nc.partitions.MaterializingPipelinedPartition;
+import edu.uci.ics.hyracks.control.nc.partitions.PipelinedPartition;
+import edu.uci.ics.hyracks.control.nc.partitions.ReceiveSideMaterializingCollector;
+import edu.uci.ics.hyracks.control.nc.profiling.ProfilingPartitionWriterFactory;
+
+public class StartTasksWork extends SynchronizableWork {
+ private static final Logger LOGGER = Logger.getLogger(StartTasksWork.class.getName());
+
+ private final NodeControllerService ncs;
+
+ private final String appName;
+
+ private final JobId jobId;
+
+ private final byte[] jagBytes;
+
+ private final List<TaskAttemptDescriptor> taskDescriptors;
+
+ private final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPoliciesMap;
+
+ public StartTasksWork(NodeControllerService ncs, String appName, JobId jobId, byte[] jagBytes,
+ List<TaskAttemptDescriptor> taskDescriptors,
+ Map<ConnectorDescriptorId, IConnectorPolicy> connectorPoliciesMap) {
+ this.ncs = ncs;
+ this.appName = appName;
+ this.jobId = jobId;
+ this.jagBytes = jagBytes;
+ this.taskDescriptors = taskDescriptors;
+ this.connectorPoliciesMap = connectorPoliciesMap;
+ }
+
+ @Override
+ protected void doRun() throws Exception {
+ try {
+ Map<String, NCApplicationContext> applications = ncs.getApplications();
+ NCApplicationContext appCtx = applications.get(appName);
+ final Joblet joblet = getOrCreateLocalJoblet(jobId, appCtx, jagBytes == null ? null
+ : (JobActivityGraph) appCtx.deserialize(jagBytes));
+ final JobActivityGraph jag = joblet.getJobActivityGraph();
+
+ IRecordDescriptorProvider rdp = new IRecordDescriptorProvider() {
+ @Override
+ public RecordDescriptor getOutputRecordDescriptor(OperatorDescriptorId opId, int outputIndex) {
+ return jag.getJobSpecification().getOperatorOutputRecordDescriptor(opId, outputIndex);
+ }
+
+ @Override
+ public RecordDescriptor getInputRecordDescriptor(OperatorDescriptorId opId, int inputIndex) {
+ return jag.getJobSpecification().getOperatorInputRecordDescriptor(opId, inputIndex);
+ }
+ };
+
+ for (TaskAttemptDescriptor td : taskDescriptors) {
+ TaskAttemptId taId = td.getTaskAttemptId();
+ TaskId tid = taId.getTaskId();
+ IActivity han = jag.getActivityNodeMap().get(tid.getActivityId());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Initializing " + taId + " -> " + han);
+ }
+ final int partition = tid.getPartition();
+ Task task = new Task(joblet, taId, han.getClass().getName(), ncs.getExecutor());
+ IOperatorNodePushable operator = han.createPushRuntime(task, rdp, partition, td.getPartitionCount());
+
+ List<IPartitionCollector> collectors = new ArrayList<IPartitionCollector>();
+
+ List<IConnectorDescriptor> inputs = jag.getActivityInputConnectorDescriptors(tid.getActivityId());
+ if (inputs != null) {
+ for (int i = 0; i < inputs.size(); ++i) {
+ IConnectorDescriptor conn = inputs.get(i);
+ IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("input: " + i + ": " + conn.getConnectorId());
+ }
+ RecordDescriptor recordDesc = jag.getJobSpecification().getConnectorRecordDescriptor(conn);
+ IPartitionCollector collector = createPartitionCollector(td, partition, task, i, conn,
+ recordDesc, cPolicy);
+ collectors.add(collector);
+ }
+ }
+ List<IConnectorDescriptor> outputs = jag.getActivityOutputConnectorDescriptors(tid.getActivityId());
+ if (outputs != null) {
+ for (int i = 0; i < outputs.size(); ++i) {
+ final IConnectorDescriptor conn = outputs.get(i);
+ RecordDescriptor recordDesc = jag.getJobSpecification().getConnectorRecordDescriptor(conn);
+ IConnectorPolicy cPolicy = connectorPoliciesMap.get(conn.getConnectorId());
+
+ IPartitionWriterFactory pwFactory = createPartitionWriterFactory(task, cPolicy, jobId, conn,
+ partition, taId, jag.getJobFlags());
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("output: " + i + ": " + conn.getConnectorId());
+ }
+ IFrameWriter writer = conn.createPartitioner(task, recordDesc, pwFactory, partition,
+ td.getPartitionCount(), td.getOutputPartitionCounts()[i]);
+ operator.setOutputFrameWriter(i, writer, recordDesc);
+ }
+ }
+
+ task.setTaskRuntime(collectors.toArray(new IPartitionCollector[collectors.size()]), operator);
+ joblet.addTask(task);
+
+ task.start();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ private Joblet getOrCreateLocalJoblet(JobId jobId, INCApplicationContext appCtx, JobActivityGraph jag)
+ throws Exception {
+ Map<JobId, Joblet> jobletMap = ncs.getJobletMap();
+ Joblet ji = jobletMap.get(jobId);
+ if (ji == null) {
+ if (jag == null) {
+ throw new NullPointerException("JobActivityGraph was null");
+ }
+ ji = new Joblet(ncs, jobId, appCtx, jag);
+ IJobletEventListenerFactory jelf = jag.getJobSpecification().getJobletEventListenerFactory();
+ if (jelf != null) {
+ IJobletEventListener listener = jelf.createListener(ji);
+ ji.setJobletEventListener(listener);
+ listener.jobletStart();
+ }
+ jobletMap.put(jobId, ji);
+ }
+ return ji;
+ }
+
+ private IPartitionCollector createPartitionCollector(TaskAttemptDescriptor td, final int partition, Task task,
+ int i, IConnectorDescriptor conn, RecordDescriptor recordDesc, IConnectorPolicy cPolicy)
+ throws HyracksDataException {
+ IPartitionCollector collector = conn.createPartitionCollector(task, recordDesc, partition,
+ td.getInputPartitionCounts()[i], td.getPartitionCount());
+ if (cPolicy.materializeOnReceiveSide()) {
+ return new ReceiveSideMaterializingCollector(ncs.getRootContext(), ncs.getPartitionManager(), collector,
+ task.getTaskAttemptId(), ncs.getExecutor());
+ } else {
+ return collector;
+ }
+ }
+
+ private IPartitionWriterFactory createPartitionWriterFactory(IHyracksTaskContext ctx, IConnectorPolicy cPolicy,
+ final JobId jobId, final IConnectorDescriptor conn, final int senderIndex, final TaskAttemptId taId,
+ EnumSet<JobFlag> flags) {
+ IPartitionWriterFactory factory;
+ if (cPolicy.materializeOnSendSide()) {
+ if (cPolicy.consumerWaitsForProducerToFinish()) {
+ factory = new IPartitionWriterFactory() {
+ @Override
+ public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException {
+ return new MaterializedPartitionWriter(ncs.getRootContext(), ncs.getPartitionManager(),
+ new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId,
+ ncs.getExecutor());
+ }
+ };
+ } else {
+ factory = new IPartitionWriterFactory() {
+ @Override
+ public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException {
+ return new MaterializingPipelinedPartition(ncs.getRootContext(), ncs.getPartitionManager(),
+ new PartitionId(jobId, conn.getConnectorId(), senderIndex, receiverIndex), taId,
+ ncs.getExecutor());
+ }
+ };
+ }
+ } else {
+ factory = new IPartitionWriterFactory() {
+ @Override
+ public IFrameWriter createFrameWriter(int receiverIndex) throws HyracksDataException {
+ return new PipelinedPartition(ncs.getPartitionManager(), new PartitionId(jobId,
+ conn.getConnectorId(), senderIndex, receiverIndex), taId);
+ }
+ };
+ }
+ if (flags.contains(JobFlag.PROFILE_RUNTIME)) {
+ factory = new ProfilingPartitionWriterFactory(ctx, conn, senderIndex, factory);
+ }
+ return factory;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/pom.xml b/hyracks-data/hyracks-data-std/pom.xml
new file mode 100644
index 0000000..2bc5d40
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/pom.xml
@@ -0,0 +1,31 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-data-std</artifactId>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-api</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/PointableBinaryComparatorFactory.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/PointableBinaryComparatorFactory.java
new file mode 100644
index 0000000..bab393e
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/PointableBinaryComparatorFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.accessors;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public class PointableBinaryComparatorFactory implements IBinaryComparatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final IPointableFactory pf;
+
+ public static PointableBinaryComparatorFactory of(IPointableFactory pf) {
+ return new PointableBinaryComparatorFactory(pf);
+ }
+
+ public PointableBinaryComparatorFactory(IPointableFactory pf) {
+ this.pf = pf;
+ }
+
+ @Override
+ public IBinaryComparator createBinaryComparator() {
+ final IPointable p = pf.createPointable();
+ return new IBinaryComparator() {
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ p.set(b1, s1, l1);
+ return ((IComparable) p).compareTo(b2, s2, l2);
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/PointableBinaryHashFunctionFactory.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/PointableBinaryHashFunctionFactory.java
new file mode 100644
index 0000000..fddbe3d
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/PointableBinaryHashFunctionFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.accessors;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.api.IHashable;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public class PointableBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final IPointableFactory pf;
+
+ public static PointableBinaryHashFunctionFactory of(IPointableFactory pf) {
+ return new PointableBinaryHashFunctionFactory(pf);
+ }
+
+ public PointableBinaryHashFunctionFactory(IPointableFactory pf) {
+ this.pf = pf;
+ }
+
+ @Override
+ public IBinaryHashFunction createBinaryHashFunction() {
+ final IPointable p = pf.createPointable();
+ return new IBinaryHashFunction() {
+ @Override
+ public int hash(byte[] bytes, int offset, int length) {
+ p.set(bytes, offset, length);
+ return ((IHashable) p).hash();
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/UTF8StringBinaryHashFunctionFamily.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/UTF8StringBinaryHashFunctionFamily.java
new file mode 100644
index 0000000..e5beadf
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/accessors/UTF8StringBinaryHashFunctionFamily.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.accessors;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+
+public class UTF8StringBinaryHashFunctionFamily implements IBinaryHashFunctionFamily {
+ public static final IBinaryHashFunctionFamily INSTANCE = new UTF8StringBinaryHashFunctionFamily();
+
+ private static final long serialVersionUID = 1L;
+
+ static final int[] primeCoefficents = { 31, 23, 53, 97, 71, 337, 11, 877, 3, 29 };
+
+ private UTF8StringBinaryHashFunctionFamily() {
+ }
+
+ @Override
+ public IBinaryHashFunction createBinaryHashFunction(int seed) {
+ final int coefficient = primeCoefficents[seed % primeCoefficents.length];
+ final int r = primeCoefficents[(seed + 1) % primeCoefficents.length];
+
+ return new IBinaryHashFunction() {
+ @Override
+ public int hash(byte[] bytes, int offset, int length) {
+ int h = 0;
+ int utflen = UTF8StringPointable.getUTFLen(bytes, offset);
+ int sStart = offset + 2;
+ int c = 0;
+
+ while (c < utflen) {
+ char ch = UTF8StringPointable.charAt(bytes, sStart + c);
+ h = (coefficient * h + ch) % r;
+ c += UTF8StringPointable.charSize(bytes, sStart + c);
+ }
+ return h;
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/AbstractPointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/AbstractPointable.java
new file mode 100644
index 0000000..2d0d87c
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/AbstractPointable.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.api;
+
+public abstract class AbstractPointable implements IPointable {
+ protected byte[] bytes;
+
+ protected int start;
+
+ protected int length;
+
+ @Override
+ public void set(byte[] bytes, int start, int length) {
+ this.bytes = bytes;
+ this.start = start;
+ this.length = length;
+ }
+
+ @Override
+ public void set(IPointable pointer) {
+ set(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public byte[] getByteArray() {
+ return bytes;
+ }
+
+ @Override
+ public int getStartOffset() {
+ return start;
+ }
+
+ @Override
+ public int getLength() {
+ return length;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IComparable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IComparable.java
new file mode 100644
index 0000000..d68f356
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IComparable.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.api;
+
+public interface IComparable {
+ public int compareTo(IPointable pointer);
+
+ public int compareTo(byte[] bytes, int start, int length);
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IHashable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IHashable.java
new file mode 100644
index 0000000..3385298
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IHashable.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.api;
+
+public interface IHashable {
+ public int hash();
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/INumeric.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/INumeric.java
new file mode 100644
index 0000000..72fc221
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/INumeric.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.api;
+
+public interface INumeric {
+ public byte byteValue();
+
+ public short shortValue();
+
+ public int intValue();
+
+ public long longValue();
+
+ public float floatValue();
+
+ public double doubleValue();
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointable.java
new file mode 100644
index 0000000..17e9b03
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointable.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.api;
+
+public interface IPointable {
+ public void set(byte[] bytes, int start, int length);
+
+ public void set(IPointable pointer);
+
+ public byte[] getByteArray();
+
+ public int getStartOffset();
+
+ public int getLength();
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointableFactory.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointableFactory.java
new file mode 100644
index 0000000..41895a1
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/api/IPointableFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.api;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+
+public interface IPointableFactory extends Serializable {
+ public IPointable createPointable();
+
+ public ITypeTraits getTypeTraits();
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/BooleanPointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/BooleanPointable.java
new file mode 100644
index 0000000..7a10a01
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/BooleanPointable.java
@@ -0,0 +1,85 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.AbstractPointable;
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.api.IHashable;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public final class BooleanPointable extends AbstractPointable implements IHashable, IComparable {
+ public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return 1;
+ }
+ };
+
+ public static final IPointableFactory FACTORY = new IPointableFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IPointable createPointable() {
+ return new BooleanPointable();
+ }
+
+ @Override
+ public ITypeTraits getTypeTraits() {
+ return TYPE_TRAITS;
+ }
+ };
+
+ public static boolean getBoolean(byte[] bytes, int start) {
+ return bytes[start] == 0 ? false : true;
+ }
+
+ public static void setBoolean(byte[] bytes, int start, boolean value) {
+ bytes[start] = (byte) (value ? 1 : 0);
+ }
+
+ public boolean getBoolean() {
+ return getBoolean(bytes, start);
+ }
+
+ public void setBoolean(boolean value) {
+ setBoolean(bytes, start, value);
+ }
+
+ @Override
+ public int compareTo(IPointable pointer) {
+ return compareTo(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public int compareTo(byte[] bytes, int start, int length) {
+ boolean b = getBoolean();
+ boolean ob = getBoolean(bytes, start);
+ return b == ob ? 0 : (b ? 1 : -1);
+ }
+
+ @Override
+ public int hash() {
+ return getBoolean() ? Boolean.TRUE.hashCode() : Boolean.FALSE.hashCode();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/BytePointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/BytePointable.java
new file mode 100644
index 0000000..a157d7e
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/BytePointable.java
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.AbstractPointable;
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.api.IHashable;
+import edu.uci.ics.hyracks.data.std.api.INumeric;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public final class BytePointable extends AbstractPointable implements IHashable, IComparable, INumeric {
+ public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return 1;
+ }
+ };
+
+ public static final IPointableFactory FACTORY = new IPointableFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IPointable createPointable() {
+ return new BytePointable();
+ }
+
+ @Override
+ public ITypeTraits getTypeTraits() {
+ return TYPE_TRAITS;
+ }
+ };
+
+ public static byte getByte(byte[] bytes, int start) {
+ return bytes[start];
+ }
+
+ public static void setByte(byte[] bytes, int start, byte value) {
+ bytes[start] = value;
+ }
+
+ public byte getByte() {
+ return getByte(bytes, start);
+ }
+
+ public void setByte(byte value) {
+ setByte(bytes, start, value);
+ }
+
+ public byte preIncrement() {
+ byte v = getByte();
+ ++v;
+ setByte(v);
+ return v;
+ }
+
+ public byte postIncrement() {
+ byte v = getByte();
+ byte ov = v++;
+ setByte(v);
+ return ov;
+ }
+
+ @Override
+ public int compareTo(IPointable pointer) {
+ return compareTo(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public int compareTo(byte[] bytes, int start, int length) {
+ byte b = getByte();
+ byte ob = getByte(bytes, start);
+ return b < ob ? -1 : (b > ob ? 1 : 0);
+ }
+
+ @Override
+ public int hash() {
+ return getByte();
+ }
+
+ @Override
+ public byte byteValue() {
+ return getByte();
+ }
+
+ @Override
+ public short shortValue() {
+ return getByte();
+ }
+
+ @Override
+ public int intValue() {
+ return getByte();
+ }
+
+ @Override
+ public long longValue() {
+ return getByte();
+ }
+
+ @Override
+ public float floatValue() {
+ return getByte();
+ }
+
+ @Override
+ public double doubleValue() {
+ return getByte();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/DoublePointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/DoublePointable.java
new file mode 100644
index 0000000..5267086
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/DoublePointable.java
@@ -0,0 +1,137 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.AbstractPointable;
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.api.IHashable;
+import edu.uci.ics.hyracks.data.std.api.INumeric;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public final class DoublePointable extends AbstractPointable implements IHashable, IComparable, INumeric {
+ public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return 8;
+ }
+ };
+
+ public static final IPointableFactory FACTORY = new IPointableFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IPointable createPointable() {
+ return new DoublePointable();
+ }
+
+ @Override
+ public ITypeTraits getTypeTraits() {
+ return TYPE_TRAITS;
+ }
+ };
+
+ public static long getLongBits(byte[] bytes, int start) {
+ return LongPointable.getLong(bytes, start);
+ }
+
+ public static double getDouble(byte[] bytes, int start) {
+ long bits = getLongBits(bytes, start);
+ return Double.longBitsToDouble(bits);
+ }
+
+ public static void setDouble(byte[] bytes, int start, double value) {
+ long bits = Double.doubleToLongBits(value);
+ LongPointable.setLong(bytes, start, bits);
+ }
+
+ public double getDouble() {
+ return getDouble(bytes, start);
+ }
+
+ public void setDouble(double value) {
+ setDouble(bytes, start, value);
+ }
+
+ public double preIncrement() {
+ double v = getDouble();
+ ++v;
+ setDouble(v);
+ return v;
+ }
+
+ public double postIncrement() {
+ double v = getDouble();
+ double ov = v++;
+ setDouble(v);
+ return ov;
+ }
+
+ @Override
+ public int compareTo(IPointable pointer) {
+ return compareTo(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public int compareTo(byte[] bytes, int start, int length) {
+ double v = getDouble();
+ double ov = getDouble(bytes, start);
+ return v < ov ? -1 : (v > ov ? 1 : 0);
+ }
+
+ @Override
+ public int hash() {
+ long bits = getLongBits(bytes, start);
+ return (int) (bits ^ (bits >>> 32));
+ }
+
+ @Override
+ public byte byteValue() {
+ return (byte) getDouble();
+ }
+
+ @Override
+ public short shortValue() {
+ return (short) getDouble();
+ }
+
+ @Override
+ public int intValue() {
+ return (int) getDouble();
+ }
+
+ @Override
+ public long longValue() {
+ return (long) getDouble();
+ }
+
+ @Override
+ public float floatValue() {
+ return (float) getDouble();
+ }
+
+ @Override
+ public double doubleValue() {
+ return getDouble();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/FloatPointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/FloatPointable.java
new file mode 100644
index 0000000..c751f8b
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/FloatPointable.java
@@ -0,0 +1,136 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.AbstractPointable;
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.api.IHashable;
+import edu.uci.ics.hyracks.data.std.api.INumeric;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public final class FloatPointable extends AbstractPointable implements IHashable, IComparable, INumeric {
+ public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return 4;
+ }
+ };
+
+ public static final IPointableFactory FACTORY = new IPointableFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IPointable createPointable() {
+ return new FloatPointable();
+ }
+
+ @Override
+ public ITypeTraits getTypeTraits() {
+ return TYPE_TRAITS;
+ }
+ };
+
+ private static int getIntBits(byte[] bytes, int start) {
+ return IntegerPointable.getInteger(bytes, start);
+ }
+
+ public static float getFloat(byte[] bytes, int start) {
+ int bits = getIntBits(bytes, start);
+ return Float.intBitsToFloat(bits);
+ }
+
+ public static void setFloat(byte[] bytes, int start, float value) {
+ int bits = Float.floatToIntBits(value);
+ IntegerPointable.setInteger(bytes, start, bits);
+ }
+
+ public float getFloat() {
+ return getFloat(bytes, start);
+ }
+
+ public void setFloat(float value) {
+ setFloat(bytes, start, value);
+ }
+
+ public float preIncrement() {
+ float v = getFloat();
+ ++v;
+ setFloat(v);
+ return v;
+ }
+
+ public float postIncrement() {
+ float v = getFloat();
+ float ov = v++;
+ setFloat(v);
+ return ov;
+ }
+
+ @Override
+ public int compareTo(IPointable pointer) {
+ return compareTo(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public int compareTo(byte[] bytes, int start, int length) {
+ float v = getFloat();
+ float ov = getFloat(bytes, start);
+ return v < ov ? -1 : (v > ov ? 1 : 0);
+ }
+
+ @Override
+ public int hash() {
+ return getIntBits(bytes, start);
+ }
+
+ @Override
+ public byte byteValue() {
+ return (byte) getFloat();
+ }
+
+ @Override
+ public short shortValue() {
+ return (short) getFloat();
+ }
+
+ @Override
+ public int intValue() {
+ return (int) getFloat();
+ }
+
+ @Override
+ public long longValue() {
+ return (long) getFloat();
+ }
+
+ @Override
+ public float floatValue() {
+ return getFloat();
+ }
+
+ @Override
+ public double doubleValue() {
+ return getFloat();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/IntegerPointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/IntegerPointable.java
new file mode 100644
index 0000000..f18ae09
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/IntegerPointable.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.AbstractPointable;
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.api.IHashable;
+import edu.uci.ics.hyracks.data.std.api.INumeric;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public final class IntegerPointable extends AbstractPointable implements IHashable, IComparable, INumeric {
+ public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return 4;
+ }
+ };
+
+ public static final IPointableFactory FACTORY = new IPointableFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IPointable createPointable() {
+ return new IntegerPointable();
+ }
+
+ @Override
+ public ITypeTraits getTypeTraits() {
+ return TYPE_TRAITS;
+ }
+ };
+
+ public static int getInteger(byte[] bytes, int start) {
+ return ((bytes[start] & 0xff) << 24) + ((bytes[start + 1] & 0xff) << 16) + ((bytes[start + 2] & 0xff) << 8)
+ + ((bytes[start + 3] & 0xff) << 0);
+ }
+
+ public static void setInteger(byte[] bytes, int start, int value) {
+ bytes[start] = (byte) ((value >>> 24) & 0xFF);
+ bytes[start + 1] = (byte) ((value >>> 16) & 0xFF);
+ bytes[start + 2] = (byte) ((value >>> 8) & 0xFF);
+ bytes[start + 3] = (byte) ((value >>> 0) & 0xFF);
+ }
+
+ public int getInteger() {
+ return getInteger(bytes, start);
+ }
+
+ public void setInteger(int value) {
+ setInteger(bytes, start, value);
+ }
+
+ public int preIncrement() {
+ int v = getInteger();
+ ++v;
+ setInteger(v);
+ return v;
+ }
+
+ public int postIncrement() {
+ int v = getInteger();
+ int ov = v++;
+ setInteger(v);
+ return ov;
+ }
+
+ @Override
+ public int compareTo(IPointable pointer) {
+ return compareTo(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public int compareTo(byte[] bytes, int start, int length) {
+ int v = getInteger();
+ int ov = getInteger(bytes, start);
+ return v < ov ? -1 : (v > ov ? 1 : 0);
+ }
+
+ @Override
+ public int hash() {
+ return getInteger();
+ }
+
+ @Override
+ public byte byteValue() {
+ return (byte) getInteger();
+ }
+
+ @Override
+ public short shortValue() {
+ return (short) getInteger();
+ }
+
+ @Override
+ public int intValue() {
+ return getInteger();
+ }
+
+ @Override
+ public long longValue() {
+ return getInteger();
+ }
+
+ @Override
+ public float floatValue() {
+ return getInteger();
+ }
+
+ @Override
+ public double doubleValue() {
+ return getInteger();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/LongPointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/LongPointable.java
new file mode 100644
index 0000000..89f9868
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/LongPointable.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.AbstractPointable;
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.api.IHashable;
+import edu.uci.ics.hyracks.data.std.api.INumeric;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public final class LongPointable extends AbstractPointable implements IHashable, IComparable, INumeric {
+ public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return 8;
+ }
+ };
+
+ public static final IPointableFactory FACTORY = new IPointableFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IPointable createPointable() {
+ return new LongPointable();
+ }
+
+ @Override
+ public ITypeTraits getTypeTraits() {
+ return TYPE_TRAITS;
+ }
+ };
+
+ public static long getLong(byte[] bytes, int start) {
+ return (((long) (bytes[start] & 0xff)) << 56) + (((long) (bytes[start + 1] & 0xff)) << 48)
+ + (((long) (bytes[start + 2] & 0xff)) << 40) + (((long) (bytes[start + 3] & 0xff)) << 32)
+ + (((long) (bytes[start + 4] & 0xff)) << 24) + (((long) (bytes[start + 5] & 0xff)) << 16)
+ + (((long) (bytes[start + 6] & 0xff)) << 8) + (((long) (bytes[start + 7] & 0xff)) << 0);
+ }
+
+ public static void setLong(byte[] bytes, int start, long value) {
+ bytes[start] = (byte) ((value >>> 56) & 0xFF);
+ bytes[start + 1] = (byte) ((value >>> 48) & 0xFF);
+ bytes[start + 2] = (byte) ((value >>> 40) & 0xFF);
+ bytes[start + 3] = (byte) ((value >>> 32) & 0xFF);
+ bytes[start + 4] = (byte) ((value >>> 24) & 0xFF);
+ bytes[start + 5] = (byte) ((value >>> 16) & 0xFF);
+ bytes[start + 6] = (byte) ((value >>> 8) & 0xFF);
+ bytes[start + 7] = (byte) ((value >>> 0) & 0xFF);
+ }
+
+ public long getLong() {
+ return getLong(bytes, start);
+ }
+
+ public void setLong(long value) {
+ setLong(bytes, start, value);
+ }
+
+ public long preIncrement() {
+ long v = getLong();
+ ++v;
+ setLong(v);
+ return v;
+ }
+
+ public long postIncrement() {
+ long v = getLong();
+ long ov = v++;
+ setLong(v);
+ return ov;
+ }
+
+ @Override
+ public int compareTo(IPointable pointer) {
+ return compareTo(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public int compareTo(byte[] bytes, int start, int length) {
+ long v = getLong();
+ long ov = getLong(bytes, start);
+ return v < ov ? -1 : (v > ov ? 1 : 0);
+ }
+
+ @Override
+ public int hash() {
+ long v = getLong();
+ return (int) (v ^ (v >>> 32));
+ }
+
+ @Override
+ public byte byteValue() {
+ return (byte) getLong();
+ }
+
+ @Override
+ public short shortValue() {
+ return (short) getLong();
+ }
+
+ @Override
+ public int intValue() {
+ return (int) getLong();
+ }
+
+ @Override
+ public long longValue() {
+ return getLong();
+ }
+
+ @Override
+ public float floatValue() {
+ return getLong();
+ }
+
+ @Override
+ public double doubleValue() {
+ return getLong();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/ShortPointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/ShortPointable.java
new file mode 100644
index 0000000..57b9a1b
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/ShortPointable.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.AbstractPointable;
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.api.IHashable;
+import edu.uci.ics.hyracks.data.std.api.INumeric;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public final class ShortPointable extends AbstractPointable implements IHashable, IComparable, INumeric {
+ public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isFixedLength() {
+ return true;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return 2;
+ }
+ };
+
+ public static final IPointableFactory FACTORY = new IPointableFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IPointable createPointable() {
+ return new ShortPointable();
+ }
+
+ @Override
+ public ITypeTraits getTypeTraits() {
+ return TYPE_TRAITS;
+ }
+ };
+
+ public static short getShort(byte[] bytes, int start) {
+ return (short) (((bytes[start] & 0xff) << 8) + (bytes[start + 1] & 0xff));
+ }
+
+ public static void setShort(byte[] bytes, int start, short value) {
+ bytes[start] = (byte) ((value >>> 24) & 0xFF);
+ bytes[start + 1] = (byte) ((value >>> 16) & 0xFF);
+ }
+
+ public short getShort() {
+ return getShort(bytes, start);
+ }
+
+ public void setShort(short value) {
+ setShort(bytes, start, value);
+ }
+
+ public short preIncrement() {
+ short v = getShort();
+ ++v;
+ setShort(v);
+ return v;
+ }
+
+ public short postIncrement() {
+ short v = getShort();
+ short ov = v++;
+ setShort(v);
+ return ov;
+ }
+
+ @Override
+ public int compareTo(IPointable pointer) {
+ return compareTo(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public int compareTo(byte[] bytes, int start, int length) {
+ short v = getShort();
+ short ov = getShort(bytes, start);
+ return v < ov ? -1 : (v > ov ? 1 : 0);
+ }
+
+ @Override
+ public int hash() {
+ return getShort();
+ }
+
+ @Override
+ public byte byteValue() {
+ return (byte) getShort();
+ }
+
+ @Override
+ public short shortValue() {
+ return getShort();
+ }
+
+ @Override
+ public int intValue() {
+ return getShort();
+ }
+
+ @Override
+ public long longValue() {
+ return getShort();
+ }
+
+ @Override
+ public float floatValue() {
+ return getShort();
+ }
+
+ @Override
+ public double doubleValue() {
+ return getShort();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringPointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringPointable.java
new file mode 100644
index 0000000..21caf77
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/UTF8StringPointable.java
@@ -0,0 +1,169 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.AbstractPointable;
+import edu.uci.ics.hyracks.data.std.api.IComparable;
+import edu.uci.ics.hyracks.data.std.api.IHashable;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public final class UTF8StringPointable extends AbstractPointable implements IHashable, IComparable {
+ public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isFixedLength() {
+ return false;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return 0;
+ }
+ };
+
+ public static final IPointableFactory FACTORY = new IPointableFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IPointable createPointable() {
+ return new UTF8StringPointable();
+ }
+
+ @Override
+ public ITypeTraits getTypeTraits() {
+ return TYPE_TRAITS;
+ }
+ };
+
+ public static char charAt(byte[] b, int s) {
+ int c = b[s] & 0xff;
+ switch (c >> 4) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ return (char) c;
+
+ case 12:
+ case 13:
+ return (char) (((c & 0x1F) << 6) | ((b[s + 1]) & 0x3F));
+
+ case 14:
+ return (char) (((c & 0x0F) << 12) | (((b[s + 1]) & 0x3F) << 6) | (((b[s + 2]) & 0x3F) << 0));
+
+ default:
+ throw new IllegalArgumentException();
+ }
+ }
+
+ public static int charSize(byte[] b, int s) {
+ int c = b[s] & 0xff;
+ switch (c >> 4) {
+ case 0:
+ case 1:
+ case 2:
+ case 3:
+ case 4:
+ case 5:
+ case 6:
+ case 7:
+ return 1;
+
+ case 12:
+ case 13:
+ return 2;
+
+ case 14:
+ return 3;
+ }
+ throw new IllegalStateException();
+ }
+
+ public static int getModifiedUTF8Len(char c) {
+ if (c >= 0x0000 && c <= 0x007F) {
+ return 1;
+ } else if (c <= 0x07FF) {
+ return 2;
+ } else {
+ return 3;
+ }
+ }
+
+ public static int getStrLen(byte[] b, int s) {
+ int pos = s + 2;
+ int end = pos + getUTFLen(b, s);
+ int charCount = 0;
+ while (pos < end) {
+ charCount++;
+ pos += charSize(b, pos);
+ }
+ return charCount;
+ }
+
+ public static int getUTFLen(byte[] b, int s) {
+ return ((b[s] & 0xff) << 8) + ((b[s + 1] & 0xff) << 0);
+ }
+
+ @Override
+ public int compareTo(IPointable pointer) {
+ return compareTo(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+ }
+
+ @Override
+ public int compareTo(byte[] bytes, int start, int length) {
+ int utflen1 = getUTFLen(this.bytes, this.start);
+ int utflen2 = getUTFLen(bytes, start);
+
+ int c1 = 0;
+ int c2 = 0;
+
+ int s1Start = this.start + 2;
+ int s2Start = start + 2;
+
+ while (c1 < utflen1 && c2 < utflen2) {
+ char ch1 = charAt(this.bytes, s1Start + c1);
+ char ch2 = charAt(bytes, s2Start + c2);
+
+ if (ch1 != ch2) {
+ return ch1 - ch2;
+ }
+ c1 += charSize(this.bytes, s1Start + c1);
+ c2 += charSize(bytes, s2Start + c2);
+ }
+ return utflen1 - utflen2;
+ }
+
+ @Override
+ public int hash() {
+ int h = 0;
+ int utflen = getUTFLen(bytes, start);
+ int sStart = start + 2;
+ int c = 0;
+
+ while (c < utflen) {
+ char ch = charAt(bytes, sStart + c);
+ h = 31 * h + ch;
+ c += charSize(bytes, sStart + c);
+ }
+ return h;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/VoidPointable.java b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/VoidPointable.java
new file mode 100644
index 0000000..4faf606
--- /dev/null
+++ b/hyracks-data/hyracks-data-std/src/main/java/edu/uci/ics/hyracks/data/std/primitive/VoidPointable.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.data.std.primitive;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.AbstractPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+
+public final class VoidPointable extends AbstractPointable {
+ public static final ITypeTraits TYPE_TRAITS = new ITypeTraits() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public boolean isFixedLength() {
+ return false;
+ }
+
+ @Override
+ public int getFixedLength() {
+ return 0;
+ }
+ };
+
+ public static final IPointableFactory FACTORY = new IPointableFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IPointable createPointable() {
+ return new VoidPointable();
+ }
+
+ @Override
+ public ITypeTraits getTypeTraits() {
+ return TYPE_TRAITS;
+ }
+ };
+}
\ No newline at end of file
diff --git a/hyracks-data/pom.xml b/hyracks-data/pom.xml
new file mode 100644
index 0000000..ed777e9
--- /dev/null
+++ b/hyracks-data/pom.xml
@@ -0,0 +1,15 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-data</artifactId>
+ <packaging>pom</packaging>
+
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <modules>
+ <module>hyracks-data-std</module>
+ </modules>
+</project>
diff --git a/hyracks-dataflow-common/.classpath b/hyracks-dataflow-common/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-dataflow-common/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-dataflow-common/.project b/hyracks-dataflow-common/.project
deleted file mode 100644
index 8fd9acf..0000000
--- a/hyracks-dataflow-common/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-dataflow-common</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-dataflow-common/.settings/org.eclipse.jdt.core.prefs b/hyracks-dataflow-common/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 450f5c4..0000000
--- a/hyracks-dataflow-common/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:04 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-dataflow-common/.settings/org.maven.ide.eclipse.prefs b/hyracks-dataflow-common/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index c411710..0000000
--- a/hyracks-dataflow-common/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Jul 29 14:49:35 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-dataflow-common/pom.xml b/hyracks-dataflow-common/pom.xml
index 3a0453c..142a91e 100644
--- a/hyracks-dataflow-common/pom.xml
+++ b/hyracks-dataflow-common/pom.xml
@@ -1,13 +1,10 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,9 +24,14 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java
index c39023b..0e9e3c0 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java
@@ -23,6 +23,11 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IDataOutputProvider;
+/**
+ * Array backed tuple builder.
+ *
+ * @author vinayakb
+ */
public class ArrayTupleBuilder implements IDataOutputProvider {
private final ByteArrayAccessibleOutputStream baaos;
private final DataOutputStream dos;
@@ -35,23 +40,56 @@
fEndOffsets = new int[nFields];
}
+ /**
+ * Resets the builder.
+ *
+ * reset() must be called before attempting to create a new tuple.
+ */
public void reset() {
nextField = 0;
baaos.reset();
}
+ /**
+ * Get the end offsets of the fields in this tuple.
+ *
+ * @return end offsets of the fields.
+ */
public int[] getFieldEndOffsets() {
return fEndOffsets;
}
+ /**
+ * Get the data area in this builder.
+ *
+ * @return Data byte array.
+ */
public byte[] getByteArray() {
return baaos.getByteArray();
}
+ /**
+ * Get the size of the data area.
+ *
+ * @return data area size.
+ */
public int getSize() {
return baaos.size();
}
+ /**
+ * Add a field to the tuple from a field in a frame.
+ *
+ * @param accessor
+ * - Frame that contains the field to be copied into the tuple
+ * builder.
+ * @param tIndex
+ * - Tuple index of the tuple that contains the field to be
+ * copied.
+ * @param fIndex
+ * - Field index of the field to be copied.
+ * @throws HyracksDataException
+ */
public void addField(IFrameTupleAccessor accessor, int tIndex, int fIndex) throws HyracksDataException {
int startOffset = accessor.getTupleStartOffset(tIndex);
int fStartOffset = accessor.getFieldStartOffset(tIndex, fIndex);
@@ -67,11 +105,32 @@
fEndOffsets[nextField++] = baaos.size();
}
+ /**
+ * Add a field to the tuple by serializing the given object using the given
+ * serializer.
+ *
+ * @param serDeser
+ * - Serializer
+ * @param instance
+ * - Object to serialize
+ * @throws HyracksDataException
+ */
public <T> void addField(ISerializerDeserializer<T> serDeser, T instance) throws HyracksDataException {
serDeser.serialize(instance, dos);
fEndOffsets[nextField++] = baaos.size();
}
+ /**
+ * Add a field to the tuple by copying the data bytes from a byte array.
+ *
+ * @param bytes
+ * - Byte array to copy the field data from.
+ * @param start
+ * - Start offset of the field to be copied in the byte array.
+ * @param length
+ * - Length of the field to be copied.
+ * @throws HyracksDataException
+ */
public void addField(byte[] bytes, int start, int length) throws HyracksDataException {
try {
dos.write(bytes, start, length);
@@ -81,11 +140,20 @@
fEndOffsets[nextField++] = baaos.size();
}
+ /**
+ * Get the {@link DataOutput} interface to write into the tuple builder.
+ */
@Override
public DataOutput getDataOutput() {
return dos;
}
+ /**
+ * Sets the byte offset of the end of the field into the field offset array.
+ * Make sure this method is called when the {@link DataOutput} interface is
+ * used to add field data into the tuple, at the end of adding a field's
+ * data.
+ */
public void addFieldEndOffset() {
fEndOffsets[nextField++] = baaos.size();
}
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleReference.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleReference.java
new file mode 100644
index 0000000..b265283
--- /dev/null
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/ArrayTupleReference.java
@@ -0,0 +1,41 @@
+package edu.uci.ics.hyracks.dataflow.common.comm.io;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * An ArrayTupleReference provides access to a tuple that is not serialized into
+ * a frame. It is meant to be reset directly with the field slots and tuple data
+ * provided by ArrayTupleBuilder. The purpose is to avoid coping the built tuple
+ * into a frame before being able to use it as an ITupleReference.
+ *
+ * @author alexander.behm
+ */
+public class ArrayTupleReference implements ITupleReference {
+ private int[] fEndOffsets;
+ private byte[] tupleData;
+
+ public void reset(int[] fEndOffsets, byte[] tupleData) {
+ this.fEndOffsets = fEndOffsets;
+ this.tupleData = tupleData;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return fEndOffsets.length;
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ return tupleData;
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ return (fIdx == 0) ? 0 : fEndOffsets[fIdx - 1];
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ return (fIdx == 0) ? fEndOffsets[0] : fEndOffsets[fIdx] - fEndOffsets[fIdx - 1];
+ }
+}
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializer.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializer.java
index 61638d1..beaf862 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializer.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameDeserializer.java
@@ -62,7 +62,7 @@
public Object[] deserializeRecord() throws HyracksDataException {
int start = frameTupleAccessor.getTupleStartOffset(tIndex) + frameTupleAccessor.getFieldSlotsLength();
bbis.setByteBuffer(buffer, start);
- Object[] record = new Object[recordDescriptor.getFields().length];
+ Object[] record = new Object[recordDescriptor.getFieldCount()];
for (int i = 0; i < record.length; ++i) {
Object instance = recordDescriptor.getFields()[i].deserialize(di);
if (LOGGER.isLoggable(Level.FINEST)) {
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAccessor.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAccessor.java
index 8ab5d4f..6c51d14 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAccessor.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAccessor.java
@@ -24,11 +24,12 @@
import edu.uci.ics.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
/**
- * FrameTupleCursor is used to navigate over tuples in a Frame.
- * A frame is formatted with tuple data concatenated starting at offset 0, one tuple after another.
- * Offset FS - 4 holds an int indicating the number of tuples (N) in the frame. FS - ((i + 1) * 4) for i from
- * 0 to N - 1 holds an int indicating the offset of the (i + 1)^th tuple.
- * Every tuple is organized as a sequence of ints indicating the end of each field in the tuple relative to the end of the
+ * FrameTupleCursor is used to navigate over tuples in a Frame. A frame is
+ * formatted with tuple data concatenated starting at offset 0, one tuple after
+ * another. Offset FS - 4 holds an int indicating the number of tuples (N) in
+ * the frame. FS - ((i + 1) * 4) for i from 0 to N - 1 holds an int indicating
+ * the offset of the (i + 1)^th tuple. Every tuple is organized as a sequence of
+ * ints indicating the end of each field in the tuple relative to the end of the
* field slots.
*
* @author vinayakb
@@ -86,7 +87,7 @@
@Override
public int getFieldSlotsLength() {
- return recordDescriptor.getFields().length * 4;
+ return getFieldCount() * 4;
}
public void prettyPrint() {
@@ -96,7 +97,7 @@
System.err.println("TC: " + tc);
for (int i = 0; i < tc; ++i) {
System.err.print(i + ":(" + getTupleStartOffset(i) + ", " + getTupleEndOffset(i) + ")[");
- for (int j = 0; j < recordDescriptor.getFields().length; ++j) {
+ for (int j = 0; j < getFieldCount(); ++j) {
System.err.print(j + ":(" + getFieldStartOffset(i, j) + ", " + getFieldEndOffset(i, j) + ") ");
System.err.print("{");
bbis.setByteBuffer(buffer, getTupleStartOffset(i) + getFieldSlotsLength() + getFieldStartOffset(i, j));
@@ -113,6 +114,6 @@
@Override
public int getFieldCount() {
- return recordDescriptor.getFields().length;
+ return recordDescriptor.getFieldCount();
}
}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAppender.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAppender.java
index 6573ffa..956f5da 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAppender.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/FrameTupleAppender.java
@@ -59,23 +59,38 @@
}
return false;
}
-
- /*
- * ADDED BY POURIA (for his sort operator)
- * bytes already has the header (the fields offset)
- */
- public boolean append(byte[] bytes, int offset, int length){
- if (tupleDataEndOffset + length + 4 + (tupleCount + 1) * 4 <= frameSize) {
- System.arraycopy(bytes, offset, buffer.array(), tupleDataEndOffset, length);
- tupleDataEndOffset += length;
- buffer.putInt(FrameHelper.getTupleCountOffset(frameSize) - 4 * (tupleCount + 1), tupleDataEndOffset);
- ++tupleCount;
- buffer.putInt(FrameHelper.getTupleCountOffset(frameSize), tupleCount);
- return true;
- }
- return false;
+
+ public boolean append(byte[] bytes, int offset, int length) {
+ if (tupleDataEndOffset + length + 4 + (tupleCount + 1) * 4 <= frameSize) {
+ System.arraycopy(bytes, offset, buffer.array(), tupleDataEndOffset, length);
+ tupleDataEndOffset += length;
+ buffer.putInt(FrameHelper.getTupleCountOffset(frameSize) - 4 * (tupleCount + 1), tupleDataEndOffset);
+ ++tupleCount;
+ buffer.putInt(FrameHelper.getTupleCountOffset(frameSize), tupleCount);
+ return true;
+ }
+ return false;
}
-
+
+ public boolean appendSkipEmptyField(int[] fieldSlots, byte[] bytes, int offset, int length) {
+ if (tupleDataEndOffset + fieldSlots.length * 4 + length + 4 + (tupleCount + 1) * 4 <= frameSize) {
+ int effectiveSlots = 0;
+ for (int i = 0; i < fieldSlots.length; ++i) {
+ if (fieldSlots[i] > 0) {
+ buffer.putInt(tupleDataEndOffset + i * 4, fieldSlots[i]);
+ effectiveSlots++;
+ }
+ }
+ System.arraycopy(bytes, offset, buffer.array(), tupleDataEndOffset + effectiveSlots * 4, length);
+ tupleDataEndOffset += effectiveSlots * 4 + length;
+ buffer.putInt(FrameHelper.getTupleCountOffset(frameSize) - 4 * (tupleCount + 1), tupleDataEndOffset);
+ ++tupleCount;
+ buffer.putInt(FrameHelper.getTupleCountOffset(frameSize), tupleCount);
+ return true;
+ }
+ return false;
+ }
+
public boolean append(IFrameTupleAccessor tupleAccessor, int tStartOffset, int tEndOffset) {
int length = tEndOffset - tStartOffset;
if (tupleDataEndOffset + length + 4 + (tupleCount + 1) * 4 <= frameSize) {
@@ -116,8 +131,7 @@
System.arraycopy(src0.array(), startOffset0, buffer.array(), tupleDataEndOffset, slotsLen0);
// Copy slots from accessor1 with the following transformation: newSlotIdx = oldSlotIdx + dataLen0
for (int i = 0; i < slotsLen1 / 4; ++i) {
- buffer.putInt(tupleDataEndOffset + slotsLen0 + i * 4,
- src1.getInt(startOffset1 + i * 4) + dataLen0);
+ buffer.putInt(tupleDataEndOffset + slotsLen0 + i * 4, src1.getInt(startOffset1 + i * 4) + dataLen0);
}
// Copy data0
System.arraycopy(src0.array(), startOffset0 + slotsLen0, buffer.array(), tupleDataEndOffset + slotsLen0
@@ -133,16 +147,16 @@
}
return false;
}
-
- public boolean appendConcat(IFrameTupleAccessor accessor0, int tIndex0, int[] fieldSlots1, byte[] bytes1, int offset1,
- int dataLen1) {
+
+ public boolean appendConcat(IFrameTupleAccessor accessor0, int tIndex0, int[] fieldSlots1, byte[] bytes1,
+ int offset1, int dataLen1) {
int startOffset0 = accessor0.getTupleStartOffset(tIndex0);
int endOffset0 = accessor0.getTupleEndOffset(tIndex0);
int length0 = endOffset0 - startOffset0;
int slotsLen1 = fieldSlots1.length * 4;
int length1 = slotsLen1 + dataLen1;
-
+
if (tupleDataEndOffset + length0 + length1 + 4 + (tupleCount + 1) * 4 <= frameSize) {
ByteBuffer src0 = accessor0.getBuffer();
int slotsLen0 = accessor0.getFieldSlotsLength();
@@ -168,15 +182,15 @@
return false;
}
- public boolean appendConcat(int[] fieldSlots0, byte[] bytes0, int offset0, int dataLen0, IFrameTupleAccessor accessor1,
- int tIndex1) {
+ public boolean appendConcat(int[] fieldSlots0, byte[] bytes0, int offset0, int dataLen0,
+ IFrameTupleAccessor accessor1, int tIndex1) {
int slotsLen0 = fieldSlots0.length * 4;
int length0 = slotsLen0 + dataLen0;
-
+
int startOffset1 = accessor1.getTupleStartOffset(tIndex1);
int endOffset1 = accessor1.getTupleEndOffset(tIndex1);
int length1 = endOffset1 - startOffset1;
-
+
if (tupleDataEndOffset + length0 + length1 + 4 + (tupleCount + 1) * 4 <= frameSize) {
ByteBuffer src1 = accessor1.getBuffer();
int slotsLen1 = accessor1.getFieldSlotsLength();
@@ -190,10 +204,9 @@
buffer.putInt(tupleDataEndOffset + slotsLen0 + i * 4, src1.getInt(startOffset1 + i * 4) + dataLen0);
}
// Copy bytes0
- System.arraycopy(bytes0, offset0, buffer.array(), tupleDataEndOffset + slotsLen0 + slotsLen1,
- dataLen0);
+ System.arraycopy(bytes0, offset0, buffer.array(), tupleDataEndOffset + slotsLen0 + slotsLen1, dataLen0);
// Copy data1
- System.arraycopy(src1.array(), startOffset1 + slotsLen1, buffer.array(), tupleDataEndOffset + slotsLen0
+ System.arraycopy(src1.array(), startOffset1 + slotsLen1, buffer.array(), tupleDataEndOffset + slotsLen0
+ slotsLen1 + dataLen0, dataLen1);
tupleDataEndOffset += (length0 + length1);
buffer.putInt(FrameHelper.getTupleCountOffset(frameSize) - 4 * (tupleCount + 1), tupleDataEndOffset);
@@ -203,6 +216,7 @@
}
return false;
}
+
public boolean appendProjection(IFrameTupleAccessor accessor, int tIndex, int[] fields) {
int fTargetSlotsLength = fields.length * 4;
int length = fTargetSlotsLength;
@@ -242,4 +256,4 @@
public ByteBuffer getBuffer() {
return buffer;
}
-}
\ No newline at end of file
+}
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/SerializingDataWriter.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/SerializingDataWriter.java
index e8f43cd..cbe2ae6 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/SerializingDataWriter.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/comm/io/SerializingDataWriter.java
@@ -41,7 +41,7 @@
public SerializingDataWriter(IHyracksTaskContext ctx, RecordDescriptor recordDescriptor, IFrameWriter frameWriter) {
buffer = ctx.allocateFrame();
- tb = new ArrayTupleBuilder(recordDescriptor.getFields().length);
+ tb = new ArrayTupleBuilder(recordDescriptor.getFieldCount());
this.recordDescriptor = recordDescriptor;
this.frameWriter = frameWriter;
tupleAppender = new FrameTupleAppender(ctx.getFrameSize());
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/FrameTupleReference.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/FrameTupleReference.java
index ff45eba..a21d5e9 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/FrameTupleReference.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/accessors/FrameTupleReference.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.dataflow.common.data.accessors;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/DoubleBinaryComparatorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/DoubleBinaryComparatorFactory.java
deleted file mode 100644
index 983dcb8..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/DoubleBinaryComparatorFactory.java
+++ /dev/null
@@ -1,25 +0,0 @@
-package edu.uci.ics.hyracks.dataflow.common.data.comparators;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
-
-public class DoubleBinaryComparatorFactory implements IBinaryComparatorFactory {
- private static final long serialVersionUID = 1L;
-
- public static final DoubleBinaryComparatorFactory INSTANCE = new DoubleBinaryComparatorFactory();
-
- private DoubleBinaryComparatorFactory() {
- }
-
- @Override
- public IBinaryComparator createBinaryComparator() {
- return new IBinaryComparator() {
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- return Double.compare(DoubleSerializerDeserializer.getDouble(b1, s1), DoubleSerializerDeserializer
- .getDouble(b2, s2));
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/FloatBinaryComparatorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/FloatBinaryComparatorFactory.java
deleted file mode 100644
index 46abc34..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/FloatBinaryComparatorFactory.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.comparators;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
-
-public class FloatBinaryComparatorFactory implements IBinaryComparatorFactory {
- private static final long serialVersionUID = 1L;
-
- public static final FloatBinaryComparatorFactory INSTANCE = new FloatBinaryComparatorFactory();
-
- private FloatBinaryComparatorFactory() {
- }
-
- @Override
- public IBinaryComparator createBinaryComparator() {
- return new IBinaryComparator() {
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- return Float.compare(FloatSerializerDeserializer.getFloat(b1, s1), FloatSerializerDeserializer
- .getFloat(b2, s2));
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/FloatComparatorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/FloatComparatorFactory.java
deleted file mode 100644
index 59f36bf..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/FloatComparatorFactory.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.comparators;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
-
-public class FloatComparatorFactory implements IComparatorFactory<Float> {
- private static final long serialVersionUID = 1L;
-
- public static final FloatComparatorFactory INSTANCE = new FloatComparatorFactory();
-
- private FloatComparatorFactory() {
- }
-
- @Override
- public IComparator<Float> createComparator() {
- return new IComparator<Float>() {
- @Override
- public int compare(Float o1, Float o2) {
- return o1.compareTo(o2);
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/IntegerBinaryComparatorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/IntegerBinaryComparatorFactory.java
deleted file mode 100644
index 493d4ee..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/IntegerBinaryComparatorFactory.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.comparators;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-
-public class IntegerBinaryComparatorFactory implements IBinaryComparatorFactory {
- private static final long serialVersionUID = 1L;
-
- public static final IntegerBinaryComparatorFactory INSTANCE = new IntegerBinaryComparatorFactory();
-
- private IntegerBinaryComparatorFactory() {
- }
-
- @Override
- public IBinaryComparator createBinaryComparator() {
- return new IBinaryComparator() {
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- int v1 = IntegerSerializerDeserializer.getInt(b1, s1);
- int v2 = IntegerSerializerDeserializer.getInt(b2, s2);
- return v1 < v2 ? -1 : (v1 > v2 ? 1 : 0);
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/IntegerComparatorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/IntegerComparatorFactory.java
deleted file mode 100644
index 9044e47..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/IntegerComparatorFactory.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.comparators;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
-
-public class IntegerComparatorFactory implements IComparatorFactory<Integer> {
- private static final long serialVersionUID = 1L;
-
- public static final IntegerComparatorFactory INSTANCE = new IntegerComparatorFactory();
-
- private IntegerComparatorFactory() {
- }
-
- @Override
- public IComparator<Integer> createComparator() {
- return new IComparator<Integer>() {
- @Override
- public int compare(Integer o1, Integer o2) {
- return o1.compareTo(o2);
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/StringComparatorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/StringComparatorFactory.java
deleted file mode 100644
index d79def5..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/StringComparatorFactory.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.comparators;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IComparatorFactory;
-
-public class StringComparatorFactory implements IComparatorFactory<String> {
- private static final long serialVersionUID = 1L;
-
- public static final StringComparatorFactory INSTANCE = new StringComparatorFactory();
-
- private StringComparatorFactory() {
- }
-
- @Override
- public IComparator<String> createComparator() {
- return new IComparator<String>() {
- @Override
- public int compare(String o1, String o2) {
- return o1.compareTo(o2);
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/UTF8StringBinaryComparatorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/UTF8StringBinaryComparatorFactory.java
deleted file mode 100644
index 439859d..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/comparators/UTF8StringBinaryComparatorFactory.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.comparators;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
-
-public class UTF8StringBinaryComparatorFactory implements IBinaryComparatorFactory {
- private static final long serialVersionUID = 1L;
-
- public static final UTF8StringBinaryComparatorFactory INSTANCE = new UTF8StringBinaryComparatorFactory();
-
- private UTF8StringBinaryComparatorFactory() {
- }
-
- @Override
- public IBinaryComparator createBinaryComparator() {
- return new IBinaryComparator() {
- @Override
- public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
- int utflen1 = StringUtils.getUTFLen(b1, s1);
- int utflen2 = StringUtils.getUTFLen(b2, s2);
-
- int c1 = 0;
- int c2 = 0;
-
- int s1Start = s1 + 2;
- int s2Start = s2 + 2;
-
- while (c1 < utflen1 && c2 < utflen2) {
- char ch1 = StringUtils.charAt(b1, s1Start + c1);
- char ch2 = StringUtils.charAt(b2, s2Start + c2);
-
- if (ch1 != ch2) {
- return ch1 - ch2;
- }
- c1 += StringUtils.charSize(b1, s1Start + c1);
- c2 += StringUtils.charSize(b2, s2Start + c2);
- }
- return utflen1 - utflen2;
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/FloatBinaryHashFunctionFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/FloatBinaryHashFunctionFactory.java
deleted file mode 100644
index 72294e8..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/FloatBinaryHashFunctionFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.hash;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
-
-public class FloatBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
- private static final long serialVersionUID = 1L;
-
- public static final FloatBinaryHashFunctionFactory INSTANCE = new FloatBinaryHashFunctionFactory();
-
- private FloatBinaryHashFunctionFactory() {
- }
-
- @Override
- public IBinaryHashFunction createBinaryHashFunction() {
- return new IBinaryHashFunction() {
- @Override
- public int hash(byte[] bytes, int offset, int length) {
- return FloatSerializerDeserializer.getIntBits(bytes, offset);
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/FloatHashFunctionFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/FloatHashFunctionFactory.java
deleted file mode 100644
index 481c111..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/FloatHashFunctionFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.hash;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IHashFunction;
-import edu.uci.ics.hyracks.api.dataflow.value.IHashFunctionFactory;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public class FloatHashFunctionFactory implements IHashFunctionFactory<Float> {
- private static final long serialVersionUID = 1L;
-
- public static final FloatHashFunctionFactory INSTANCE = new FloatHashFunctionFactory();
-
- private FloatHashFunctionFactory() {
- }
-
- @Override
- public IHashFunction<Float> createHashFunction() {
- return new IHashFunction<Float>() {
- @Override
- public int hash(Float o) throws HyracksDataException {
- return o.hashCode();
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/IntegerBinaryHashFunctionFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/IntegerBinaryHashFunctionFactory.java
deleted file mode 100644
index 51cd494..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/IntegerBinaryHashFunctionFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.hash;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-
-public class IntegerBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
- private static final long serialVersionUID = 1L;
-
- public static final IntegerBinaryHashFunctionFactory INSTANCE = new IntegerBinaryHashFunctionFactory();
-
- private IntegerBinaryHashFunctionFactory() {
- }
-
- @Override
- public IBinaryHashFunction createBinaryHashFunction() {
- return new IBinaryHashFunction() {
- @Override
- public int hash(byte[] bytes, int offset, int length) {
- return IntegerSerializerDeserializer.getInt(bytes, offset);
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/IntegerHashFunctionFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/IntegerHashFunctionFactory.java
deleted file mode 100644
index c3880bf..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/IntegerHashFunctionFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.hash;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IHashFunction;
-import edu.uci.ics.hyracks.api.dataflow.value.IHashFunctionFactory;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public class IntegerHashFunctionFactory implements IHashFunctionFactory<Integer> {
- private static final long serialVersionUID = 1L;
-
- public static final IntegerHashFunctionFactory INSTANCE = new IntegerHashFunctionFactory();
-
- private IntegerHashFunctionFactory() {
- }
-
- @Override
- public IHashFunction<Integer> createHashFunction() {
- return new IHashFunction<Integer>() {
- @Override
- public int hash(Integer o) throws HyracksDataException {
- return o.hashCode();
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/StringHashFunctionFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/StringHashFunctionFactory.java
deleted file mode 100644
index 3a6d9a9..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/StringHashFunctionFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.hash;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IHashFunction;
-import edu.uci.ics.hyracks.api.dataflow.value.IHashFunctionFactory;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public class StringHashFunctionFactory implements IHashFunctionFactory<String> {
- private static final long serialVersionUID = 1L;
-
- public static final StringHashFunctionFactory INSTANCE = new StringHashFunctionFactory();
-
- private StringHashFunctionFactory() {
- }
-
- @Override
- public IHashFunction<String> createHashFunction() {
- return new IHashFunction<String>() {
- @Override
- public int hash(String o) throws HyracksDataException {
- return o.hashCode();
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/UTF8StringBinaryHashFunctionFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/UTF8StringBinaryHashFunctionFactory.java
deleted file mode 100644
index 98634b6..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/UTF8StringBinaryHashFunctionFactory.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.common.data.hash;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
-
-public class UTF8StringBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
- public static final UTF8StringBinaryHashFunctionFactory INSTANCE = new UTF8StringBinaryHashFunctionFactory();
-
- private static final long serialVersionUID = 1L;
-
- private UTF8StringBinaryHashFunctionFactory() {
- }
-
- @Override
- public IBinaryHashFunction createBinaryHashFunction() {
- return new IBinaryHashFunction() {
- @Override
- public int hash(byte[] bytes, int offset, int length) {
- int h = 0;
- int utflen = StringUtils.getUTFLen(bytes, offset);
- int sStart = offset + 2;
- int c = 0;
-
- while (c < utflen) {
- char ch = StringUtils.charAt(bytes, sStart + c);
- h = 31 * h + ch;
- c += StringUtils.charSize(bytes, sStart + c);
- }
- return h;
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/UTF8StringBinaryHashFunctionGeneratorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/UTF8StringBinaryHashFunctionGeneratorFactory.java
deleted file mode 100644
index 6ba9bd6..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/hash/UTF8StringBinaryHashFunctionGeneratorFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package edu.uci.ics.hyracks.dataflow.common.data.hash;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionGeneratorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
-
-public class UTF8StringBinaryHashFunctionGeneratorFactory implements IBinaryHashFunctionGeneratorFactory {
-
-
- public static final UTF8StringBinaryHashFunctionGeneratorFactory INSTANCE = new UTF8StringBinaryHashFunctionGeneratorFactory();
- private static final long serialVersionUID = 1L;
-
- static final int[] primeCoefficents = {31, 23, 53, 97, 71, 337, 11, 877, 3, 29};
-
- private UTF8StringBinaryHashFunctionGeneratorFactory(){
- }
-
- @Override
- public IBinaryHashFunction createBinaryHashFunction(int seed) {
- final int coefficient = primeCoefficents[seed % primeCoefficents.length];
- final int r = primeCoefficents[(seed+1) % primeCoefficents.length];
-
- return new IBinaryHashFunction() {
- @Override
- public int hash(byte[] bytes, int offset, int length) {
- int h = 0;
- int utflen = StringUtils.getUTFLen(bytes, offset);
- int sStart = offset + 2;
- int c = 0;
-
- while (c < utflen) {
- char ch = StringUtils.charAt(bytes, sStart + c);
- h = (coefficient * h + ch)%r;
- c += StringUtils.charSize(bytes, sStart + c);
- }
- return h;
- }
- };
- }
-
-}
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/marshalling/IntegerSerializerDeserializer.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/marshalling/IntegerSerializerDeserializer.java
index 032fddf..171dbb7 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/marshalling/IntegerSerializerDeserializer.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/marshalling/IntegerSerializerDeserializer.java
@@ -51,4 +51,11 @@
return ((bytes[offset] & 0xff) << 24) + ((bytes[offset + 1] & 0xff) << 16) + ((bytes[offset + 2] & 0xff) << 8)
+ ((bytes[offset + 3] & 0xff) << 0);
}
+
+ public static void putInt(int val, byte[] bytes, int offset) {
+ bytes[offset] = (byte)((val >>> 24) & 0xFF);
+ bytes[offset + 1] = (byte)((val >>> 16) & 0xFF);
+ bytes[offset + 2] = (byte)((val >>> 8) & 0xFF);
+ bytes[offset + 3] = (byte)((val >>> 0) & 0xFF);
+ }
}
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/normalizers/UTF8StringNormalizedKeyComputerFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/normalizers/UTF8StringNormalizedKeyComputerFactory.java
index 53b8d23..60fd395 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/normalizers/UTF8StringNormalizedKeyComputerFactory.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/normalizers/UTF8StringNormalizedKeyComputerFactory.java
@@ -16,7 +16,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputer;
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
public class UTF8StringNormalizedKeyComputerFactory implements INormalizedKeyComputerFactory {
private static final long serialVersionUID = 1L;
@@ -26,14 +26,14 @@
return new INormalizedKeyComputer() {
@Override
public int normalize(byte[] bytes, int start, int length) {
- int len = StringUtils.getUTFLen(bytes, start);
+ int len = UTF8StringPointable.getUTFLen(bytes, start);
int nk = 0;
int offset = start + 2;
for (int i = 0; i < 2; ++i) {
nk <<= 16;
if (i < len) {
- nk += ((int) StringUtils.charAt(bytes, offset)) & 0xffff;
- offset += StringUtils.charSize(bytes, offset);
+ nk += ((int) UTF8StringPointable.charAt(bytes, offset)) & 0xffff;
+ offset += UTF8StringPointable.charSize(bytes, offset);
}
}
return nk;
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java
index 8d69b95..7769539 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFactory.java
@@ -39,6 +39,9 @@
return new ITuplePartitionComputer() {
@Override
public int partition(IFrameTupleAccessor accessor, int tIndex, int nParts) {
+ if (nParts == 1) {
+ return 0;
+ }
int h = 0;
int startOffset = accessor.getTupleStartOffset(tIndex);
int slotLength = accessor.getFieldSlotsLength();
@@ -49,7 +52,7 @@
int fEnd = accessor.getFieldEndOffset(tIndex, fIdx);
int fh = hashFn
.hash(accessor.getBuffer().array(), startOffset + slotLength + fStart, fEnd - fStart);
- h += fh;
+ h = h * 31 + fh;
}
if (h < 0) {
h = -h;
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFamily.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFamily.java
new file mode 100644
index 0000000..51645c4
--- /dev/null
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerFamily.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.common.data.partition;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFamily;
+
+public class FieldHashPartitionComputerFamily implements ITuplePartitionComputerFamily {
+ private static final long serialVersionUID = 1L;
+ private final int[] hashFields;
+ private final IBinaryHashFunctionFamily[] hashFunctionGeneratorFactories;
+
+ public FieldHashPartitionComputerFamily(int[] hashFields, IBinaryHashFunctionFamily[] hashFunctionGeneratorFactories) {
+ this.hashFields = hashFields;
+ this.hashFunctionGeneratorFactories = hashFunctionGeneratorFactories;
+ }
+
+ @Override
+ public ITuplePartitionComputer createPartitioner(int seed) {
+ final IBinaryHashFunction[] hashFunctions = new IBinaryHashFunction[hashFunctionGeneratorFactories.length];
+ for (int i = 0; i < hashFunctionGeneratorFactories.length; ++i) {
+ hashFunctions[i] = hashFunctionGeneratorFactories[i].createBinaryHashFunction(seed);
+ }
+ return new ITuplePartitionComputer() {
+ @Override
+ public int partition(IFrameTupleAccessor accessor, int tIndex, int nParts) {
+ int h = 0;
+ int startOffset = accessor.getTupleStartOffset(tIndex);
+ int slotLength = accessor.getFieldSlotsLength();
+ for (int j = 0; j < hashFields.length; ++j) {
+ int fIdx = hashFields[j];
+ IBinaryHashFunction hashFn = hashFunctions[j];
+ int fStart = accessor.getFieldStartOffset(tIndex, fIdx);
+ int fEnd = accessor.getFieldEndOffset(tIndex, fIdx);
+ int fh = hashFn
+ .hash(accessor.getBuffer().array(), startOffset + slotLength + fStart, fEnd - fStart);
+ h += fh;
+ }
+ if (h < 0) {
+ h = -h;
+ }
+ return h % nParts;
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerGeneratorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerGeneratorFactory.java
deleted file mode 100644
index a7898c8..0000000
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/FieldHashPartitionComputerGeneratorFactory.java
+++ /dev/null
@@ -1,55 +0,0 @@
-package edu.uci.ics.hyracks.dataflow.common.data.partition;
-
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionGeneratorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerGeneratorFactory;
-
-
-
-public class FieldHashPartitionComputerGeneratorFactory implements
- ITuplePartitionComputerGeneratorFactory {
-
- private static final long serialVersionUID = 1L;
- private final int[] hashFields;
- private final IBinaryHashFunctionGeneratorFactory[] hashFunctionGeneratorFactories;
-
- public FieldHashPartitionComputerGeneratorFactory(int[] hashFields, IBinaryHashFunctionGeneratorFactory[] hashFunctionGeneratorFactories) {
- this.hashFields = hashFields;
- this.hashFunctionGeneratorFactories = hashFunctionGeneratorFactories;
- }
-
- @Override
- public ITuplePartitionComputer createPartitioner(int seed) {
- final IBinaryHashFunction[] hashFunctions = new IBinaryHashFunction[hashFunctionGeneratorFactories.length];
- for (int i = 0; i < hashFunctionGeneratorFactories.length; ++i) {
- hashFunctions[i] = hashFunctionGeneratorFactories[i].createBinaryHashFunction(seed);
- }
- return new ITuplePartitionComputer() {
- @Override
- public int partition(IFrameTupleAccessor accessor, int tIndex, int nParts) {
- int h = 0;
- int startOffset = accessor.getTupleStartOffset(tIndex);
- int slotLength = accessor.getFieldSlotsLength();
- for (int j = 0; j < hashFields.length; ++j) {
- int fIdx = hashFields[j];
- IBinaryHashFunction hashFn = hashFunctions[j];
- int fStart = accessor.getFieldStartOffset(tIndex, fIdx);
- int fEnd = accessor.getFieldEndOffset(tIndex, fIdx);
- int fh = hashFn
- .hash(accessor.getBuffer().array(), startOffset + slotLength + fStart, fEnd - fStart);
- h += fh;
- }
- if (h < 0) {
- h = -h;
- }
- return h % nParts;
- }
- };
- }
-
-
-
-}
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/RepartitionComputerGeneratorFactory.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/RepartitionComputerGeneratorFactory.java
index 4f540ef..7b4c96b 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/RepartitionComputerGeneratorFactory.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/partition/RepartitionComputerGeneratorFactory.java
@@ -2,17 +2,17 @@
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerGeneratorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFamily;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-public class RepartitionComputerGeneratorFactory implements ITuplePartitionComputerGeneratorFactory{
+public class RepartitionComputerGeneratorFactory implements ITuplePartitionComputerFamily{
private static final long serialVersionUID = 1L;
private int factor;
- private ITuplePartitionComputerGeneratorFactory delegateFactory;
+ private ITuplePartitionComputerFamily delegateFactory;
- public RepartitionComputerGeneratorFactory(int factor, ITuplePartitionComputerGeneratorFactory delegate) {
+ public RepartitionComputerGeneratorFactory(int factor, ITuplePartitionComputerFamily delegate) {
this.factor = factor;
this.delegateFactory = delegate;
}
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/util/StringUtils.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/util/StringUtils.java
index d11245b..a29209c 100644
--- a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/util/StringUtils.java
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/data/util/StringUtils.java
@@ -18,195 +18,22 @@
import java.io.IOException;
public class StringUtils {
- public static char charAt(byte[] b, int s) {
- int c = b[s] & 0xff;
- switch (c >> 4) {
- case 0:
- case 1:
- case 2:
- case 3:
- case 4:
- case 5:
- case 6:
- case 7:
- return (char) c;
+ public static void writeCharAsModifiedUTF8(char c, DataOutput dos) throws IOException {
- case 12:
- case 13:
- return (char) (((c & 0x1F) << 6) | ((b[s + 1]) & 0x3F));
+ if (c >= 0x0000 && c <= 0x007F) {
+ dos.writeByte(c);
+ } else if (c <= 0x07FF) {
+ dos.writeByte((byte) (0xC0 | ((c >> 6) & 0x3F)));
+ dos.writeByte((byte) (0x80 | (c & 0x3F)));
+ } else {
+ dos.writeByte((byte) (0xE0 | ((c >> 12) & 0x0F)));
+ dos.writeByte((byte) (0x80 | ((c >> 6) & 0x3F)));
+ dos.writeByte((byte) (0x80 | (c & 0x3F)));
+ }
+ }
- case 14:
- return (char) (((c & 0x0F) << 12) | (((b[s + 1]) & 0x3F) << 6) | (((b[s + 2]) & 0x3F) << 0));
-
- default:
- throw new IllegalArgumentException();
- }
- }
-
- public static int charSize(byte[] b, int s) {
- int c = b[s] & 0xff;
- switch (c >> 4) {
- case 0:
- case 1:
- case 2:
- case 3:
- case 4:
- case 5:
- case 6:
- case 7:
- return 1;
-
- case 12:
- case 13:
- return 2;
-
- case 14:
- return 3;
- }
- throw new IllegalStateException();
- }
-
- public static int getModifiedUTF8Len(char c) {
- if (c >= 0x0000 && c <= 0x007F) {
- return 1;
- } else if (c <= 0x07FF) {
- return 2;
- } else {
- return 3;
- }
- }
-
- public static int getStrLen(byte[] b, int s) {
- int pos = s + 2;
- int end = pos + getUTFLen(b, s);
- int charCount = 0;
- while (pos < end) {
- charCount++;
- pos += charSize(b, pos);
- }
- return charCount;
- }
-
- public static int getUTFLen(byte[] b, int s) {
- return ((b[s] & 0xff) << 8) + ((b[s + 1] & 0xff) << 0);
- }
-
- public static char toLowerCase(char c) {
- switch (c) {
- case 'A':
- return 'a';
- case 'B':
- return 'b';
- case 'C':
- return 'c';
- case 'D':
- return 'd';
- case 'E':
- return 'e';
- case 'F':
- return 'f';
- case 'G':
- return 'g';
- case 'H':
- return 'h';
- case 'I':
- return 'i';
- case 'J':
- return 'j';
- case 'K':
- return 'k';
- case 'L':
- return 'l';
- case 'M':
- return 'm';
- case 'N':
- return 'n';
- case 'O':
- return 'o';
- case 'P':
- return 'p';
- case 'Q':
- return 'q';
- case 'R':
- return 'r';
- case 'S':
- return 's';
- case 'T':
- return 't';
- case 'U':
- return 'u';
- case 'V':
- return 'v';
- case 'W':
- return 'w';
- case 'X':
- return 'x';
- case 'Y':
- return 'y';
- case 'Z':
- return 'z';
- case 'Ä':
- return 'ä';
- case 'Ǟ':
- return 'ǟ';
- case 'Ë':
- return 'ë';
- case 'Ḧ':
- return 'ḧ';
- case 'Ï':
- return 'ï';
- case 'Ḯ':
- return 'ḯ';
- case 'Ö':
- return 'ö';
- case 'Ȫ':
- return 'ȫ';
- case 'Ṏ':
- return 'ṏ';
- case 'Ü':
- return 'ü';
- case 'Ǖ':
- return 'ǖ';
- case 'Ǘ':
- return 'ǘ';
- case 'Ǚ':
- return 'ǚ';
- case 'Ǜ':
- return 'ǜ';
- case 'Ṳ':
- return 'ṳ';
- case 'Ṻ':
- return 'ṻ';
- case 'Ẅ':
- return 'ẅ';
- case 'Ẍ':
- return 'ẍ';
- case 'Ÿ':
- return 'ÿ';
- default:
- // since I probably missed some chars above
- // use Java to convert to lower case to be safe
- return Character.toLowerCase(c);
- }
- }
-
- public static void writeCharAsModifiedUTF8(char c, DataOutput dos)
- throws IOException {
-
- if (c >= 0x0000 && c <= 0x007F) {
- dos.writeByte(c);
- } else if (c <= 0x07FF) {
- dos.writeByte((byte) (0xC0 | ((c >> 6) & 0x3F)));
- dos.writeByte((byte) (0x80 | (c & 0x3F)));
- } else {
- dos.writeByte((byte) (0xE0 | ((c >> 12) & 0x0F)));
- dos.writeByte((byte) (0x80 | ((c >> 6) & 0x3F)));
- dos.writeByte((byte) (0x80 | (c & 0x3F)));
- }
- }
-
- public static void writeUTF8Len(int len, DataOutput dos) throws IOException {
- dos.write((len >>> 8) & 0xFF);
- dos.write((len >>> 0) & 0xFF);
- }
+ public static void writeUTF8Len(int len, DataOutput dos) throws IOException {
+ dos.write((len >>> 8) & 0xFF);
+ dos.write((len >>> 0) & 0xFF);
+ }
}
\ No newline at end of file
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
new file mode 100644
index 0000000..87d9b35
--- /dev/null
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/SerdeUtils.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.dataflow.common.util;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.BooleanPointable;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.FloatPointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.LongPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.BooleanSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.Integer64SerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+
+@SuppressWarnings("rawtypes")
+public class SerdeUtils {
+ public static ITypeTraits[] serdesToTypeTraits(ISerializerDeserializer[] serdes, int numSerdes) {
+ ITypeTraits[] typeTraits = new ITypeTraits[numSerdes];
+ for (int i = 0; i < numSerdes; i++) {
+ typeTraits[i] = serdeToTypeTrait(serdes[i]);
+ }
+ return typeTraits;
+ }
+
+ public static ITypeTraits serdeToTypeTrait(ISerializerDeserializer serde) {
+ if (serde instanceof IntegerSerializerDeserializer) {
+ return IntegerPointable.TYPE_TRAITS;
+ }
+ if (serde instanceof Integer64SerializerDeserializer) {
+ return LongPointable.TYPE_TRAITS;
+ }
+ if (serde instanceof FloatSerializerDeserializer) {
+ return FloatPointable.TYPE_TRAITS;
+ }
+ if (serde instanceof DoubleSerializerDeserializer) {
+ return DoublePointable.TYPE_TRAITS;
+ }
+ if (serde instanceof BooleanSerializerDeserializer) {
+ return BooleanPointable.TYPE_TRAITS;
+ }
+ return UTF8StringPointable.TYPE_TRAITS;
+ }
+
+ public static IBinaryComparator[] serdesToComparators(ISerializerDeserializer[] serdes, int numSerdes) {
+ IBinaryComparator[] comparators = new IBinaryComparator[numSerdes];
+ for (int i = 0; i < numSerdes; i++) {
+ comparators[i] = serdeToComparator(serdes[i]);
+ }
+ return comparators;
+ }
+
+ public static IBinaryComparator serdeToComparator(ISerializerDeserializer serde) {
+ IBinaryComparatorFactory f = serdeToComparatorFactory(serde);
+ return f.createBinaryComparator();
+ }
+
+ public static IBinaryComparatorFactory[] serdesToComparatorFactories(ISerializerDeserializer[] serdes, int numSerdes) {
+ IBinaryComparatorFactory[] comparatorsFactories = new IBinaryComparatorFactory[numSerdes];
+ for (int i = 0; i < numSerdes; i++) {
+ comparatorsFactories[i] = serdeToComparatorFactory(serdes[i]);
+ }
+ return comparatorsFactories;
+ }
+
+ public static IBinaryComparatorFactory serdeToComparatorFactory(ISerializerDeserializer serde) {
+ if (serde instanceof IntegerSerializerDeserializer) {
+ return PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ }
+ if (serde instanceof Integer64SerializerDeserializer) {
+ return PointableBinaryComparatorFactory.of(LongPointable.FACTORY);
+ }
+ if (serde instanceof FloatSerializerDeserializer) {
+ return PointableBinaryComparatorFactory.of(FloatPointable.FACTORY);
+ }
+ if (serde instanceof DoubleSerializerDeserializer) {
+ return PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ }
+ if (serde instanceof BooleanSerializerDeserializer) {
+ throw new UnsupportedOperationException("Binary comparator factory for Boolean not implemented.");
+ }
+ if (serde instanceof UTF8StringSerializerDeserializer) {
+ return PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ }
+ throw new UnsupportedOperationException("Binary comparator for + " + serde.toString() + " not implemented.");
+ }
+}
diff --git a/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java
new file mode 100644
index 0000000..14c4d66
--- /dev/null
+++ b/hyracks-dataflow-common/src/main/java/edu/uci/ics/hyracks/dataflow/common/util/TupleUtils.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.dataflow.common.util;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+@SuppressWarnings("rawtypes")
+public class TupleUtils {
+ @SuppressWarnings("unchecked")
+ public static void createTuple(ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple, ISerializerDeserializer[] fieldSerdes, final Object... fields) throws HyracksDataException {
+ DataOutput dos = tupleBuilder.getDataOutput();
+ tupleBuilder.reset();
+ int numFields = Math.min(tupleBuilder.getFieldEndOffsets().length, fields.length);
+ for (int i = 0; i < numFields; i++) {
+ fieldSerdes[i].serialize(fields[i], dos);
+ tupleBuilder.addFieldEndOffset();
+ }
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ }
+
+ public static ITupleReference createTuple(ISerializerDeserializer[] fieldSerdes, final Object... fields) throws HyracksDataException {
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fields.length);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ createTuple(tupleBuilder, tuple, fieldSerdes, fields);
+ return tuple;
+ }
+
+ public static void createIntegerTuple(ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple,
+ final int... fields) throws HyracksDataException {
+ DataOutput dos = tupleBuilder.getDataOutput();
+ tupleBuilder.reset();
+ for (final int i : fields) {
+ IntegerSerializerDeserializer.INSTANCE.serialize(i, dos);
+ tupleBuilder.addFieldEndOffset();
+ }
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ }
+
+ public static ITupleReference createIntegerTuple(final int... fields) throws HyracksDataException {
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fields.length);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ createIntegerTuple(tupleBuilder, tuple, fields);
+ return tuple;
+ }
+
+ public static String printTuple(ITupleReference tuple,
+ ISerializerDeserializer[] fields) throws HyracksDataException {
+ StringBuilder strBuilder = new StringBuilder();
+ int numPrintFields = Math.min(tuple.getFieldCount(), fields.length);
+ for (int i = 0; i < numPrintFields; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(
+ tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Object o = fields[i].deserialize(dataIn);
+ strBuilder.append(o.toString());
+ if (i != fields.length - 1) {
+ strBuilder.append(" ");
+ }
+ }
+ return strBuilder.toString();
+ }
+}
diff --git a/hyracks-dataflow-hadoop/.classpath b/hyracks-dataflow-hadoop/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-dataflow-hadoop/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-dataflow-hadoop/.project b/hyracks-dataflow-hadoop/.project
deleted file mode 100644
index d6edecf..0000000
--- a/hyracks-dataflow-hadoop/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-dataflow-hadoop</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-dataflow-hadoop/.settings/org.eclipse.jdt.core.prefs b/hyracks-dataflow-hadoop/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 0272d6e..0000000
--- a/hyracks-dataflow-hadoop/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,274 +0,0 @@
-#Thu Jun 02 13:11:16 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.codeComplete.argumentPrefixes=
-org.eclipse.jdt.core.codeComplete.argumentSuffixes=
-org.eclipse.jdt.core.codeComplete.fieldPrefixes=
-org.eclipse.jdt.core.codeComplete.fieldSuffixes=
-org.eclipse.jdt.core.codeComplete.localPrefixes=
-org.eclipse.jdt.core.codeComplete.localSuffixes=
-org.eclipse.jdt.core.codeComplete.staticFieldPrefixes=
-org.eclipse.jdt.core.codeComplete.staticFieldSuffixes=
-org.eclipse.jdt.core.codeComplete.staticFinalFieldPrefixes=
-org.eclipse.jdt.core.codeComplete.staticFinalFieldSuffixes=
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=48
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=48
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=true
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true
-org.eclipse.jdt.core.formatter.comment.format_line_comments=true
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=true
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_member=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=120
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
diff --git a/hyracks-dataflow-hadoop/.settings/org.maven.ide.eclipse.prefs b/hyracks-dataflow-hadoop/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 9e0c887..0000000
--- a/hyracks-dataflow-hadoop/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Fri Aug 13 04:07:00 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-dataflow-hadoop/pom.xml b/hyracks-dataflow-hadoop/pom.xml
index c8ef50e..e3f9b1e 100644
--- a/hyracks-dataflow-hadoop/pom.xml
+++ b/hyracks-dataflow-hadoop/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-hadoop</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,14 +27,14 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
@@ -54,7 +54,7 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopMapperOperatorDescriptor.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopMapperOperatorDescriptor.java
index 26501ed..0ea56c4 100644
--- a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopMapperOperatorDescriptor.java
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopMapperOperatorDescriptor.java
@@ -40,7 +40,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.SerializingDataWriter;
import edu.uci.ics.hyracks.dataflow.hadoop.util.DatatypeHelper;
@@ -379,7 +378,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
JobConf conf = getJobConf();
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReadOperatorDescriptor.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReadOperatorDescriptor.java
index b84d576..2841509 100644
--- a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReadOperatorDescriptor.java
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReadOperatorDescriptor.java
@@ -37,7 +37,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
@@ -143,7 +142,7 @@
@SuppressWarnings("deprecation")
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
throws HyracksDataException {
return new AbstractUnaryOutputSourceOperatorNodePushable() {
@@ -192,7 +191,7 @@
RecordDescriptor outputRecordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor(
(Class<? extends Writable>) hadoopRecordReader.createKey().getClass(),
(Class<? extends Writable>) hadoopRecordReader.createValue().getClass());
- int nFields = outputRecordDescriptor.getFields().length;
+ int nFields = outputRecordDescriptor.getFieldCount();
ArrayTupleBuilder tb = new ArrayTupleBuilder(nFields);
writer.open();
try {
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReducerOperatorDescriptor.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReducerOperatorDescriptor.java
index 0af0596..73a8c16 100644
--- a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReducerOperatorDescriptor.java
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/HadoopReducerOperatorDescriptor.java
@@ -23,14 +23,13 @@
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;
+import org.apache.hadoop.mapred.Counters.Counter;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.RawKeyValueIterator;
import org.apache.hadoop.mapred.Reducer;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.hadoop.mapred.Counters.Counter;
import org.apache.hadoop.mapreduce.JobContext;
-import org.apache.hadoop.mapreduce.StatusReporter;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.util.Progress;
import org.apache.hadoop.util.ReflectionUtils;
@@ -44,7 +43,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.hadoop.data.KeyComparatorFactory;
import edu.uci.ics.hyracks.dataflow.hadoop.data.RawComparingComparatorFactory;
@@ -348,7 +346,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
try {
if (this.comparatorFactory == null) {
@@ -395,8 +393,9 @@
RecordDescriptor recordDescriptor = null;
try {
if (classFactory == null) {
- recordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor((Class<? extends Writable>) Class
- .forName(outputKeyClassName), (Class<? extends Writable>) Class.forName(outputValueClassName));
+ recordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor(
+ (Class<? extends Writable>) Class.forName(outputKeyClassName),
+ (Class<? extends Writable>) Class.forName(outputValueClassName));
} else {
recordDescriptor = DatatypeHelper.createKeyValueRecordDescriptor(
(Class<? extends Writable>) classFactory.loadClass(outputKeyClassName),
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HadoopHelper.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HadoopHelper.java
new file mode 100644
index 0000000..e8b6e8b
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HadoopHelper.java
@@ -0,0 +1,268 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.Partitioner;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.util.ReflectionUtils;
+
+import edu.uci.ics.hyracks.api.context.IHyracksCommonContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.hadoop.data.HadoopNewPartitionerTuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.hadoop.data.WritableComparingBinaryComparatorFactory;
+import edu.uci.ics.hyracks.dataflow.hadoop.util.DatatypeHelper;
+
+public class HadoopHelper {
+ public static final int KEY_FIELD_INDEX = 0;
+ public static final int VALUE_FIELD_INDEX = 1;
+ public static final int BLOCKID_FIELD_INDEX = 2;
+ private static final int[] KEY_SORT_FIELDS = new int[] { 0 };
+
+ private MarshalledWritable<Configuration> mConfig;
+ private Configuration config;
+ private Job job;
+
+ public HadoopHelper(MarshalledWritable<Configuration> mConfig) throws HyracksDataException {
+ this.mConfig = mConfig;
+ ClassLoader ctxCL = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ config = mConfig.get();
+ config.setClassLoader(getClass().getClassLoader());
+ job = new Job(config);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ } finally {
+ Thread.currentThread().setContextClassLoader(ctxCL);
+ }
+ }
+
+ public RecordDescriptor getMapOutputRecordDescriptor() throws HyracksDataException {
+ try {
+ return new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ DatatypeHelper.createSerializerDeserializer((Class<? extends Writable>) job
+ .getMapOutputKeyClass()),
+ DatatypeHelper.createSerializerDeserializer((Class<? extends Writable>) job
+ .getMapOutputValueClass()), IntegerSerializerDeserializer.INSTANCE });
+
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public RecordDescriptor getMapOutputRecordDescriptorWithoutExtraFields() throws HyracksDataException {
+ try {
+ return new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ DatatypeHelper.createSerializerDeserializer((Class<? extends Writable>) job
+ .getMapOutputKeyClass()),
+ DatatypeHelper.createSerializerDeserializer((Class<? extends Writable>) job
+ .getMapOutputValueClass()) });
+
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public TaskAttemptContext createTaskAttemptContext(TaskAttemptID taId) {
+ ClassLoader ctxCL = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(config.getClassLoader());
+ return new TaskAttemptContext(config, taId);
+ } finally {
+ Thread.currentThread().setContextClassLoader(ctxCL);
+ }
+ }
+
+ public JobContext createJobContext() {
+ ClassLoader ctxCL = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(config.getClassLoader());
+ return new JobContext(config, null);
+ } finally {
+ Thread.currentThread().setContextClassLoader(ctxCL);
+ }
+ }
+
+ public <K1, V1, K2, V2> Mapper<K1, V1, K2, V2> getMapper() throws HyracksDataException {
+ try {
+ return (Mapper<K1, V1, K2, V2>) HadoopTools.newInstance(job.getMapperClass());
+ } catch (ClassNotFoundException e) {
+ throw new HyracksDataException(e);
+ } catch (InstantiationException e) {
+ throw new HyracksDataException(e);
+ } catch (IllegalAccessException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public <K2, V2, K3, V3> Reducer<K2, V2, K3, V3> getReducer() throws HyracksDataException {
+ try {
+ return (Reducer<K2, V2, K3, V3>) HadoopTools.newInstance(job.getReducerClass());
+ } catch (ClassNotFoundException e) {
+ throw new HyracksDataException(e);
+ } catch (InstantiationException e) {
+ throw new HyracksDataException(e);
+ } catch (IllegalAccessException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public <K2, V2> Reducer<K2, V2, K2, V2> getCombiner() throws HyracksDataException {
+ try {
+ return (Reducer<K2, V2, K2, V2>) HadoopTools.newInstance(job.getCombinerClass());
+ } catch (ClassNotFoundException e) {
+ throw new HyracksDataException(e);
+ } catch (InstantiationException e) {
+ throw new HyracksDataException(e);
+ } catch (IllegalAccessException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public <K, V> InputFormat<K, V> getInputFormat() throws HyracksDataException {
+ try {
+ return (InputFormat<K, V>) ReflectionUtils.newInstance(job.getInputFormatClass(), config);
+ } catch (ClassNotFoundException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public <K, V> List<InputSplit> getInputSplits() throws HyracksDataException {
+ ClassLoader ctxCL = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ InputFormat<K, V> fmt = getInputFormat();
+ JobContext jCtx = new JobContext(config, null);
+ try {
+ return fmt.getSplits(jCtx);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ } finally {
+ Thread.currentThread().setContextClassLoader(ctxCL);
+ }
+ }
+
+ public IBinaryComparatorFactory[] getSortComparatorFactories() {
+ WritableComparingBinaryComparatorFactory comparatorFactory = new WritableComparingBinaryComparatorFactory(job
+ .getSortComparator().getClass());
+
+ return new IBinaryComparatorFactory[] { comparatorFactory };
+ }
+
+ public IBinaryComparatorFactory[] getGroupingComparatorFactories() {
+ WritableComparingBinaryComparatorFactory comparatorFactory = new WritableComparingBinaryComparatorFactory(job
+ .getGroupingComparator().getClass());
+
+ return new IBinaryComparatorFactory[] { comparatorFactory };
+ }
+
+ public RawComparator<?> getRawGroupingComparator() {
+ return job.getGroupingComparator();
+ }
+
+ public int getSortFrameLimit(IHyracksCommonContext ctx) {
+ int sortMemory = job.getConfiguration().getInt("io.sort.mb", 100);
+ return (int) (((long) sortMemory * 1024 * 1024) / ctx.getFrameSize());
+ }
+
+ public Job getJob() {
+ return job;
+ }
+
+ public MarshalledWritable<Configuration> getMarshalledConfiguration() {
+ return mConfig;
+ }
+
+ public Configuration getConfiguration() {
+ return config;
+ }
+
+ public ITuplePartitionComputerFactory getTuplePartitionComputer() throws HyracksDataException {
+ int nReducers = job.getNumReduceTasks();
+ try {
+ return new HadoopNewPartitionerTuplePartitionComputerFactory<Writable, Writable>(
+ (Class<? extends Partitioner<Writable, Writable>>) job.getPartitionerClass(),
+ (ISerializerDeserializer<Writable>) DatatypeHelper
+ .createSerializerDeserializer((Class<? extends Writable>) job.getMapOutputKeyClass()),
+ (ISerializerDeserializer<Writable>) DatatypeHelper
+ .createSerializerDeserializer((Class<? extends Writable>) job.getMapOutputValueClass()));
+ } catch (ClassNotFoundException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public int[] getSortFields() {
+ return KEY_SORT_FIELDS;
+ }
+
+ public <K> ISerializerDeserializer<K> getMapOutputKeySerializerDeserializer() {
+ return (ISerializerDeserializer<K>) DatatypeHelper.createSerializerDeserializer((Class<? extends Writable>) job
+ .getMapOutputKeyClass());
+ }
+
+ public <V> ISerializerDeserializer<V> getMapOutputValueSerializerDeserializer() {
+ return (ISerializerDeserializer<V>) DatatypeHelper.createSerializerDeserializer((Class<? extends Writable>) job
+ .getMapOutputValueClass());
+ }
+
+ public FileSystem getFilesystem() throws HyracksDataException {
+ try {
+ return FileSystem.get(config);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public <K, V> OutputFormat<K, V> getOutputFormat() throws HyracksDataException {
+ try {
+ return (OutputFormat<K, V>) ReflectionUtils.newInstance(job.getOutputFormatClass(), config);
+ } catch (ClassNotFoundException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public boolean hasCombiner() throws HyracksDataException {
+ try {
+ return job.getCombinerClass() != null;
+ } catch (ClassNotFoundException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HadoopTools.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HadoopTools.java
new file mode 100644
index 0000000..7e4d67f
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HadoopTools.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+public class HadoopTools {
+ public static Object newInstance(String className) throws ClassNotFoundException, InstantiationException,
+ IllegalAccessException {
+ ClassLoader ctxCL = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(HadoopTools.class.getClassLoader());
+ Class<?> clazz = Class.forName(className, true, HadoopTools.class.getClassLoader());
+ return newInstance(clazz);
+ } finally {
+ Thread.currentThread().setContextClassLoader(ctxCL);
+ }
+ }
+
+ public static Object newInstance(Class<?> clazz) throws InstantiationException, IllegalAccessException {
+ return clazz.newInstance();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HashPartitioningShuffleConnectorDescriptor.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HashPartitioningShuffleConnectorDescriptor.java
new file mode 100644
index 0000000..7c6bf86
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/HashPartitioningShuffleConnectorDescriptor.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.util.BitSet;
+
+import org.apache.hadoop.conf.Configuration;
+
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.comm.IPartitionCollector;
+import edu.uci.ics.hyracks.api.comm.IPartitionWriterFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
+import edu.uci.ics.hyracks.dataflow.std.collectors.PartitionCollector;
+import edu.uci.ics.hyracks.dataflow.std.connectors.PartitionDataWriter;
+
+public class HashPartitioningShuffleConnectorDescriptor extends AbstractMToNConnectorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ private final MarshalledWritable<Configuration> mConfig;
+
+ public HashPartitioningShuffleConnectorDescriptor(JobSpecification spec, MarshalledWritable<Configuration> mConfig) {
+ super(spec);
+ this.mConfig = mConfig;
+ }
+
+ @Override
+ public IFrameWriter createPartitioner(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
+ IPartitionWriterFactory edwFactory, int index, int nProducerPartitions, int nConsumerPartitions)
+ throws HyracksDataException {
+ HadoopHelper helper = new HadoopHelper(mConfig);
+ ITuplePartitionComputerFactory tpcf = helper.getTuplePartitionComputer();
+ return new PartitionDataWriter(ctx, nConsumerPartitions, edwFactory, recordDesc, tpcf.createPartitioner());
+ }
+
+ @Override
+ public IPartitionCollector createPartitionCollector(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
+ int receiverIndex, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
+ BitSet expectedPartitions = new BitSet();
+ expectedPartitions.set(0, nProducerPartitions);
+ NonDeterministicChannelReader channelReader = new NonDeterministicChannelReader(nProducerPartitions,
+ expectedPartitions);
+ IFrameReader frameReader = new ShuffleFrameReader(ctx, channelReader, mConfig);
+ return new PartitionCollector(ctx, getConnectorId(), receiverIndex, expectedPartitions, frameReader,
+ channelReader);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/IInputSplitProvider.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/IInputSplitProvider.java
new file mode 100644
index 0000000..1545c06
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/IInputSplitProvider.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import org.apache.hadoop.mapreduce.InputSplit;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IInputSplitProvider {
+ public InputSplit next() throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/IInputSplitProviderFactory.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/IInputSplitProviderFactory.java
new file mode 100644
index 0000000..73588ab
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/IInputSplitProviderFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IInputSplitProviderFactory extends Serializable {
+ public IInputSplitProvider createInputSplitProvider(int id) throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/InputFileSplit.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/InputFileSplit.java
new file mode 100644
index 0000000..b37084e
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/InputFileSplit.java
@@ -0,0 +1,92 @@
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.lib.input.FileSplit;
+
+public class InputFileSplit extends InputSplit implements Writable {
+ private Path file;
+ private long start;
+ private long length;
+ private int blockId;
+ private String[] hosts;
+ private long scheduleTime;
+
+ public InputFileSplit() {
+ }
+
+ public InputFileSplit(int blockId, Path file, long start, long length, String[] hosts, long schedule_time) {
+ this.blockId = blockId;
+ this.file = file;
+ this.start = start;
+ this.length = length;
+ this.hosts = hosts;
+ this.scheduleTime = schedule_time;
+ }
+
+ public int blockId() {
+ return blockId;
+ }
+
+ public long scheduleTime() {
+ return this.scheduleTime;
+ }
+
+ public Path getPath() {
+ return file;
+ }
+
+ /** The position of the first byte in the file to process. */
+ public long getStart() {
+ return start;
+ }
+
+ /** The number of bytes in the file to process. */
+ @Override
+ public long getLength() {
+ return length;
+ }
+
+ @Override
+ public String toString() {
+ return file + ":" + start + "+" + length;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ Text.writeString(out, file.toString());
+ out.writeLong(start);
+ out.writeLong(length);
+ out.writeInt(blockId);
+ out.writeLong(this.scheduleTime);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ file = new Path(Text.readString(in));
+ start = in.readLong();
+ length = in.readLong();
+ hosts = null;
+ this.blockId = in.readInt();
+ this.scheduleTime = in.readLong();
+ }
+
+ @Override
+ public String[] getLocations() throws IOException {
+ if (this.hosts == null) {
+ return new String[] {};
+ } else {
+ return this.hosts;
+ }
+ }
+
+ public FileSplit toFileSplit() {
+ return new FileSplit(file, start, length, hosts);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/KVIterator.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/KVIterator.java
new file mode 100644
index 0000000..a00fa7f
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/KVIterator.java
@@ -0,0 +1,109 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.hadoop.io.DataInputBuffer;
+import org.apache.hadoop.mapred.RawKeyValueIterator;
+import org.apache.hadoop.util.Progress;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+
+public class KVIterator implements RawKeyValueIterator {
+ private final HadoopHelper helper;
+ private FrameTupleAccessor accessor;
+ private DataInputBuffer kBuffer;
+ private DataInputBuffer vBuffer;
+ private List<ByteBuffer> buffers;
+ private int bSize;
+ private int bPtr;
+ private int tIdx;
+ private boolean eog;
+
+ public KVIterator(IHyracksTaskContext ctx, HadoopHelper helper, RecordDescriptor recordDescriptor) {
+ this.helper = helper;
+ accessor = new FrameTupleAccessor(ctx.getFrameSize(), recordDescriptor);
+ kBuffer = new DataInputBuffer();
+ vBuffer = new DataInputBuffer();
+ }
+
+ void reset(List<ByteBuffer> buffers, int bSize) {
+ this.buffers = buffers;
+ this.bSize = bSize;
+ bPtr = 0;
+ tIdx = 0;
+ eog = false;
+ if (bSize > 0) {
+ accessor.reset(buffers.get(0));
+ tIdx = -1;
+ } else {
+ eog = true;
+ }
+ }
+
+ @Override
+ public DataInputBuffer getKey() throws IOException {
+ return kBuffer;
+ }
+
+ @Override
+ public DataInputBuffer getValue() throws IOException {
+ return vBuffer;
+ }
+
+ @Override
+ public boolean next() throws IOException {
+ while (true) {
+ if (eog) {
+ return false;
+ }
+ ++tIdx;
+ if (accessor.getTupleCount() <= tIdx) {
+ ++bPtr;
+ if (bPtr >= bSize) {
+ eog = true;
+ continue;
+ }
+ tIdx = -1;
+ accessor.reset(buffers.get(bPtr));
+ continue;
+ }
+ kBuffer.reset(accessor.getBuffer().array(),
+ FrameUtils.getAbsoluteFieldStartOffset(accessor, tIdx, helper.KEY_FIELD_INDEX),
+ accessor.getFieldLength(tIdx, helper.KEY_FIELD_INDEX));
+ vBuffer.reset(accessor.getBuffer().array(),
+ FrameUtils.getAbsoluteFieldStartOffset(accessor, tIdx, helper.VALUE_FIELD_INDEX),
+ accessor.getFieldLength(tIdx, helper.VALUE_FIELD_INDEX));
+ break;
+ }
+ return true;
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @Override
+ public Progress getProgress() {
+ return null;
+ }
+}
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MapperOperatorDescriptor.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MapperOperatorDescriptor.java
new file mode 100644
index 0000000..4a61296
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MapperOperatorDescriptor.java
@@ -0,0 +1,271 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.hadoop.mapreduce.InputSplit;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.RecordReader;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
+import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortRunGenerator;
+import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortRunMerger;
+
+public class MapperOperatorDescriptor<K1 extends Writable, V1 extends Writable, K2 extends Writable, V2 extends Writable>
+ extends AbstractSingleActivityOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+ private final int jobId;
+ private final MarshalledWritable<Configuration> config;
+ private final IInputSplitProviderFactory factory;
+
+ public MapperOperatorDescriptor(JobSpecification spec, int jobId, MarshalledWritable<Configuration> config,
+ IInputSplitProviderFactory factory) throws HyracksDataException {
+ super(spec, 0, 1);
+ this.jobId = jobId;
+ this.config = config;
+ this.factory = factory;
+ HadoopHelper helper = new HadoopHelper(config);
+ recordDescriptors[0] = helper.getMapOutputRecordDescriptor();
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions)
+ throws HyracksDataException {
+ final HadoopHelper helper = new HadoopHelper(config);
+ final Configuration conf = helper.getConfiguration();
+ final Mapper<K1, V1, K2, V2> mapper = helper.getMapper();
+ final InputFormat<K1, V1> inputFormat = helper.getInputFormat();
+ final IInputSplitProvider isp = factory.createInputSplitProvider(partition);
+ final TaskAttemptID taId = new TaskAttemptID("foo", jobId, true, partition, 0);
+ final TaskAttemptContext taskAttemptContext = helper.createTaskAttemptContext(taId);
+
+ final int framesLimit = helper.getSortFrameLimit(ctx);
+ final IBinaryComparatorFactory[] comparatorFactories = helper.getSortComparatorFactories();
+
+ class SortingRecordWriter extends RecordWriter<K2, V2> {
+ private final ArrayTupleBuilder tb;
+ private final ByteBuffer frame;
+ private final FrameTupleAppender fta;
+ private ExternalSortRunGenerator runGen;
+ private int blockId;
+
+ public SortingRecordWriter() throws HyracksDataException {
+ tb = new ArrayTupleBuilder(2);
+ frame = ctx.allocateFrame();
+ fta = new FrameTupleAppender(ctx.getFrameSize());
+ fta.reset(frame, true);
+ }
+
+ public void initBlock(int blockId) throws HyracksDataException {
+ runGen = new ExternalSortRunGenerator(ctx, new int[] { 0 }, null, comparatorFactories,
+ helper.getMapOutputRecordDescriptorWithoutExtraFields(), framesLimit);
+ this.blockId = blockId;
+ }
+
+ @Override
+ public void close(TaskAttemptContext arg0) throws IOException, InterruptedException {
+ }
+
+ @Override
+ public void write(K2 key, V2 value) throws IOException, InterruptedException {
+ DataOutput dos = tb.getDataOutput();
+ tb.reset();
+ key.write(dos);
+ tb.addFieldEndOffset();
+ value.write(dos);
+ tb.addFieldEndOffset();
+ if (!fta.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ runGen.nextFrame(frame);
+ fta.reset(frame, true);
+ if (!fta.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ public void sortAndFlushBlock(final IFrameWriter writer) throws HyracksDataException {
+ if (fta.getTupleCount() > 0) {
+ runGen.nextFrame(frame);
+ fta.reset(frame, true);
+ }
+ runGen.close();
+ IFrameWriter delegatingWriter = new IFrameWriter() {
+ private final FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ private final ByteBuffer outFrame = ctx.allocateFrame();
+ private final FrameTupleAccessor fta = new FrameTupleAccessor(ctx.getFrameSize(),
+ helper.getMapOutputRecordDescriptorWithoutExtraFields());
+ private final ArrayTupleBuilder tb = new ArrayTupleBuilder(3);
+
+ @Override
+ public void open() throws HyracksDataException {
+ appender.reset(outFrame, true);
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ fta.reset(buffer);
+ int n = fta.getTupleCount();
+ for (int i = 0; i < n; ++i) {
+ tb.reset();
+ tb.addField(fta, i, 0);
+ tb.addField(fta, i, 1);
+ try {
+ tb.getDataOutput().writeInt(blockId);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ tb.addFieldEndOffset();
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ FrameUtils.flushFrame(outFrame, writer);
+ appender.reset(outFrame, true);
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (appender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(outFrame, writer);
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
+ };
+ if (helper.hasCombiner()) {
+ Reducer<K2, V2, K2, V2> combiner = helper.getCombiner();
+ TaskAttemptID ctaId = new TaskAttemptID("foo", jobId, true, partition, 0);
+ TaskAttemptContext ctaskAttemptContext = helper.createTaskAttemptContext(taId);
+ final IFrameWriter outputWriter = delegatingWriter;
+ RecordWriter<K2, V2> recordWriter = new RecordWriter<K2, V2>() {
+ private final FrameTupleAppender fta = new FrameTupleAppender(ctx.getFrameSize());
+ private final ByteBuffer buffer = ctx.allocateFrame();
+ private final ArrayTupleBuilder tb = new ArrayTupleBuilder(2);
+
+ {
+ fta.reset(buffer, true);
+ outputWriter.open();
+ }
+
+ @Override
+ public void write(K2 key, V2 value) throws IOException, InterruptedException {
+ DataOutput dos = tb.getDataOutput();
+ tb.reset();
+ key.write(dos);
+ tb.addFieldEndOffset();
+ value.write(dos);
+ tb.addFieldEndOffset();
+ if (!fta.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ FrameUtils.flushFrame(buffer, outputWriter);
+ fta.reset(buffer, true);
+ if (!fta.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ @Override
+ public void close(TaskAttemptContext context) throws IOException, InterruptedException {
+ if (fta.getTupleCount() > 0) {
+ FrameUtils.flushFrame(buffer, outputWriter);
+ outputWriter.close();
+ }
+ }
+ };
+ delegatingWriter = new ReduceWriter<K2, V2, K2, V2>(ctx, helper,
+ new int[] { HadoopHelper.KEY_FIELD_INDEX }, helper.getGroupingComparatorFactories(),
+ helper.getMapOutputRecordDescriptorWithoutExtraFields(), combiner, recordWriter, ctaId,
+ ctaskAttemptContext);
+ }
+ IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ ExternalSortRunMerger merger = new ExternalSortRunMerger(ctx, runGen.getFrameSorter(),
+ runGen.getRuns(), new int[] { 0 }, comparators,
+ helper.getMapOutputRecordDescriptorWithoutExtraFields(), framesLimit, delegatingWriter);
+ merger.process();
+ }
+ }
+
+ return new AbstractUnaryOutputSourceOperatorNodePushable() {
+ @Override
+ public void initialize() throws HyracksDataException {
+ writer.open();
+ try {
+ SortingRecordWriter recordWriter = new SortingRecordWriter();
+ InputSplit split = null;
+ int blockId = 0;
+ while ((split = isp.next()) != null) {
+ try {
+ RecordReader<K1, V1> recordReader = inputFormat.createRecordReader(split,
+ taskAttemptContext);
+ ClassLoader ctxCL = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
+ recordReader.initialize(split, taskAttemptContext);
+ } finally {
+ Thread.currentThread().setContextClassLoader(ctxCL);
+ }
+ recordWriter.initBlock(blockId);
+ Mapper<K1, V1, K2, V2>.Context mCtx = mapper.new Context(conf, taId, recordReader,
+ recordWriter, null, null, split);
+ mapper.run(mCtx);
+ recordReader.close();
+ recordWriter.sortAndFlushBlock(writer);
+ ++blockId;
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ } finally {
+ writer.close();
+ }
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MarshalledWritable.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MarshalledWritable.java
new file mode 100644
index 0000000..be05f22
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/MarshalledWritable.java
@@ -0,0 +1,51 @@
+/*
+ * Copyright 2009-2010 University of California, Irvine
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.Serializable;
+
+import org.apache.hadoop.io.Writable;
+
+public class MarshalledWritable<T extends Writable> implements Serializable {
+ private static final long serialVersionUID = 1L;
+
+ private byte[] bytes;
+
+ public MarshalledWritable() {
+ }
+
+ public void set(T o) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ dos.writeUTF(o.getClass().getName());
+ o.write(dos);
+ dos.close();
+ bytes = baos.toByteArray();
+ }
+
+ public T get() throws Exception {
+ ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
+ DataInputStream dis = new DataInputStream(bais);
+ String className = dis.readUTF();
+ T o = (T) HadoopTools.newInstance(className);
+ o.readFields(dis);
+ return o;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReduceWriter.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReduceWriter.java
new file mode 100644
index 0000000..33d58af
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReduceWriter.java
@@ -0,0 +1,187 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+
+public class ReduceWriter<K2, V2, K3, V3> implements IFrameWriter {
+ private final IHyracksTaskContext ctx;
+ private final HadoopHelper helper;
+ private final int[] groupFields;
+ private final FrameTupleAccessor accessor0;
+ private final FrameTupleAccessor accessor1;
+ private final ByteBuffer copyFrame;
+ private final IBinaryComparator[] comparators;
+ private final KVIterator kvi;
+ private final Reducer<K2, V2, K3, V3> reducer;
+ private final RecordWriter<K3, V3> recordWriter;
+ private final TaskAttemptID taId;
+ private final TaskAttemptContext taskAttemptContext;
+
+ private boolean first;
+ private boolean groupStarted;
+ private List<ByteBuffer> group;
+ private int bPtr;
+ private FrameTupleAppender fta;
+ private Counter keyCounter;
+ private Counter valueCounter;
+
+ public ReduceWriter(IHyracksTaskContext ctx, HadoopHelper helper, int[] groupFields,
+ IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor,
+ Reducer<K2, V2, K3, V3> reducer, RecordWriter<K3, V3> recordWriter, TaskAttemptID taId,
+ TaskAttemptContext taskAttemptContext) {
+ this.ctx = ctx;
+ this.helper = helper;
+ this.groupFields = groupFields;
+ accessor0 = new FrameTupleAccessor(ctx.getFrameSize(), recordDescriptor);
+ accessor1 = new FrameTupleAccessor(ctx.getFrameSize(), recordDescriptor);
+ copyFrame = ctx.allocateFrame();
+ accessor1.reset(copyFrame);
+ comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ this.reducer = reducer;
+ this.recordWriter = recordWriter;
+ this.taId = taId;
+ this.taskAttemptContext = taskAttemptContext;
+
+ kvi = new KVIterator(ctx, helper, recordDescriptor);
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ first = true;
+ groupStarted = false;
+ group = new ArrayList<ByteBuffer>();
+ bPtr = 0;
+ group.add(ctx.allocateFrame());
+ fta = new FrameTupleAppender(ctx.getFrameSize());
+ keyCounter = new Counter() {
+ };
+ valueCounter = new Counter() {
+ };
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor0.reset(buffer);
+ int nTuples = accessor0.getTupleCount();
+ for (int i = 0; i < nTuples; ++i) {
+ if (first) {
+ groupInit();
+ first = false;
+ } else {
+ if (i == 0) {
+ switchGroupIfRequired(accessor1, accessor1.getTupleCount() - 1, accessor0, i);
+ } else {
+ switchGroupIfRequired(accessor0, i - 1, accessor0, i);
+ }
+ }
+ accumulate(accessor0, i);
+ }
+ FrameUtils.copy(buffer, copyFrame);
+ }
+
+ private void accumulate(FrameTupleAccessor accessor, int tIndex) {
+ if (!fta.append(accessor, tIndex)) {
+ ++bPtr;
+ if (group.size() <= bPtr) {
+ group.add(ctx.allocateFrame());
+ }
+ fta.reset(group.get(bPtr), true);
+ if (!fta.append(accessor, tIndex)) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ private void switchGroupIfRequired(FrameTupleAccessor prevTupleAccessor, int prevTupleIndex,
+ FrameTupleAccessor currTupleAccessor, int currTupleIndex) throws HyracksDataException {
+ if (!sameGroup(prevTupleAccessor, prevTupleIndex, currTupleAccessor, currTupleIndex)) {
+ reduce();
+ groupInit();
+ }
+ }
+
+ private void groupInit() {
+ groupStarted = true;
+ bPtr = 0;
+ fta.reset(group.get(0), true);
+ }
+
+ private void reduce() throws HyracksDataException {
+ kvi.reset(group, bPtr + 1);
+ try {
+ Reducer<K2, V2, K3, V3>.Context rCtx = reducer.new Context(helper.getConfiguration(), taId, kvi,
+ keyCounter, valueCounter, recordWriter, null, null,
+ (RawComparator<K2>) helper.getRawGroupingComparator(), (Class<K2>) helper.getJob()
+ .getMapOutputKeyClass(), (Class<V2>) helper.getJob().getMapOutputValueClass());
+ reducer.run(rCtx);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ groupStarted = false;
+ }
+
+ private boolean sameGroup(FrameTupleAccessor a1, int t1Idx, FrameTupleAccessor a2, int t2Idx) {
+ for (int i = 0; i < comparators.length; ++i) {
+ int fIdx = groupFields[i];
+ int s1 = a1.getTupleStartOffset(t1Idx) + a1.getFieldSlotsLength() + a1.getFieldStartOffset(t1Idx, fIdx);
+ int l1 = a1.getFieldLength(t1Idx, fIdx);
+ int s2 = a2.getTupleStartOffset(t2Idx) + a2.getFieldSlotsLength() + a2.getFieldStartOffset(t2Idx, fIdx);
+ int l2 = a2.getFieldLength(t2Idx, fIdx);
+ if (comparators[i].compare(a1.getBuffer().array(), s1, l1, a2.getBuffer().array(), s2, l2) != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ if (groupStarted) {
+ reduce();
+ }
+ try {
+ recordWriter.close(taskAttemptContext);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReducerOperatorDescriptor.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReducerOperatorDescriptor.java
new file mode 100644
index 0000000..bbb0d74
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ReducerOperatorDescriptor.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.RecordWriter;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
+
+public class ReducerOperatorDescriptor<K2 extends Writable, V2 extends Writable, K3 extends Writable, V3 extends Writable>
+ extends AbstractSingleActivityOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ private final int jobId;
+
+ private MarshalledWritable<Configuration> mConfig;
+
+ public ReducerOperatorDescriptor(JobSpecification spec, int jobId, MarshalledWritable<Configuration> mConfig) {
+ super(spec, 1, 0);
+ this.jobId = jobId;
+ this.mConfig = mConfig;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ final HadoopHelper helper = new HadoopHelper(mConfig);
+ final Reducer<K2, V2, K3, V3> reducer = helper.getReducer();
+ final RecordDescriptor recordDescriptor = helper.getMapOutputRecordDescriptor();
+ final int[] groupFields = helper.getSortFields();
+ IBinaryComparatorFactory[] groupingComparators = helper.getGroupingComparatorFactories();
+
+ final TaskAttemptID taId = new TaskAttemptID("foo", jobId, false, partition, 0);
+ final TaskAttemptContext taskAttemptContext = helper.createTaskAttemptContext(taId);
+ final RecordWriter recordWriter;
+ try {
+ recordWriter = helper.getOutputFormat().getRecordWriter(taskAttemptContext);
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+
+ final ReduceWriter<K2, V2, K3, V3> rw = new ReduceWriter<K2, V2, K3, V3>(ctx, helper, groupFields,
+ groupingComparators, recordDescriptor, reducer, recordWriter, taId, taskAttemptContext);
+
+ return new AbstractUnaryInputSinkOperatorNodePushable() {
+ @Override
+ public void open() throws HyracksDataException {
+ rw.open();
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ rw.nextFrame(buffer);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ rw.close();
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ShuffleFrameReader.java b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ShuffleFrameReader.java
new file mode 100644
index 0000000..8e03eae
--- /dev/null
+++ b/hyracks-dataflow-hadoop/src/main/java/edu/uci/ics/hyracks/dataflow/hadoop/mapreduce/ShuffleFrameReader.java
@@ -0,0 +1,178 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.hadoop.mapreduce;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+
+import edu.uci.ics.hyracks.api.channels.IInputChannel;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
+import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
+import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortRunMerger;
+
+public class ShuffleFrameReader implements IFrameReader {
+ private final IHyracksTaskContext ctx;
+ private final NonDeterministicChannelReader channelReader;
+ private final HadoopHelper helper;
+ private final RecordDescriptor recordDescriptor;
+ private List<RunFileWriter> runFileWriters;
+ private RunFileReader reader;
+
+ public ShuffleFrameReader(IHyracksTaskContext ctx, NonDeterministicChannelReader channelReader,
+ MarshalledWritable<Configuration> mConfig) throws HyracksDataException {
+ this.ctx = ctx;
+ this.channelReader = channelReader;
+ helper = new HadoopHelper(mConfig);
+ this.recordDescriptor = helper.getMapOutputRecordDescriptor();
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ channelReader.open();
+ int nSenders = channelReader.getSenderPartitionCount();
+ runFileWriters = new ArrayList<RunFileWriter>();
+ RunInfo[] infos = new RunInfo[nSenders];
+ FrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recordDescriptor);
+ IInputChannel[] channels = channelReader.getChannels();
+ while (true) {
+ int entry = channelReader.findNextSender();
+ if (entry < 0) {
+ break;
+ }
+ RunInfo info = infos[entry];
+ IInputChannel channel = channels[entry];
+ ByteBuffer netBuffer = channel.getNextBuffer();
+ accessor.reset(netBuffer);
+ int nTuples = accessor.getTupleCount();
+ for (int i = 0; i < nTuples; ++i) {
+ int tBlockId = IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
+ FrameUtils.getAbsoluteFieldStartOffset(accessor, i, HadoopHelper.BLOCKID_FIELD_INDEX));
+ if (info == null) {
+ info = new RunInfo();
+ info.reset(tBlockId);
+ infos[entry] = info;
+ } else if (info.blockId != tBlockId) {
+ info.close();
+ info.reset(tBlockId);
+ }
+ info.write(accessor, i);
+ }
+ channel.recycleBuffer(netBuffer);
+ }
+ for (int i = 0; i < infos.length; ++i) {
+ RunInfo info = infos[i];
+ if (info != null) {
+ info.close();
+ }
+ }
+ infos = null;
+
+ FileReference outFile = ctx.createManagedWorkspaceFile(ShuffleFrameReader.class.getName() + ".run");
+ int framesLimit = helper.getSortFrameLimit(ctx);
+ IBinaryComparatorFactory[] comparatorFactories = helper.getSortComparatorFactories();
+ IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+ List<IFrameReader> runs = new LinkedList<IFrameReader>();
+ for (RunFileWriter rfw : runFileWriters) {
+ runs.add(rfw.createReader());
+ }
+ RunFileWriter rfw = new RunFileWriter(outFile, ctx.getIOManager());
+ ExternalSortRunMerger merger = new ExternalSortRunMerger(ctx, null, runs, new int[] { 0 }, comparators,
+ recordDescriptor, framesLimit, rfw);
+ merger.process();
+
+ reader = rfw.createReader();
+ reader.open();
+ }
+
+ @Override
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ return reader.nextFrame(buffer);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ reader.close();
+ }
+
+ private class RunInfo {
+ private final ByteBuffer buffer;
+ private final FrameTupleAppender fta;
+
+ private FileReference file;
+ private RunFileWriter rfw;
+ private int blockId;
+
+ public RunInfo() {
+ buffer = ctx.allocateFrame();
+ fta = new FrameTupleAppender(ctx.getFrameSize());
+ }
+
+ public void reset(int blockId) throws HyracksDataException {
+ this.blockId = blockId;
+ fta.reset(buffer, true);
+ try {
+ file = ctx.createManagedWorkspaceFile(ShuffleFrameReader.class.getName() + ".run");
+ rfw = new RunFileWriter(file, ctx.getIOManager());
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ public void write(FrameTupleAccessor accessor, int tIdx) throws HyracksDataException {
+ if (!fta.append(accessor, tIdx)) {
+ flush();
+ if (!fta.append(accessor, tIdx)) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+
+ public void close() throws HyracksDataException {
+ flush();
+ rfw.close();
+ runFileWriters.add(rfw);
+ }
+
+ private void flush() throws HyracksDataException {
+ if (fta.getTupleCount() <= 0) {
+ return;
+ }
+ buffer.limit(buffer.capacity());
+ buffer.position(0);
+ rfw.nextFrame(buffer);
+ fta.reset(buffer, true);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/.classpath b/hyracks-dataflow-std/.classpath
deleted file mode 100644
index 31cf404..0000000
--- a/hyracks-dataflow-std/.classpath
+++ /dev/null
@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-dataflow-std/.project b/hyracks-dataflow-std/.project
deleted file mode 100644
index 1101a65..0000000
--- a/hyracks-dataflow-std/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-dataflow-std</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-dataflow-std/.settings/org.maven.ide.eclipse.prefs b/hyracks-dataflow-std/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index e96df89..0000000
--- a/hyracks-dataflow-std/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Jul 29 14:32:56 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-dataflow-std/pom.xml b/hyracks-dataflow-std/pom.xml
index 79ce38c..4e9b42b 100644
--- a/hyracks-dataflow-std/pom.xml
+++ b/hyracks-dataflow-std/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,14 +27,14 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/AvgAggregatorDescriptorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/AvgAggregatorDescriptorFactory.java
deleted file mode 100644
index 45d49e2..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/AvgAggregatorDescriptorFactory.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-
-public class AvgAggregatorDescriptorFactory implements IAggregatorDescriptorFactory {
- private static final long serialVersionUID = 1L;
- private final int avgField;
- private int outField = -1;
-
- public AvgAggregatorDescriptorFactory(int avgField) {
- this.avgField = avgField;
- }
-
- public AvgAggregatorDescriptorFactory(int avgField, int outField) {
- this.avgField = avgField;
- this.outField = outField;
- }
-
- @Override
- public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
- RecordDescriptor outRecordDescriptor, int[] keyFields) throws HyracksDataException {
-
- if (this.outField < 0)
- this.outField = keyFields.length;
-
- return new IAggregatorDescriptor() {
-
- @Override
- public void reset() {
-
- }
-
- @Override
- public void outputResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, outField);
- int sum = IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
- int count = IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart + 4);
-
- try {
- tupleBuilder.getDataOutput().writeInt(sum / count);
- tupleBuilder.addFieldEndOffset();
- } catch (IOException e) {
- throw new HyracksDataException();
- }
- }
-
- @Override
- public void outputPartialResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, outField);
- int sum = IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
- int count = IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart + 4);
- try {
- tupleBuilder.getDataOutput().writeInt(sum);
- tupleBuilder.getDataOutput().writeInt(count);
- tupleBuilder.addFieldEndOffset();
- } catch (IOException e) {
- throw new HyracksDataException();
- }
- }
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- // Init aggregation value
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, avgField);
- int sum = IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
- int count = 1;
-
- try {
- tupleBuilder.getDataOutput().writeInt(sum);
- tupleBuilder.getDataOutput().writeInt(count);
- tupleBuilder.addFieldEndOffset();
- } catch (IOException e) {
- throw new HyracksDataException();
- }
- }
-
- @Override
- public void close() {
-
- }
-
- @Override
- public int aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset, int length)
- throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, outField);
- int sum1 = IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
- int count1 = 1;
-
- int sum2 = IntegerSerializerDeserializer.getInt(data, offset);
- int count2 = IntegerSerializerDeserializer.getInt(data, offset + 4);
-
- ByteBuffer buf = ByteBuffer.wrap(data, offset, 8);
- buf.putInt(sum1 + sum2);
- buf.putInt(count1 + count2);
-
- return 8;
- }
- };
- }
-
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/ConcatAggregatorDescriptorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/ConcatAggregatorDescriptorFactory.java
deleted file mode 100644
index 1cc3340..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/ConcatAggregatorDescriptorFactory.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-
-public class ConcatAggregatorDescriptorFactory implements IAggregatorDescriptorFactory {
- private static final long serialVersionUID = 1L;
- private static final int INIT_ACCUMULATORS_SIZE = 8;
- private final int concatField;
- private int outField = -1;
-
- /**
- * Initialize the aggregator, with the field to be concatenated.
- *
- * @param concatField
- */
- public ConcatAggregatorDescriptorFactory(int concatField) {
- this.concatField = concatField;
- }
-
- /**
- * Initialize the aggregator, with the field index to be concatenated, and
- * also the field where the aggregation result will be outputted.
- *
- * @param concatField
- * @param outField
- */
- public ConcatAggregatorDescriptorFactory(int concatField, int outField) {
- this.concatField = concatField;
- this.outField = outField;
- }
-
- /**
- * Create a concatenation aggregator. A byte buffer will be allocated inside of the
- * aggregator to contain the partial aggregation results. A reference will be written
- * onto the output frame for indexing the aggregation result from the buffer.
- */
- @Override
- public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
- RecordDescriptor outRecordDescriptor, final int[] keyFields) throws HyracksDataException {
-
- if (this.outField < 0)
- this.outField = keyFields.length;
-
- return new IAggregatorDescriptor() {
-
- byte[][] buf = new byte[INIT_ACCUMULATORS_SIZE][];
-
- int currentAggregatorIndex = -1;
- int aggregatorCount = 0;
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- // Initialize the aggregation value
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, concatField);
- int fieldLength = accessor.getFieldLength(tIndex, concatField);
- int appendOffset = tupleOffset + accessor.getFieldSlotsLength() + fieldStart;
- // Get the initial value
- currentAggregatorIndex++;
- if (currentAggregatorIndex >= buf.length) {
- byte[][] newBuf = new byte[buf.length * 2][];
- for (int i = 0; i < buf.length; i++) {
- newBuf[i] = buf[i];
- }
- this.buf = newBuf;
- }
- buf[currentAggregatorIndex] = new byte[fieldLength];
- System.arraycopy(accessor.getBuffer().array(), appendOffset, buf[currentAggregatorIndex], 0,
- fieldLength);
- // Update the aggregator index
- aggregatorCount++;
-
- try {
- tupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, currentAggregatorIndex);
- } catch (IOException e) {
- throw new HyracksDataException();
- }
- }
-
- @Override
- public void reset() {
- currentAggregatorIndex = -1;
- aggregatorCount = 0;
- }
-
- @Override
- public void close() {
- currentAggregatorIndex = -1;
- aggregatorCount = 0;
- for (int i = 0; i < buf.length; i++) {
- buf[i] = null;
- }
- }
-
- @Override
- public int aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset, int length)
- throws HyracksDataException {
- int refIndex = IntegerSerializerDeserializer.getInt(data, offset);
- // FIXME Should be done in binary way
- StringBuilder sbder = new StringBuilder();
- sbder.append(UTF8StringSerializerDeserializer.INSTANCE.deserialize(new DataInputStream(
- new ByteArrayInputStream(buf[refIndex]))));
- // Get the new data
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, concatField);
- int fieldLength = accessor.getFieldLength(tIndex, concatField);
- sbder.append(UTF8StringSerializerDeserializer.INSTANCE.deserialize(new DataInputStream(
- new ByteArrayInputStream(accessor.getBuffer().array(), tupleOffset
- + accessor.getFieldSlotsLength() + fieldStart, fieldLength))));
-
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(sbder.toString(), new DataOutputStream(baos));
- buf[refIndex] = baos.toByteArray();
- return 4;
- }
-
- @Override
- public void outputResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, outField);
- int refIndex = IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(), tupleOffset
- + accessor.getFieldSlotsLength() + fieldStart);
-
- try {
- if (refIndex >= 0)
- tupleBuilder.getDataOutput().write(buf[refIndex]);
- else {
- int fieldLength = accessor.getFieldLength(tIndex, outField);
- tupleBuilder.getDataOutput().write(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart + 4, fieldLength - 4);
- }
- tupleBuilder.addFieldEndOffset();
- } catch (IOException e) {
- throw new HyracksDataException();
- }
- }
-
- @Override
- public void outputPartialResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldOffset = accessor.getFieldStartOffset(tIndex, outField);
- int fieldLength = accessor.getFieldLength(tIndex, outField);
- int refIndex = IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(), tupleOffset
- + accessor.getFieldSlotsLength() + fieldOffset);
-
- try {
- tupleBuilder.getDataOutput().writeInt(-1);
- if (refIndex < 0) {
- tupleBuilder.getDataOutput().write(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldOffset + 4, fieldLength - 4);
- } else {
- tupleBuilder.getDataOutput().write(buf[refIndex], 0, buf[refIndex].length);
- }
- tupleBuilder.addFieldEndOffset();
- } catch (IOException e) {
- throw new HyracksDataException();
- }
- }
- };
- }
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/CountAggregatorDescriptorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/CountAggregatorDescriptorFactory.java
deleted file mode 100644
index 74ac53a..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/CountAggregatorDescriptorFactory.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-
-public class CountAggregatorDescriptorFactory implements IAggregatorDescriptorFactory {
- private static final long serialVersionUID = 1L;
- private int outField = -1;
-
- public CountAggregatorDescriptorFactory() {
- }
-
- public CountAggregatorDescriptorFactory(int outField) {
- this.outField = outField;
- }
-
- @Override
- public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
- RecordDescriptor outRecordDescriptor, final int[] keyFields) throws HyracksDataException {
-
- if (this.outField < 0) {
- this.outField = keyFields.length;
- }
- return new IAggregatorDescriptor() {
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- tupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, 1);
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public int aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset, int length)
- throws HyracksDataException {
- ByteBuffer buf = ByteBuffer.wrap(data);
- int count = buf.getInt(offset);
- buf.putInt(offset, count + 1);
- return 4;
- }
-
- @Override
- public void outputPartialResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, outField);
-
- try {
- tupleBuilder.getDataOutput().write(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart, 4);
- tupleBuilder.addFieldEndOffset();
- } catch (IOException e) {
- throw new HyracksDataException("Failed to write int sum as a partial result.");
- }
- }
-
- @Override
- public void outputResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, outField);
-
- try {
- tupleBuilder.getDataOutput().write(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart, 4);
- tupleBuilder.addFieldEndOffset();
- } catch (IOException e) {
- throw new HyracksDataException("Failed to write int sum as a partial result.");
- }
- }
-
- @Override
- public void reset() {
-
- }
- };
- }
-
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/CountAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/CountAggregatorFactory.java
deleted file mode 100644
index 2ddffde..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/CountAggregatorFactory.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-
-public class CountAggregatorFactory implements IFieldValueResultingAggregatorFactory {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IFieldValueResultingAggregator createFieldValueResultingAggregator() {
- return new IFieldValueResultingAggregator() {
- private int count;
-
- @Override
- public void output(DataOutput resultAcceptor) throws HyracksDataException {
- try {
- resultAcceptor.writeInt(count);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- count = 0;
- }
-
- @Override
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- ++count;
- }
- };
- }
-
- @Override
- public ISpillableFieldValueResultingAggregator createSpillableFieldValueResultingAggregator() {
- return new ISpillableFieldValueResultingAggregator() {
- private int count;
-
- @Override
- public void output(DataOutput resultAcceptor) throws HyracksDataException {
- try {
- resultAcceptor.writeInt(count);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public void initFromPartial(IFrameTupleAccessor accessor, int tIndex, int fIndex)
- throws HyracksDataException {
- count = IntegerSerializerDeserializer.getInt(
- accessor.getBuffer().array(),
- accessor.getTupleStartOffset(tIndex) + accessor.getFieldCount() * 4
- + accessor.getFieldStartOffset(tIndex, fIndex));
-
- }
-
- @Override
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- count++;
- }
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- count = 0;
- }
-
- @Override
- public void accumulatePartialResult(IFrameTupleAccessor accessor, int tIndex, int fIndex)
- throws HyracksDataException {
- count += IntegerSerializerDeserializer.getInt(
- accessor.getBuffer().array(),
- accessor.getTupleStartOffset(tIndex) + accessor.getFieldCount() * 4
- + accessor.getFieldStartOffset(tIndex, fIndex));
- }
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/FloatSumAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/FloatSumAggregatorFactory.java
deleted file mode 100644
index 8fd1f13..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/FloatSumAggregatorFactory.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
-
-/**
- * SUM aggregator on float type data.
- *
- * @author jarodwen
- */
-public class FloatSumAggregatorFactory implements IFieldValueResultingAggregatorFactory {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
- private int sumField;
-
- public FloatSumAggregatorFactory(int field) {
- this.sumField = field;
- }
-
- /*
- * (non-Javadoc)
- *
- * @see edu.uci.ics.hyracks.dataflow.std.aggregators.
- * IFieldValueResultingAggregatorFactory
- * #createFieldValueResultingAggregator()
- */
- @Override
- public IFieldValueResultingAggregator createFieldValueResultingAggregator() {
- return new IFieldValueResultingAggregator() {
-
- private float sum;
-
- @Override
- public void output(DataOutput resultAcceptor) throws HyracksDataException {
- try {
- resultAcceptor.writeFloat(sum);
- } catch (IOException ex) {
- throw new HyracksDataException(ex);
- }
- }
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- sum = 0;
- }
-
- @Override
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldCount = accessor.getFieldCount();
- int fieldStart = accessor.getFieldStartOffset(tIndex, sumField);
- sum += FloatSerializerDeserializer.getFloat(accessor.getBuffer().array(), tupleOffset + 2 * fieldCount
- + fieldStart);
- }
- };
- }
-
- /*
- * (non-Javadoc)
- *
- * @see edu.uci.ics.hyracks.dataflow.std.aggregators.
- * IFieldValueResultingAggregatorFactory
- * #createSpillableFieldValueResultingAggregator()
- */
- @Override
- public ISpillableFieldValueResultingAggregator createSpillableFieldValueResultingAggregator() {
- return new ISpillableFieldValueResultingAggregator() {
-
- private float sum;
-
- @Override
- public void output(DataOutput resultAcceptor) throws HyracksDataException {
- try {
- resultAcceptor.writeFloat(sum);
- } catch (IOException ex) {
- throw new HyracksDataException(ex);
- }
- }
-
- @Override
- public void initFromPartial(IFrameTupleAccessor accessor, int tIndex, int fIndex)
- throws HyracksDataException {
- sum = FloatSerializerDeserializer.getFloat(
- accessor.getBuffer().array(),
- accessor.getTupleStartOffset(tIndex) + accessor.getFieldCount() * 4
- + accessor.getFieldStartOffset(tIndex, fIndex));
- }
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- sum = 0;
- }
-
- @Override
- public void accumulatePartialResult(IFrameTupleAccessor accessor, int tIndex, int fIndex)
- throws HyracksDataException {
- sum += FloatSerializerDeserializer.getFloat(
- accessor.getBuffer().array(),
- accessor.getTupleStartOffset(tIndex) + accessor.getFieldCount() * 4
- + accessor.getFieldStartOffset(tIndex, fIndex));
- }
-
- @Override
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldCount = accessor.getFieldCount();
- int fieldStart = accessor.getFieldStartOffset(tIndex, sumField);
- sum += FloatSerializerDeserializer.getFloat(accessor.getBuffer().array(), tupleOffset + 2 * fieldCount
- + fieldStart);
- }
- };
- }
-
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IAggregatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IAggregatorDescriptor.java
deleted file mode 100644
index dfc31cd..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IAggregatorDescriptor.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-
-public interface IAggregatorDescriptor {
-
- /**
- * Initialize the aggregator with an input tuple specified by the input
- * frame and tuple index. This function will write the initialized partial
- * result into the tuple builder.
- *
- * @param accessor
- * @param tIndex
- * @param tupleBuilder
- * @throws HyracksDataException
- */
- public void init(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException;
-
- /**
- * Aggregate the input tuple with the partial result specified by the bytes.
- * The new value then is written back to the bytes field specified.
- * It is the developer's responsibility to have the new result not exceed
- * the given bytes.
- *
- * @param accessor
- * @param tIndex
- * @param data
- * @param offset
- * @param length
- * @return
- * @throws HyracksDataException
- */
- public int aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset, int length)
- throws HyracksDataException;
-
- /**
- * Output the partial aggregation result to an array tuple builder.
- * Necessary additional information for aggregation should be maintained.
- * For example, for an aggregator calculating AVG, the count and also the
- * current average should be maintained as the partial results.
- *
- * @param accessor
- * @param tIndex
- * @param tupleBuilder
- * @throws HyracksDataException
- */
- public void outputPartialResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException;
-
- /**
- * Output the final aggregation result to an array tuple builder.
- *
- * @param accessor
- * @param tIndex
- * @param tupleBuilder
- * @return
- * @throws HyracksDataException
- */
- public void outputResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException;
-
- /**
- * reset the internal states
- */
- public void reset();
-
- /**
- * Close the aggregator. Necessary clean-up code should be implemented here.
- */
- public void close();
-
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IAggregatorDescriptorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IAggregatorDescriptorFactory.java
deleted file mode 100644
index 7f6928b..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IAggregatorDescriptorFactory.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public interface IAggregatorDescriptorFactory extends Serializable {
-
- /**
- * Create an aggregator.
- *
- * @param ctx
- * @param inRecordDescriptor
- * @param outRecordDescriptor
- * @param keyFields
- * @return
- * @throws HyracksDataException
- */
- IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
- RecordDescriptor outRecordDescriptor, int[] keyFields) throws HyracksDataException;
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IFieldValueResultingAggregator.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IFieldValueResultingAggregator.java
deleted file mode 100644
index 783467b..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IFieldValueResultingAggregator.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.DataOutput;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public interface IFieldValueResultingAggregator {
- /**
- * Called once per aggregator before calling accumulate for the first time.
- *
- * @param accessor
- * - Accessor to the data tuple.
- * @param tIndex
- * - Index of the tuple in the accessor.
- * @throws HyracksDataException
- */
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException;
-
- /**
- * Called once per tuple that belongs to this group.
- *
- * @param accessor
- * - Accessor to data tuple.
- * @param tIndex
- * - Index of tuple in the accessor.
- * @throws HyracksDataException
- */
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException;
-
- /**
- * Called finally to emit output.
- *
- * @param resultAcceptor
- * - Interface to write the result to.
- * @throws HyracksDataException
- */
- public void output(DataOutput resultAcceptor) throws HyracksDataException;
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IFieldValueResultingAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IFieldValueResultingAggregatorFactory.java
deleted file mode 100644
index 1b7da7f..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IFieldValueResultingAggregatorFactory.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.Serializable;
-
-public interface IFieldValueResultingAggregatorFactory extends Serializable {
- public IFieldValueResultingAggregator createFieldValueResultingAggregator();
-
- public ISpillableFieldValueResultingAggregator createSpillableFieldValueResultingAggregator();
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/ISpillableFieldValueResultingAggregator.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/ISpillableFieldValueResultingAggregator.java
deleted file mode 100644
index d52f458..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/ISpillableFieldValueResultingAggregator.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-/**
- * An extended version of the {@link IFieldValueResultingAggregator} supporting
- * external aggregation.
- */
-public interface ISpillableFieldValueResultingAggregator extends IFieldValueResultingAggregator {
-
- /**
- * Called once per aggregator before calling accumulate for the first time.
- *
- * @param accessor
- * - Accessor to the data tuple.
- * @param tIndex
- * - Index of the tuple in the accessor.
- * @throws HyracksDataException
- */
- public void initFromPartial(IFrameTupleAccessor accessor, int tIndex, int fIndex) throws HyracksDataException;
-
- /**
- * Aggregate another partial result.
- *
- * @param accessor
- * @param tIndex
- * @param fIndex
- * @throws HyracksDataException
- */
- public void accumulatePartialResult(IFrameTupleAccessor accessor, int tIndex, int fIndex)
- throws HyracksDataException;
-
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IntSumAggregatorDescriptorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IntSumAggregatorDescriptorFactory.java
deleted file mode 100644
index 770d36b..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/IntSumAggregatorDescriptorFactory.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.IOException;
-import java.nio.ByteBuffer;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-
-public class IntSumAggregatorDescriptorFactory implements IAggregatorDescriptorFactory {
- private static final long serialVersionUID = 1L;
- private final int aggField;
- private int outField = -1;
-
- public IntSumAggregatorDescriptorFactory(int aggField) {
- this.aggField = aggField;
- }
-
- public IntSumAggregatorDescriptorFactory(int aggField, int outField) {
- this.aggField = aggField;
- this.outField = outField;
- }
-
- @Override
- public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
- RecordDescriptor outRecordDescriptor, final int[] keyFields) throws HyracksDataException {
-
- if (this.outField < 0) {
- this.outField = keyFields.length;
- }
-
- return new IAggregatorDescriptor() {
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- int sum = 0;
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
- sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
-
- tupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, sum);
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public int aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset, int length)
- throws HyracksDataException {
- int sum = 0;
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
- sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
- // Update the value of tuple 2
- ByteBuffer buf = ByteBuffer.wrap(data);
- sum += buf.getInt(offset);
- buf.putInt(offset, sum);
- return 4;
- }
-
- @Override
- public void outputPartialResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, outField);
-
- try {
- tupleBuilder.getDataOutput().write(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart, 4);
- tupleBuilder.addFieldEndOffset();
- } catch (IOException e) {
- throw new HyracksDataException("Failed to write int sum as a partial result.");
- }
- }
-
- @Override
- public void outputResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldStart = accessor.getFieldStartOffset(tIndex, outField);
-
- try {
- tupleBuilder.getDataOutput().write(accessor.getBuffer().array(),
- tupleOffset + accessor.getFieldSlotsLength() + fieldStart, 4);
- tupleBuilder.addFieldEndOffset();
- } catch (IOException e) {
- throw new HyracksDataException("Failed to write int sum as a partial result.");
- }
- }
-
- @Override
- public void reset() {
-
- }
- };
- }
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MinMaxAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MinMaxAggregatorFactory.java
deleted file mode 100644
index cc999d5..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MinMaxAggregatorFactory.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
-
-/**
- * Min/Max aggregator factory
- */
-public class MinMaxAggregatorFactory implements IFieldValueResultingAggregatorFactory {
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- /**
- * indicate the type of the value: true: max false: min
- */
- private boolean type;
-
- /**
- * The field to be aggregated.
- */
- private int field;
-
- public MinMaxAggregatorFactory(boolean type, int field) {
- this.type = type;
- this.field = field;
- }
-
- @Override
- public IFieldValueResultingAggregator createFieldValueResultingAggregator() {
- return new IFieldValueResultingAggregator() {
-
- private float minmax;
-
- @Override
- public void output(DataOutput resultAcceptor) throws HyracksDataException {
- try {
- resultAcceptor.writeFloat(minmax);
- } catch (IOException ex) {
- throw new HyracksDataException(ex);
- }
- }
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- minmax = 0;
- }
-
- @Override
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldCount = accessor.getFieldCount();
- int fieldStart = accessor.getFieldStartOffset(tIndex, field);
- float nval = FloatSerializerDeserializer.getFloat(accessor.getBuffer().array(), tupleOffset + 2
- * fieldCount + fieldStart);
- if ((type ? (nval > minmax) : (nval < minmax))) {
- minmax = nval;
- }
- }
- };
- }
-
- @Override
- public ISpillableFieldValueResultingAggregator createSpillableFieldValueResultingAggregator() {
- return new ISpillableFieldValueResultingAggregator() {
-
- private float minmax;
-
- @Override
- public void output(DataOutput resultAcceptor) throws HyracksDataException {
- try {
- resultAcceptor.writeFloat(minmax);
- } catch (IOException ex) {
- throw new HyracksDataException(ex);
- }
- }
-
- @Override
- public void initFromPartial(IFrameTupleAccessor accessor, int tIndex, int fIndex)
- throws HyracksDataException {
- minmax = FloatSerializerDeserializer.getFloat(
- accessor.getBuffer().array(),
- accessor.getTupleStartOffset(tIndex) + accessor.getFieldCount() * 4
- + accessor.getFieldStartOffset(tIndex, fIndex));
-
- }
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- minmax = 0;
- }
-
- @Override
- public void accumulatePartialResult(IFrameTupleAccessor accessor, int tIndex, int fIndex)
- throws HyracksDataException {
- minmax = FloatSerializerDeserializer.getFloat(
- accessor.getBuffer().array(),
- accessor.getTupleStartOffset(tIndex) + accessor.getFieldCount() * 4
- + accessor.getFieldStartOffset(tIndex, fIndex));
-
- }
-
- @Override
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldCount = accessor.getFieldCount();
- int fieldStart = accessor.getFieldStartOffset(tIndex, field);
- float nval = FloatSerializerDeserializer.getFloat(accessor.getBuffer().array(), tupleOffset + 2
- * fieldCount + fieldStart);
- if ((type ? (nval > minmax) : (nval < minmax))) {
- minmax = nval;
- }
- }
- };
- }
-
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorDescriptorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorDescriptorFactory.java
deleted file mode 100644
index 053b9e2..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorDescriptorFactory.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-
-public class MultiAggregatorDescriptorFactory implements IAggregatorDescriptorFactory {
- private static final long serialVersionUID = 1L;
- private final IAggregatorDescriptorFactory[] aggregatorFactories;
-
- public MultiAggregatorDescriptorFactory(IAggregatorDescriptorFactory[] aggregatorFactories) {
- this.aggregatorFactories = aggregatorFactories;
- }
-
- @Override
- public IAggregatorDescriptor createAggregator(final IHyracksTaskContext ctx,
- final RecordDescriptor inRecordDescriptor, final RecordDescriptor outRecordDescriptor, final int[] keyFields)
- throws HyracksDataException {
-
- final IAggregatorDescriptor[] aggregators = new IAggregatorDescriptor[this.aggregatorFactories.length];
- for (int i = 0; i < aggregators.length; i++) {
- aggregators[i] = aggregatorFactories[i].createAggregator(ctx, inRecordDescriptor, outRecordDescriptor,
- keyFields);
- }
-
- return new IAggregatorDescriptor() {
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- for (int i = 0; i < aggregators.length; i++) {
- aggregators[i].init(accessor, tIndex, tupleBuilder);
- }
- }
-
- @Override
- public int aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset, int length)
- throws HyracksDataException {
- int adjust = 0;
- for (int i = 0; i < aggregators.length; i++) {
- adjust += aggregators[i].aggregate(accessor, tIndex, data, offset + adjust, length - adjust);
- }
- return adjust;
- }
-
- @Override
- public void outputPartialResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- for (int i = 0; i < aggregators.length; i++) {
- aggregators[i].outputPartialResult(accessor, tIndex, tupleBuilder);
- }
- }
-
- @Override
- public void outputResult(IFrameTupleAccessor accessor, int tIndex, ArrayTupleBuilder tupleBuilder)
- throws HyracksDataException {
- for (int i = 0; i < aggregators.length; i++) {
- aggregators[i].outputResult(accessor, tIndex, tupleBuilder);
- }
- }
-
- @Override
- public void close() {
- for (int i = 0; i < aggregators.length; i++) {
- aggregators[i].close();
- }
- }
-
- @Override
- public void reset() {
- for (int i = 0; i < aggregators.length; i++) {
- aggregators[i].reset();
- }
- }
-
- };
- }
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorFactory.java
deleted file mode 100644
index 8237110..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/MultiAggregatorFactory.java
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.DataOutput;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.std.group.IAccumulatingAggregator;
-import edu.uci.ics.hyracks.dataflow.std.group.IAccumulatingAggregatorFactory;
-
-public class MultiAggregatorFactory implements IAccumulatingAggregatorFactory {
- private static final long serialVersionUID = 1L;
-
- private IFieldValueResultingAggregatorFactory[] aFactories;
-
- public MultiAggregatorFactory(IFieldValueResultingAggregatorFactory[] aFactories) {
- this.aFactories = aFactories;
- }
-
- @Override
- public IAccumulatingAggregator createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDesc,
- RecordDescriptor outRecordDescriptor) throws HyracksDataException {
- final IFieldValueResultingAggregator aggregators[] = new IFieldValueResultingAggregator[aFactories.length];
- for (int i = 0; i < aFactories.length; ++i) {
- aggregators[i] = aFactories[i].createFieldValueResultingAggregator();
- }
- final ArrayTupleBuilder tb = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
- return new IAccumulatingAggregator() {
-
- private boolean pending;
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- tb.reset();
- for (int i = 0; i < aggregators.length; ++i) {
- aggregators[i].init(accessor, tIndex);
- }
- pending = false;
- }
-
- @Override
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- for (int i = 0; i < aggregators.length; ++i) {
- aggregators[i].accumulate(accessor, tIndex);
- }
- }
-
- @Override
- public boolean output(FrameTupleAppender appender, IFrameTupleAccessor accessor, int tIndex,
- int[] keyFieldIndexes) throws HyracksDataException {
- if (!pending) {
- tb.reset();
- for (int i = 0; i < keyFieldIndexes.length; ++i) {
- tb.addField(accessor, tIndex, keyFieldIndexes[i]);
- }
- DataOutput dos = tb.getDataOutput();
- for (int i = 0; i < aggregators.length; ++i) {
- aggregators[i].output(dos);
- tb.addFieldEndOffset();
- }
- }
- if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
- pending = true;
- return false;
- }
- return true;
- }
-
- };
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/SumAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/SumAggregatorFactory.java
deleted file mode 100644
index c6bcc81..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/aggregators/SumAggregatorFactory.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.aggregators;
-
-import java.io.DataOutput;
-import java.io.IOException;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-
-/**
- * SUM aggregator factory (for integer only; another SUM aggregator for floats
- * is available at {@link FloatSumAggregatorFactory})
- */
-public class SumAggregatorFactory implements IFieldValueResultingAggregatorFactory {
-
- private int sumField;
-
- public SumAggregatorFactory(int field) {
- sumField = field;
- }
-
- /**
- *
- */
- private static final long serialVersionUID = 1L;
-
- /*
- * (non-Javadoc)
- *
- * @see edu.uci.ics.hyracks.dataflow.std.aggregators.
- * IFieldValueResultingAggregatorFactory
- * #createFieldValueResultingAggregator()
- */
- @Override
- public IFieldValueResultingAggregator createFieldValueResultingAggregator() {
- return new IFieldValueResultingAggregator() {
- private int sum;
-
- @Override
- public void output(DataOutput resultAcceptor) throws HyracksDataException {
- try {
- resultAcceptor.writeInt(sum);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- sum++;
- }
-
- @Override
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldCount = accessor.getFieldCount();
- int fieldStart = accessor.getFieldStartOffset(tIndex, sumField);
- sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(), tupleOffset + 2 * fieldCount
- + fieldStart);
- }
- };
- }
-
- /*
- * (non-Javadoc)
- *
- * @see edu.uci.ics.hyracks.dataflow.std.aggregators.spillable.
- * ISpillableFieldValueResultingAggregatorFactory
- * #createFieldValueResultingAggregator()
- */
- @Override
- public ISpillableFieldValueResultingAggregator createSpillableFieldValueResultingAggregator() {
- return new ISpillableFieldValueResultingAggregator() {
-
- private int sum;
-
- @Override
- public void output(DataOutput resultAcceptor) throws HyracksDataException {
- try {
- resultAcceptor.writeInt(sum);
- } catch (IOException ex) {
- throw new HyracksDataException(ex);
- }
- }
-
- @Override
- public void initFromPartial(IFrameTupleAccessor accessor, int tIndex, int fIndex)
- throws HyracksDataException {
- sum = IntegerSerializerDeserializer.getInt(
- accessor.getBuffer().array(),
- accessor.getTupleStartOffset(tIndex) + accessor.getFieldCount() * 4
- + accessor.getFieldStartOffset(tIndex, fIndex));
- }
-
- @Override
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- sum = 0;
- }
-
- @Override
- public void accumulatePartialResult(IFrameTupleAccessor accessor, int tIndex, int fIndex)
- throws HyracksDataException {
- sum += IntegerSerializerDeserializer.getInt(
- accessor.getBuffer().array(),
- accessor.getTupleStartOffset(tIndex) + accessor.getFieldCount() * 4
- + accessor.getFieldStartOffset(tIndex, fIndex));
- }
-
- @Override
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- int tupleOffset = accessor.getTupleStartOffset(tIndex);
- int fieldCount = accessor.getFieldCount();
- int fieldStart = accessor.getFieldStartOffset(tIndex, sumField);
- sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(), tupleOffset + 2 * fieldCount
- + fieldStart);
- }
- };
- }
-
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
index f3535f8..7c50da9 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractConnectorDescriptor.java
@@ -28,22 +28,33 @@
private static final long serialVersionUID = 1L;
protected final ConnectorDescriptorId id;
+ protected String displayName;
+
public AbstractConnectorDescriptor(JobSpecification spec) {
this.id = spec.createConnectorDescriptor();
spec.getConnectorMap().put(id, this);
+ displayName = getClass().getName() + "[" + id + "]";
}
public ConnectorDescriptorId getConnectorId() {
return id;
}
+ public String getDisplayName() {
+ return displayName;
+ }
+
+ public void setDisplayName(String displayName) {
+ this.displayName = displayName;
+ }
+
@Override
public JSONObject toJSON() throws JSONException {
JSONObject jconn = new JSONObject();
- jconn.put("type", "connector");
- jconn.put("id", getConnectorId().getId());
+ jconn.put("id", String.valueOf(getConnectorId()));
jconn.put("java-class", getClass().getName());
+ jconn.put("display-name", displayName);
return jconn;
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractMToNConnectorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractMToNConnectorDescriptor.java
index aa550ae..ae1ead5 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractMToNConnectorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractMToNConnectorDescriptor.java
@@ -31,4 +31,11 @@
targetBitmap.clear();
targetBitmap.set(0, nConsumerPartitions);
}
+
+ @Override
+ public void indicateSourcePartitions(int nProducerPartitions, int nConsumerPartitions, int consumerIndex,
+ BitSet sourceBitmap) {
+ sourceBitmap.clear();
+ sourceBitmap.set(0, nProducerPartitions);
+ }
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
index 15fc75d..983851e 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/base/AbstractOperatorDescriptor.java
@@ -38,12 +38,15 @@
protected final int outputArity;
+ protected String displayName;
+
public AbstractOperatorDescriptor(JobSpecification spec, int inputArity, int outputArity) {
odId = spec.createOperatorDescriptorId();
this.inputArity = inputArity;
this.outputArity = outputArity;
recordDescriptors = new RecordDescriptor[outputArity];
spec.getOperatorMap().put(getOperatorId(), this);
+ displayName = getClass().getName() + "[" + odId + "]";
}
@Override
@@ -66,6 +69,14 @@
return recordDescriptors;
}
+ public String getDisplayName() {
+ return displayName;
+ }
+
+ public void setDisplayName(String displayName) {
+ this.displayName = displayName;
+ }
+
@Override
public void contributeSchedulingConstraints(IConstraintAcceptor constraintAcceptor, JobActivityGraph plan,
ICCApplicationContext appCtx) {
@@ -75,11 +86,11 @@
@Override
public JSONObject toJSON() throws JSONException {
JSONObject jop = new JSONObject();
- jop.put("type", "operator");
- jop.put("id", getOperatorId().getId());
+ jop.put("id", String.valueOf(getOperatorId()));
jop.put("java-class", getClass().getName());
jop.put("in-arity", getInputArity());
jop.put("out-arity", getOutputArity());
+ jop.put("display-name", displayName);
return jop;
}
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/IPartitionAcceptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/IPartitionAcceptor.java
new file mode 100644
index 0000000..8942ea9
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/IPartitionAcceptor.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.collectors;
+
+import edu.uci.ics.hyracks.api.channels.IInputChannel;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+
+public interface IPartitionAcceptor {
+ public void addPartition(PartitionId pid, IInputChannel channel);
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/IPartitionBatchManager.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/IPartitionBatchManager.java
new file mode 100644
index 0000000..4962d64
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/IPartitionBatchManager.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.collectors;
+
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+public interface IPartitionBatchManager extends IPartitionAcceptor {
+ public void getNextBatch(List<IFrameReader> batch, int size) throws HyracksDataException;
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/InputChannelFrameReader.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/InputChannelFrameReader.java
new file mode 100644
index 0000000..713fb0e
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/InputChannelFrameReader.java
@@ -0,0 +1,77 @@
+package edu.uci.ics.hyracks.dataflow.std.collectors;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.channels.IInputChannel;
+import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+
+public class InputChannelFrameReader implements IFrameReader, IInputChannelMonitor {
+ private final IInputChannel channel;
+
+ private int availableFrames;
+
+ private boolean eos;
+
+ private boolean failed;
+
+ public InputChannelFrameReader(IInputChannel channel) {
+ this.channel = channel;
+ availableFrames = 0;
+ eos = false;
+ failed = false;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ }
+
+ @Override
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ synchronized (this) {
+ while (!failed && !eos && availableFrames <= 0) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ if (failed) {
+ throw new HyracksDataException("Failure occurred on input");
+ }
+ if (availableFrames <= 0 && eos) {
+ return false;
+ }
+ --availableFrames;
+ }
+ ByteBuffer srcBuffer = channel.getNextBuffer();
+ FrameUtils.copy(srcBuffer, buffer);
+ channel.recycleBuffer(srcBuffer);
+ return true;
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+
+ }
+
+ @Override
+ public synchronized void notifyFailure(IInputChannel channel) {
+ failed = true;
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void notifyDataAvailability(IInputChannel channel, int nFrames) {
+ availableFrames += nFrames;
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void notifyEndOfStream(IInputChannel channel) {
+ eos = true;
+ notifyAll();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicChannelReader.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicChannelReader.java
new file mode 100644
index 0000000..578bfec
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicChannelReader.java
@@ -0,0 +1,163 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.collectors;
+
+import java.util.BitSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.channels.IInputChannel;
+import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+
+public class NonDeterministicChannelReader implements IInputChannelMonitor, IPartitionAcceptor {
+ private static final Logger LOGGER = Logger.getLogger(NonDeterministicChannelReader.class.getName());
+
+ private final int nSenderPartitions;
+
+ private final IInputChannel[] channels;
+
+ private final BitSet frameAvailability;
+
+ private final int[] availableFrameCounts;
+
+ private final BitSet eosSenders;
+
+ private final BitSet failSenders;
+
+ private final BitSet closedSenders;
+
+ private int lastReadSender;
+
+ public NonDeterministicChannelReader(int nSenderPartitions, BitSet expectedPartitions) {
+ this.nSenderPartitions = nSenderPartitions;
+ channels = new IInputChannel[nSenderPartitions];
+ eosSenders = new BitSet(nSenderPartitions);
+ failSenders = new BitSet(nSenderPartitions);
+ closedSenders = new BitSet(nSenderPartitions);
+ closedSenders.or(expectedPartitions);
+ closedSenders.flip(0, nSenderPartitions);
+ frameAvailability = new BitSet(nSenderPartitions);
+ availableFrameCounts = new int[nSenderPartitions];
+ }
+
+ @Override
+ public void addPartition(PartitionId pid, IInputChannel channel) {
+ channel.registerMonitor(this);
+ channel.setAttachment(pid);
+ synchronized (this) {
+ channels[pid.getSenderIndex()] = channel;
+ }
+ }
+
+ public int getSenderPartitionCount() {
+ return nSenderPartitions;
+ }
+
+ public void open() throws HyracksDataException {
+ lastReadSender = 0;
+ }
+
+ public IInputChannel[] getChannels() {
+ return channels;
+ }
+
+ public synchronized int findNextSender() throws HyracksDataException {
+ while (true) {
+ switch (lastReadSender) {
+ default:
+ lastReadSender = frameAvailability.nextSetBit(lastReadSender + 1);
+ if (lastReadSender >= 0) {
+ break;
+ }
+ case 0:
+ lastReadSender = frameAvailability.nextSetBit(0);
+ }
+ if (lastReadSender >= 0) {
+ assert availableFrameCounts[lastReadSender] > 0;
+ if (--availableFrameCounts[lastReadSender] == 0) {
+ frameAvailability.clear(lastReadSender);
+ }
+ return lastReadSender;
+ }
+ if (!failSenders.isEmpty()) {
+ throw new HyracksDataException("Failure occurred on input");
+ }
+ for (int i = eosSenders.nextSetBit(0); i >= 0; i = eosSenders.nextSetBit(i)) {
+ channels[i].close();
+ eosSenders.clear(i);
+ closedSenders.set(i);
+ }
+ int nextClosedBitIndex = closedSenders.nextClearBit(0);
+ if (nextClosedBitIndex < 0 || nextClosedBitIndex >= nSenderPartitions) {
+ lastReadSender = -1;
+ return lastReadSender;
+ }
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
+
+ public synchronized void close() throws HyracksDataException {
+ for (int i = closedSenders.nextClearBit(0); i >= 0 && i < nSenderPartitions; i = closedSenders
+ .nextClearBit(i + 1)) {
+ if (channels[i] != null) {
+ channels[i].close();
+ channels[i] = null;
+ }
+ }
+ }
+
+ @Override
+ public synchronized void notifyFailure(IInputChannel channel) {
+ PartitionId pid = (PartitionId) channel.getAttachment();
+ int senderIndex = pid.getSenderIndex();
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Failure: " + pid.getConnectorDescriptorId() + " sender: " + senderIndex + " receiver: "
+ + pid.getReceiverIndex());
+ }
+ failSenders.set(senderIndex);
+ eosSenders.set(senderIndex);
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void notifyDataAvailability(IInputChannel channel, int nFrames) {
+ PartitionId pid = (PartitionId) channel.getAttachment();
+ int senderIndex = pid.getSenderIndex();
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Data available: " + pid.getConnectorDescriptorId() + " sender: " + senderIndex + " receiver: "
+ + pid.getReceiverIndex());
+ }
+ availableFrameCounts[senderIndex] += nFrames;
+ frameAvailability.set(senderIndex);
+ notifyAll();
+ }
+
+ @Override
+ public synchronized void notifyEndOfStream(IInputChannel channel) {
+ PartitionId pid = (PartitionId) channel.getAttachment();
+ int senderIndex = pid.getSenderIndex();
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("EOS: " + pid);
+ }
+ eosSenders.set(senderIndex);
+ notifyAll();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicFrameReader.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicFrameReader.java
new file mode 100644
index 0000000..657165c
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicFrameReader.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.collectors;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.channels.IInputChannel;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+
+public class NonDeterministicFrameReader implements IFrameReader {
+ private final NonDeterministicChannelReader channelReader;
+
+ public NonDeterministicFrameReader(NonDeterministicChannelReader channelReader) {
+ this.channelReader = channelReader;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ channelReader.open();
+ }
+
+ @Override
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ int index = channelReader.findNextSender();
+ if (index >= 0) {
+ IInputChannel[] channels = channelReader.getChannels();
+ ByteBuffer srcFrame = channels[index].getNextBuffer();
+ FrameUtils.copy(srcFrame, buffer);
+ channels[index].recycleBuffer(srcFrame);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public synchronized void close() throws HyracksDataException {
+ channelReader.close();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionBatchManager.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionBatchManager.java
new file mode 100644
index 0000000..ba25850
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionBatchManager.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.collectors;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.channels.IInputChannel;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+
+public class NonDeterministicPartitionBatchManager implements IPartitionBatchManager {
+ private final IInputChannel[] channels;
+
+ private List<IFrameReader> partitions;
+
+ private List<IFrameReader> batch;
+
+ private int requiredSize;
+
+ public NonDeterministicPartitionBatchManager(int nSenders) {
+ channels = new IInputChannel[nSenders];
+ partitions = new ArrayList<IFrameReader>();
+ }
+
+ @Override
+ public synchronized void addPartition(PartitionId pid, IInputChannel channel) {
+ channels[pid.getSenderIndex()] = channel;
+ InputChannelFrameReader channelReader = new InputChannelFrameReader(channel);
+ channel.registerMonitor(channelReader);
+ if (batch != null && batch.size() < requiredSize) {
+ batch.add(channelReader);
+ if (batch.size() == requiredSize) {
+ notifyAll();
+ }
+ } else {
+ partitions.add(channelReader);
+ }
+ }
+
+ @Override
+ public synchronized void getNextBatch(List<IFrameReader> batch, int size) throws HyracksDataException {
+ if (partitions.size() <= size) {
+ batch.addAll(partitions);
+ partitions.clear();
+ } else if (partitions.size() > size) {
+ List<IFrameReader> sublist = partitions.subList(0, size);
+ batch.addAll(sublist);
+ sublist.clear();
+ }
+ if (batch.size() == size) {
+ return;
+ }
+ this.batch = batch;
+ this.requiredSize = size;
+ while (batch.size() < size) {
+ try {
+ wait();
+ } catch (InterruptedException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ this.batch = null;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionCollector.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionCollector.java
deleted file mode 100644
index 33819a8..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/NonDeterministicPartitionCollector.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.collectors;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.BitSet;
-import java.util.Collection;
-
-import edu.uci.ics.hyracks.api.channels.IInputChannel;
-import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
-import edu.uci.ics.hyracks.api.comm.IFrameReader;
-import edu.uci.ics.hyracks.api.comm.PartitionChannel;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-
-public class NonDeterministicPartitionCollector extends AbstractPartitionCollector {
- private final FrameReader reader;
-
- private final BitSet expectedPartitions;
-
- private final int nSenderPartitions;
-
- private final IInputChannel[] channels;
-
- private final BitSet frameAvailability;
-
- private final int[] availableFrameCounts;
-
- private final BitSet eosSenders;
-
- private final BitSet failSenders;
-
- private BitSet closedSenders;
-
- private int lastReadSender;
-
- public NonDeterministicPartitionCollector(IHyracksTaskContext ctx, ConnectorDescriptorId connectorId,
- int receiverIndex, int nSenderPartitions, BitSet expectedPartitions) {
- super(ctx, connectorId, receiverIndex);
- this.expectedPartitions = expectedPartitions;
- this.nSenderPartitions = nSenderPartitions;
- reader = new FrameReader();
- channels = new IInputChannel[nSenderPartitions];
- eosSenders = new BitSet(nSenderPartitions);
- failSenders = new BitSet(nSenderPartitions);
- closedSenders = new BitSet(nSenderPartitions);
- closedSenders.or(expectedPartitions);
- closedSenders.flip(0, nSenderPartitions);
- frameAvailability = new BitSet(nSenderPartitions);
- availableFrameCounts = new int[nSenderPartitions];
- }
-
- @Override
- public void open() throws HyracksException {
- lastReadSender = 0;
- }
-
- @Override
- public void addPartitions(Collection<PartitionChannel> partitions) throws HyracksException {
- for (PartitionChannel pc : partitions) {
- PartitionId pid = pc.getPartitionId();
- IInputChannel channel = pc.getInputChannel();
- channel.setAttachment(pid);
- channel.registerMonitor(reader);
- synchronized (this) {
- channels[pid.getSenderIndex()] = channel;
- }
- channel.open();
- }
- }
-
- @Override
- public IFrameReader getReader() throws HyracksException {
- return reader;
- }
-
- @Override
- public void close() throws HyracksException {
- }
-
- private final class FrameReader implements IFrameReader, IInputChannelMonitor {
- @Override
- public void open() throws HyracksDataException {
- }
-
- @Override
- public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
- findNextSender();
- if (lastReadSender >= 0) {
- ByteBuffer srcFrame = channels[lastReadSender].getNextBuffer();
- FrameUtils.copy(srcFrame, buffer);
- channels[lastReadSender].recycleBuffer(srcFrame);
- return true;
- }
- return false;
- }
-
- private void findNextSender() throws HyracksDataException {
- synchronized (NonDeterministicPartitionCollector.this) {
- while (true) {
- switch (lastReadSender) {
- default:
- lastReadSender = frameAvailability.nextSetBit(lastReadSender + 1);
- if (lastReadSender >= 0) {
- break;
- }
- case 0:
- lastReadSender = frameAvailability.nextSetBit(0);
- }
- if (lastReadSender >= 0) {
- assert availableFrameCounts[lastReadSender] > 0;
- if (--availableFrameCounts[lastReadSender] == 0) {
- frameAvailability.clear(lastReadSender);
- }
- return;
- }
- if (!failSenders.isEmpty()) {
- throw new HyracksDataException("Failure occurred on input");
- }
- for (int i = eosSenders.nextSetBit(0); i >= 0; i = eosSenders.nextSetBit(i)) {
- channels[i].close();
- eosSenders.clear(i);
- closedSenders.set(i);
- }
- int nextClosedBitIndex = closedSenders.nextClearBit(0);
- if (nextClosedBitIndex < 0 || nextClosedBitIndex >= nSenderPartitions) {
- lastReadSender = -1;
- return;
- }
- try {
- NonDeterministicPartitionCollector.this.wait();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
- }
- }
- }
- }
-
- @Override
- public void close() throws HyracksDataException {
- synchronized (NonDeterministicPartitionCollector.this) {
- for (int i = closedSenders.nextClearBit(0); i >= 0 && i < nSenderPartitions; i = closedSenders
- .nextClearBit(i + 1)) {
- if (channels[i] != null) {
- channels[i].close();
- channels[i] = null;
- }
- }
- }
- }
-
- @Override
- public void notifyFailure(IInputChannel channel) {
- synchronized (NonDeterministicPartitionCollector.this) {
- PartitionId pid = (PartitionId) channel.getAttachment();
- int senderIndex = pid.getSenderIndex();
- failSenders.set(senderIndex);
- NonDeterministicPartitionCollector.this.notifyAll();
- }
- }
-
- @Override
- public void notifyDataAvailability(IInputChannel channel, int nFrames) {
- synchronized (NonDeterministicPartitionCollector.this) {
- PartitionId pid = (PartitionId) channel.getAttachment();
- int senderIndex = pid.getSenderIndex();
- availableFrameCounts[senderIndex] += nFrames;
- frameAvailability.set(senderIndex);
- NonDeterministicPartitionCollector.this.notifyAll();
- }
- }
-
- @Override
- public void notifyEndOfStream(IInputChannel channel) {
- synchronized (NonDeterministicPartitionCollector.this) {
- PartitionId pid = (PartitionId) channel.getAttachment();
- int senderIndex = pid.getSenderIndex();
- eosSenders.set(senderIndex);
- NonDeterministicPartitionCollector.this.notifyAll();
- }
- }
- }
-
- @Override
- public Collection<PartitionId> getRequiredPartitionIds() throws HyracksException {
- Collection<PartitionId> c = new ArrayList<PartitionId>(expectedPartitions.cardinality());
- for (int i = expectedPartitions.nextSetBit(0); i >= 0; i = expectedPartitions.nextSetBit(i + 1)) {
- c.add(new PartitionId(getJobId(), getConnectorId(), i, getReceiverIndex()));
- }
- return c;
- }
-
- @Override
- public void abort() {
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/PartitionCollector.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/PartitionCollector.java
new file mode 100644
index 0000000..da5b0bc
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/PartitionCollector.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.collectors;
+
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collection;
+
+import edu.uci.ics.hyracks.api.channels.IInputChannel;
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.comm.PartitionChannel;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
+import edu.uci.ics.hyracks.api.exceptions.HyracksException;
+import edu.uci.ics.hyracks.api.partitions.PartitionId;
+
+public class PartitionCollector extends AbstractPartitionCollector {
+ private final BitSet expectedPartitions;
+
+ private final IFrameReader frameReader;
+
+ private final IPartitionAcceptor pa;
+
+ public PartitionCollector(IHyracksTaskContext ctx, ConnectorDescriptorId connectorId, int receiverIndex,
+ BitSet expectedPartitions, IFrameReader frameReader, IPartitionAcceptor pa) {
+ super(ctx, connectorId, receiverIndex);
+ this.expectedPartitions = expectedPartitions;
+ this.frameReader = frameReader;
+ this.pa = pa;
+ }
+
+ @Override
+ public void open() throws HyracksException {
+ }
+
+ @Override
+ public void addPartitions(Collection<PartitionChannel> partitions) throws HyracksException {
+ for (PartitionChannel pc : partitions) {
+ PartitionId pid = pc.getPartitionId();
+ IInputChannel channel = pc.getInputChannel();
+ pa.addPartition(pid, channel);
+ channel.open();
+ }
+ }
+
+ @Override
+ public IFrameReader getReader() throws HyracksException {
+ return frameReader;
+ }
+
+ @Override
+ public void close() throws HyracksException {
+
+ }
+
+ @Override
+ public Collection<PartitionId> getRequiredPartitionIds() throws HyracksException {
+ Collection<PartitionId> c = new ArrayList<PartitionId>(expectedPartitions.cardinality());
+ for (int i = expectedPartitions.nextSetBit(0); i >= 0; i = expectedPartitions.nextSetBit(i + 1)) {
+ c.add(new PartitionId(getJobId(), getConnectorId(), i, getReceiverIndex()));
+ }
+ return c;
+ }
+
+ @Override
+ public void abort() {
+
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/SortMergeFrameReader.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/SortMergeFrameReader.java
new file mode 100644
index 0000000..5f41069
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/SortMergeFrameReader.java
@@ -0,0 +1,79 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.collectors;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.std.sort.RunMergingFrameReader;
+
+public class SortMergeFrameReader implements IFrameReader {
+ private IHyracksTaskContext ctx;
+ private final int maxConcurrentMerges;
+ private final int nSenders;
+ private final int[] sortFields;
+ private final IBinaryComparator[] comparators;
+ private final RecordDescriptor recordDescriptor;
+ private final IPartitionBatchManager pbm;
+
+ private RunMergingFrameReader merger;
+
+ public SortMergeFrameReader(IHyracksTaskContext ctx, int maxConcurrentMerges, int nSenders, int[] sortFields,
+ IBinaryComparator[] comparators, RecordDescriptor recordDescriptor, IPartitionBatchManager pbm) {
+ this.ctx = ctx;
+ this.maxConcurrentMerges = maxConcurrentMerges;
+ this.nSenders = nSenders;
+ this.sortFields = sortFields;
+ this.comparators = comparators;
+ this.recordDescriptor = recordDescriptor;
+ this.pbm = pbm;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ if (maxConcurrentMerges >= nSenders) {
+ List<ByteBuffer> inFrames = new ArrayList<ByteBuffer>();
+ for (int i = 0; i < nSenders; ++i) {
+ inFrames.add(ByteBuffer.allocate(ctx.getFrameSize()));
+ }
+ List<IFrameReader> batch = new ArrayList<IFrameReader>();
+ pbm.getNextBatch(batch, nSenders);
+ merger = new RunMergingFrameReader(ctx, batch.toArray(new IFrameReader[nSenders]), inFrames, sortFields,
+ comparators, recordDescriptor);
+ } else {
+ // multi level merge.
+ throw new HyracksDataException("Not yet supported");
+ }
+ merger.open();
+ }
+
+ @Override
+ public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ buffer.position(buffer.capacity());
+ buffer.limit(buffer.capacity());
+ return merger.nextFrame(buffer);
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ merger.close();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/SortMergePartitionCollector.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/SortMergePartitionCollector.java
deleted file mode 100644
index 96b2a69..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/collectors/SortMergePartitionCollector.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.collectors;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-
-import edu.uci.ics.hyracks.api.channels.IInputChannel;
-import edu.uci.ics.hyracks.api.channels.IInputChannelMonitor;
-import edu.uci.ics.hyracks.api.comm.IFrameReader;
-import edu.uci.ics.hyracks.api.comm.PartitionChannel;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.ConnectorDescriptorId;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.exceptions.HyracksException;
-import edu.uci.ics.hyracks.api.partitions.PartitionId;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.std.sort.RunMergingFrameReader;
-
-public class SortMergePartitionCollector extends AbstractPartitionCollector {
- private final int[] sortFields;
-
- private final IBinaryComparator[] comparators;
-
- private final RecordDescriptor recordDescriptor;
-
- private final int maxConcurrentMerges;
-
- private final IInputChannel[] channels;
-
- private final int nSenders;
-
- private final boolean stable;
-
- private final FrameReader frameReader;
-
- private final PartitionBatchManager pbm;
-
- public SortMergePartitionCollector(IHyracksTaskContext ctx, ConnectorDescriptorId connectorId, int receiverIndex,
- int[] sortFields, IBinaryComparator[] comparators, RecordDescriptor recordDescriptor,
- int maxConcurrentMerges, int nSenders, boolean stable) {
- super(ctx, connectorId, receiverIndex);
- this.sortFields = sortFields;
- this.comparators = comparators;
- this.recordDescriptor = recordDescriptor;
- this.maxConcurrentMerges = maxConcurrentMerges;
- channels = new IInputChannel[nSenders];
- this.nSenders = nSenders;
- this.stable = stable;
- this.frameReader = new FrameReader();
- pbm = new NonDeterministicPartitionBatchManager();
- }
-
- @Override
- public void open() throws HyracksException {
- }
-
- @Override
- public void addPartitions(Collection<PartitionChannel> partitions) throws HyracksException {
- for (PartitionChannel pc : partitions) {
- PartitionId pid = pc.getPartitionId();
- IInputChannel channel = pc.getInputChannel();
- InputChannelFrameReader channelReader = new InputChannelFrameReader(channel);
- channel.registerMonitor(channelReader);
- channel.setAttachment(channelReader);
- int senderIndex = pid.getSenderIndex();
- synchronized (this) {
- channels[senderIndex] = channel;
- }
- pbm.addPartition(senderIndex);
- channel.open();
- }
- }
-
- @Override
- public IFrameReader getReader() throws HyracksException {
- return frameReader;
- }
-
- @Override
- public void close() throws HyracksException {
-
- }
-
- @Override
- public Collection<PartitionId> getRequiredPartitionIds() throws HyracksException {
- Collection<PartitionId> requiredPartitionIds = new ArrayList<PartitionId>();
- for (int i = 0; i < nSenders; ++i) {
- requiredPartitionIds.add(new PartitionId(getJobId(), getConnectorId(), i, receiverIndex));
- }
- return requiredPartitionIds;
- }
-
- @Override
- public void abort() {
-
- }
-
- private abstract class PartitionBatchManager {
- protected abstract void addPartition(int index);
-
- protected abstract void getNextBatch(List<IFrameReader> batch, int size) throws HyracksDataException;
- }
-
- private class NonDeterministicPartitionBatchManager extends PartitionBatchManager {
- private List<IFrameReader> partitions;
-
- private List<IFrameReader> batch;
-
- private int requiredSize;
-
- public NonDeterministicPartitionBatchManager() {
- partitions = new ArrayList<IFrameReader>();
- }
-
- @Override
- protected void addPartition(int index) {
- synchronized (SortMergePartitionCollector.this) {
- if (batch != null && batch.size() < requiredSize) {
- batch.add((IFrameReader) channels[index].getAttachment());
- if (batch.size() == requiredSize) {
- SortMergePartitionCollector.this.notifyAll();
- }
- } else {
- partitions.add((IFrameReader) channels[index].getAttachment());
- }
- }
- }
-
- @Override
- protected void getNextBatch(List<IFrameReader> batch, int size) throws HyracksDataException {
- synchronized (SortMergePartitionCollector.this) {
- if (partitions.size() <= size) {
- batch.addAll(partitions);
- partitions.clear();
- } else if (partitions.size() > size) {
- List<IFrameReader> sublist = partitions.subList(0, size);
- batch.addAll(sublist);
- sublist.clear();
- }
- if (batch.size() == size) {
- return;
- }
- this.batch = batch;
- this.requiredSize = size;
- while (batch.size() < size) {
- try {
- SortMergePartitionCollector.this.wait();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
- }
- }
- this.batch = null;
- }
- }
- }
-
- private static class InputChannelFrameReader implements IFrameReader, IInputChannelMonitor {
- private final IInputChannel channel;
-
- private int availableFrames;
-
- private boolean eos;
-
- private boolean failed;
-
- public InputChannelFrameReader(IInputChannel channel) {
- this.channel = channel;
- availableFrames = 0;
- eos = false;
- failed = false;
- }
-
- @Override
- public void open() throws HyracksDataException {
- }
-
- @Override
- public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
- synchronized (this) {
- while (!failed && !eos && availableFrames <= 0) {
- try {
- wait();
- } catch (InterruptedException e) {
- throw new HyracksDataException(e);
- }
- }
- if (failed) {
- throw new HyracksDataException("Failure occurred on input");
- }
- if (availableFrames <= 0 && eos) {
- return false;
- }
- --availableFrames;
- }
- ByteBuffer srcBuffer = channel.getNextBuffer();
- FrameUtils.copy(srcBuffer, buffer);
- channel.recycleBuffer(srcBuffer);
- return true;
- }
-
- @Override
- public void close() throws HyracksDataException {
-
- }
-
- @Override
- public synchronized void notifyFailure(IInputChannel channel) {
- failed = true;
- notifyAll();
- }
-
- @Override
- public synchronized void notifyDataAvailability(IInputChannel channel, int nFrames) {
- availableFrames += nFrames;
- notifyAll();
- }
-
- @Override
- public synchronized void notifyEndOfStream(IInputChannel channel) {
- eos = true;
- notifyAll();
- }
- }
-
- private class FrameReader implements IFrameReader {
- private RunMergingFrameReader merger;
-
- @Override
- public void open() throws HyracksDataException {
- if (maxConcurrentMerges >= nSenders) {
- List<ByteBuffer> inFrames = new ArrayList<ByteBuffer>();
- for (int i = 0; i < nSenders; ++i) {
- inFrames.add(ByteBuffer.allocate(ctx.getFrameSize()));
- }
- List<IFrameReader> batch = new ArrayList<IFrameReader>();
- pbm.getNextBatch(batch, nSenders);
- merger = new RunMergingFrameReader(ctx, batch.toArray(new IFrameReader[nSenders]), inFrames,
- sortFields, comparators, recordDescriptor);
- } else {
- // multi level merge.
- throw new HyracksDataException("Not yet supported");
- }
- merger.open();
- }
-
- @Override
- public boolean nextFrame(ByteBuffer buffer) throws HyracksDataException {
- buffer.position(buffer.capacity());
- buffer.limit(buffer.capacity());
- return merger.nextFrame(buffer);
- }
-
- @Override
- public void close() throws HyracksDataException {
- merger.close();
- }
- }
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/GlobalHashingLocalityMap.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/GlobalHashingLocalityMap.java
new file mode 100644
index 0000000..f3086f0
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/GlobalHashingLocalityMap.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.connectors;
+
+public class GlobalHashingLocalityMap implements ILocalityMap {
+
+ private static final long serialVersionUID = 1L;
+
+ private int[] consumers;
+
+ /* (non-Javadoc)
+ * @see edu.uci.ics.hyracks.examples.text.client.aggregation.helpers.ILocalityMap#getConsumers(int)
+ */
+ @Override
+ public int[] getConsumers(int senderID, int nConsumerPartitions) {
+ if (consumers == null) {
+ consumers = new int[nConsumerPartitions];
+ for (int i = 0; i < consumers.length; i++) {
+ consumers[i] = i;
+ }
+ }
+ return consumers;
+ }
+
+ /* (non-Javadoc)
+ * @see edu.uci.ics.hyracks.examples.text.client.aggregation.helpers.ILocalityMap#getConsumerPartitionCount()
+ */
+ @Override
+ public int getConsumerPartitionCount(int nConsumerPartitions) {
+ return nConsumerPartitions;
+ }
+
+ @Override
+ public boolean isConnected(int senderID, int receiverID, int nConsumerPartitions) {
+ return true;
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/HashtableLocalityMap.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/HashtableLocalityMap.java
new file mode 100644
index 0000000..a187a59
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/HashtableLocalityMap.java
@@ -0,0 +1,72 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.connectors;
+
+import java.util.BitSet;
+
+public class HashtableLocalityMap implements ILocalityMap {
+
+ private static final long serialVersionUID = 1L;
+
+ private final BitSet nodeMap;
+
+ public HashtableLocalityMap(BitSet nodeMap) {
+ this.nodeMap = nodeMap;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.examples.text.client.aggregation.helpers.ILocalityMap
+ * #getConsumers(int, int)
+ */
+ @Override
+ public int[] getConsumers(int senderID, int nConsumerPartitions) {
+ int consumersForSender = 0;
+ // Get the count of consumers
+ for (int i = senderID * nConsumerPartitions; i < (senderID + 1) * nConsumerPartitions; i++) {
+ if (nodeMap.get(i))
+ consumersForSender++;
+ }
+ int[] cons = new int[consumersForSender];
+ int consIdx = 0;
+ for (int i = senderID * nConsumerPartitions; i < (senderID + 1) * nConsumerPartitions; i++) {
+ if (nodeMap.get(i)) {
+ cons[consIdx] = i - senderID * nConsumerPartitions;
+ consIdx++;
+ }
+ }
+ return cons;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.examples.text.client.aggregation.helpers.ILocalityMap
+ * #getConsumerPartitionCount(int)
+ */
+ @Override
+ public int getConsumerPartitionCount(int nConsumerPartitions) {
+ return nConsumerPartitions;
+ }
+
+ @Override
+ public boolean isConnected(int senderID, int receiverID, int nConsumerPartitions) {
+ return nodeMap.get(senderID * nConsumerPartitions + receiverID);
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/ILocalityMap.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/ILocalityMap.java
new file mode 100644
index 0000000..b30971d
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/ILocalityMap.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.connectors;
+
+import java.io.Serializable;
+
+public interface ILocalityMap extends Serializable {
+
+ public int[] getConsumers(int senderID, int nConsumerPartitions);
+
+ public boolean isConnected(int senderID, int receiverID, int nConsumerPartitions);
+
+ public int getConsumerPartitionCount(int nConsumerPartitions);
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java
new file mode 100644
index 0000000..0e1e413
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwareMToNPartitioningConnectorDescriptor.java
@@ -0,0 +1,87 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.connectors;
+
+import java.util.BitSet;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.comm.IPartitionCollector;
+import edu.uci.ics.hyracks.api.comm.IPartitionWriterFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicFrameReader;
+import edu.uci.ics.hyracks.dataflow.std.collectors.PartitionCollector;
+
+public class LocalityAwareMToNPartitioningConnectorDescriptor extends AbstractMToNConnectorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ private ILocalityMap localityMap;
+
+ private ITuplePartitionComputerFactory tpcf;
+
+ public LocalityAwareMToNPartitioningConnectorDescriptor(JobSpecification spec, ITuplePartitionComputerFactory tpcf,
+ ILocalityMap localityMap) {
+ super(spec);
+ this.localityMap = localityMap;
+ this.tpcf = tpcf;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor#createPartitioner
+ * (edu.uci.ics.hyracks.api.context.IHyracksTaskContext,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor,
+ * edu.uci.ics.hyracks.api.comm.IPartitionWriterFactory, int, int, int)
+ */
+ @Override
+ public IFrameWriter createPartitioner(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
+ IPartitionWriterFactory edwFactory, int index, int nProducerPartitions, int nConsumerPartitions)
+ throws HyracksDataException {
+ return new LocalityAwarePartitionDataWriter(ctx, edwFactory, recordDesc, tpcf.createPartitioner(),
+ nConsumerPartitions, localityMap, index);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor#
+ * createPartitionCollector
+ * (edu.uci.ics.hyracks.api.context.IHyracksTaskContext,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor, int, int, int)
+ */
+ @Override
+ public IPartitionCollector createPartitionCollector(IHyracksTaskContext ctx, RecordDescriptor recordDesc,
+ int receiverIndex, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
+ BitSet expectedPartitions = new BitSet(nProducerPartitions);
+ for (int i = 0; i < nProducerPartitions; i++) {
+ if (localityMap.isConnected(i, receiverIndex, nConsumerPartitions))
+ expectedPartitions.set(i);
+ }
+ NonDeterministicChannelReader channelReader = new NonDeterministicChannelReader(nProducerPartitions,
+ expectedPartitions);
+ NonDeterministicFrameReader frameReader = new NonDeterministicFrameReader(channelReader);
+ return new PartitionCollector(ctx, getConnectorId(), receiverIndex, expectedPartitions, frameReader,
+ channelReader);
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwarePartitionDataWriter.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwarePartitionDataWriter.java
new file mode 100644
index 0000000..6ec9013
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/LocalityAwarePartitionDataWriter.java
@@ -0,0 +1,125 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.connectors;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.comm.IPartitionWriterFactory;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+
+public class LocalityAwarePartitionDataWriter implements IFrameWriter {
+
+ private final IFrameWriter[] pWriters;
+ private final FrameTupleAppender[] appenders;
+ private final FrameTupleAccessor tupleAccessor;
+ private final ITuplePartitionComputer tpc;
+
+ public LocalityAwarePartitionDataWriter(IHyracksTaskContext ctx, IPartitionWriterFactory pwFactory,
+ RecordDescriptor recordDescriptor, ITuplePartitionComputer tpc, int nConsumerPartitions,
+ ILocalityMap localityMap, int senderIndex) throws HyracksDataException {
+ int[] consumerPartitions = localityMap.getConsumers(senderIndex, nConsumerPartitions);
+ pWriters = new IFrameWriter[consumerPartitions.length];
+ appenders = new FrameTupleAppender[consumerPartitions.length];
+ for (int i = 0; i < consumerPartitions.length; ++i) {
+ try {
+ pWriters[i] = pwFactory.createFrameWriter(consumerPartitions[i]);
+ appenders[i] = new FrameTupleAppender(ctx.getFrameSize());
+ appenders[i].reset(ctx.allocateFrame(), true);
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ tupleAccessor = new FrameTupleAccessor(ctx.getFrameSize(), recordDescriptor);
+ this.tpc = tpc;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see edu.uci.ics.hyracks.api.comm.IFrameWriter#open()
+ */
+ @Override
+ public void open() throws HyracksDataException {
+ for (int i = 0; i < pWriters.length; ++i) {
+ pWriters[i].open();
+ appenders[i].reset(appenders[i].getBuffer(), true);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.api.comm.IFrameWriter#nextFrame(java.nio.ByteBuffer)
+ */
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ tupleAccessor.reset(buffer);
+ int tupleCount = tupleAccessor.getTupleCount();
+ for (int i = 0; i < tupleCount; ++i) {
+ int h = pWriters.length == 1 ? 0 : tpc.partition(tupleAccessor, i, pWriters.length);
+ FrameTupleAppender appender = appenders[h];
+ if (!appender.append(tupleAccessor, i)) {
+ ByteBuffer appenderBuffer = appender.getBuffer();
+ flushFrame(appenderBuffer, pWriters[h]);
+ appender.reset(appenderBuffer, true);
+ if (!appender.append(tupleAccessor, i)) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see edu.uci.ics.hyracks.api.comm.IFrameWriter#fail()
+ */
+ @Override
+ public void fail() throws HyracksDataException {
+ for (int i = 0; i < appenders.length; ++i) {
+ pWriters[i].fail();
+ }
+ }
+
+ private void flushFrame(ByteBuffer buffer, IFrameWriter frameWriter) throws HyracksDataException {
+ buffer.position(0);
+ buffer.limit(buffer.capacity());
+ frameWriter.nextFrame(buffer);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see edu.uci.ics.hyracks.api.comm.IFrameWriter#close()
+ */
+ @Override
+ public void close() throws HyracksDataException {
+ for (int i = 0; i < pWriters.length; ++i) {
+ if (appenders[i].getTupleCount() > 0) {
+ flushFrame(appenders[i].getBuffer(), pWriters[i]);
+ }
+ pWriters[i].close();
+ }
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
index 12935af..291e149 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningConnectorDescriptor.java
@@ -25,7 +25,9 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicPartitionCollector;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicFrameReader;
+import edu.uci.ics.hyracks.dataflow.std.collectors.PartitionCollector;
public class MToNPartitioningConnectorDescriptor extends AbstractMToNConnectorDescriptor {
private static final long serialVersionUID = 1L;
@@ -50,7 +52,9 @@
int index, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
BitSet expectedPartitions = new BitSet(nProducerPartitions);
expectedPartitions.set(0, nProducerPartitions);
- return new NonDeterministicPartitionCollector(ctx, getConnectorId(), index, nProducerPartitions,
+ NonDeterministicChannelReader channelReader = new NonDeterministicChannelReader(nProducerPartitions,
expectedPartitions);
+ NonDeterministicFrameReader frameReader = new NonDeterministicFrameReader(channelReader);
+ return new PartitionCollector(ctx, getConnectorId(), index, expectedPartitions, frameReader, channelReader);
}
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
index 54979f0..2576423 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNPartitioningMergingConnectorDescriptor.java
@@ -14,6 +14,9 @@
*/
package edu.uci.ics.hyracks.dataflow.std.connectors;
+import java.util.BitSet;
+
+import edu.uci.ics.hyracks.api.comm.IFrameReader;
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.comm.IPartitionCollector;
import edu.uci.ics.hyracks.api.comm.IPartitionWriterFactory;
@@ -25,7 +28,10 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.collectors.SortMergePartitionCollector;
+import edu.uci.ics.hyracks.dataflow.std.collectors.IPartitionBatchManager;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicPartitionBatchManager;
+import edu.uci.ics.hyracks.dataflow.std.collectors.PartitionCollector;
+import edu.uci.ics.hyracks.dataflow.std.collectors.SortMergeFrameReader;
public class MToNPartitioningMergingConnectorDescriptor extends AbstractMToNConnectorDescriptor {
private static final long serialVersionUID = 1L;
@@ -65,7 +71,11 @@
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
}
- return new SortMergePartitionCollector(ctx, getConnectorId(), index, sortFields, comparators, recordDesc,
- nProducerPartitions, nProducerPartitions, stable);
+ IPartitionBatchManager pbm = new NonDeterministicPartitionBatchManager(nProducerPartitions);
+ IFrameReader sortMergeFrameReader = new SortMergeFrameReader(ctx, nProducerPartitions, nProducerPartitions,
+ sortFields, comparators, recordDesc, pbm);
+ BitSet expectedPartitions = new BitSet();
+ expectedPartitions.set(0, nProducerPartitions);
+ return new PartitionCollector(ctx, getConnectorId(), index, expectedPartitions, sortMergeFrameReader, pbm);
}
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNReplicatingConnectorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNReplicatingConnectorDescriptor.java
index ba66ebc..fd67f77 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNReplicatingConnectorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/MToNReplicatingConnectorDescriptor.java
@@ -25,7 +25,9 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractMToNConnectorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicPartitionCollector;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicFrameReader;
+import edu.uci.ics.hyracks.dataflow.std.collectors.PartitionCollector;
public class MToNReplicatingConnectorDescriptor extends AbstractMToNConnectorDescriptor {
public MToNReplicatingConnectorDescriptor(JobSpecification spec) {
@@ -82,7 +84,9 @@
int index, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
BitSet expectedPartitions = new BitSet(nProducerPartitions);
expectedPartitions.set(0, nProducerPartitions);
- return new NonDeterministicPartitionCollector(ctx, getConnectorId(), index, nProducerPartitions,
+ NonDeterministicChannelReader channelReader = new NonDeterministicChannelReader(nProducerPartitions,
expectedPartitions);
+ NonDeterministicFrameReader frameReader = new NonDeterministicFrameReader(channelReader);
+ return new PartitionCollector(ctx, getConnectorId(), index, expectedPartitions, frameReader, channelReader);
}
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
index 67130c9..e8d9751 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/connectors/OneToOneConnectorDescriptor.java
@@ -30,7 +30,9 @@
import edu.uci.ics.hyracks.api.job.JobActivityGraph;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractConnectorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicPartitionCollector;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicChannelReader;
+import edu.uci.ics.hyracks.dataflow.std.collectors.NonDeterministicFrameReader;
+import edu.uci.ics.hyracks.dataflow.std.collectors.PartitionCollector;
public class OneToOneConnectorDescriptor extends AbstractConnectorDescriptor {
private static final long serialVersionUID = 1L;
@@ -51,8 +53,10 @@
int index, int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException {
BitSet expectedPartitions = new BitSet(nProducerPartitions);
expectedPartitions.set(index);
- return new NonDeterministicPartitionCollector(ctx, getConnectorId(), index, nProducerPartitions,
+ NonDeterministicChannelReader channelReader = new NonDeterministicChannelReader(nProducerPartitions,
expectedPartitions);
+ NonDeterministicFrameReader frameReader = new NonDeterministicFrameReader(channelReader);
+ return new PartitionCollector(ctx, getConnectorId(), index, expectedPartitions, frameReader, channelReader);
}
@Override
@@ -72,4 +76,11 @@
targetBitmap.clear();
targetBitmap.set(producerIndex);
}
+
+ @Override
+ public void indicateSourcePartitions(int nProducerPartitions, int nConsumerPartitions, int consumerIndex,
+ BitSet sourceBitmap) {
+ sourceBitmap.clear();
+ sourceBitmap.set(consumerIndex);
+ }
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
index f6511b4..8544891 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractDeserializedFileScanOperatorDescriptor.java
@@ -22,7 +22,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
@@ -76,7 +75,7 @@
writer.open();
try {
while (true) {
- Object[] record = new Object[desc.getFields().length];
+ Object[] record = new Object[desc.getFieldCount()];
if (!reader.read(record)) {
break;
}
@@ -108,7 +107,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
return new DeserializedOperatorNodePushable(ctx, new DeserializedFileScanOperator(partition), null);
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
index 4bb4eb9..1744b99 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/AbstractFileWriteOperatorDescriptor.java
@@ -19,7 +19,6 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
@@ -89,7 +88,7 @@
protected abstract IRecordWriter createRecordWriter(FileSplit fileSplit, int index) throws Exception;
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
return new DeserializedOperatorNodePushable(ctx, new FileWriteOperator(partition),
recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/DelimitedDataTupleParserFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/DelimitedDataTupleParserFactory.java
index 0096576..7cf437d 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/DelimitedDataTupleParserFactory.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/DelimitedDataTupleParserFactory.java
@@ -86,11 +86,7 @@
}
private enum State {
- INIT,
- IN_RECORD,
- EOR,
- CR,
- EOF
+ INIT, IN_RECORD, EOR, CR, EOF
}
private class FieldCursor {
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
index 57fa156..76cb0ae 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FileScanOperatorDescriptor.java
@@ -24,7 +24,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
@@ -45,7 +44,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
final FileSplit split = fileSplitProvider.getFileSplits()[partition];
final ITupleParser tp = tupleParserFactory.createTupleParser(ctx);
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
index 24d76aa..8397d29 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java
@@ -24,7 +24,6 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
@@ -40,7 +39,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
final FileSplit[] splits = fileSplitProvider.getFileSplits();
return new AbstractUnaryInputSinkOperatorNodePushable() {
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
index 42c86ca..6bf29d2 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java
@@ -25,7 +25,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
@@ -57,11 +56,18 @@
this.delim = delim;
}
- /* (non-Javadoc)
- * @see edu.uci.ics.hyracks.api.dataflow.IActivityNode#createPushRuntime(edu.uci.ics.hyracks.api.context.IHyracksContext, edu.uci.ics.hyracks.api.job.IOperatorEnvironment, edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider, int, int)
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.api.dataflow.IActivityNode#createPushRuntime(edu.
+ * uci.ics.hyracks.api.context.IHyracksContext,
+ * edu.uci.ics.hyracks.api.job.IOperatorEnvironment,
+ * edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider, int,
+ * int)
*/
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
throws HyracksDataException {
// Output files
@@ -97,7 +103,7 @@
int start = frameTupleAccessor.getTupleStartOffset(tIndex)
+ frameTupleAccessor.getFieldSlotsLength();
bbis.setByteBuffer(buffer, start);
- Object[] record = new Object[recordDescriptor.getFields().length];
+ Object[] record = new Object[recordDescriptor.getFieldCount()];
for (int i = 0; i < record.length; ++i) {
Object instance = recordDescriptor.getFields()[i].deserialize(di);
if (i == 0) {
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
index 38ac1a3..af345aa 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/file/RecordFileScanOperatorDescriptor.java
@@ -69,11 +69,14 @@
@Override
protected void configure() throws Exception {
- // currently a no-op, but is meant to initialize , if required before it is asked
+ // currently a no-op, but is meant to initialize , if required before it
+ // is asked
// to create a record reader
- // this is executed at the node and is useful for operators that could not be
- // initialized from the client completely, because of lack of information specific
- // to the node where the operator gets executed.
+ // this is executed at the node and is useful for operators that could
+ // not be
+ // initialized from the client completely, because of lack of
+ // information specific
+ // to the node where the operator gets executed.
}
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/AggregateState.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/AggregateState.java
new file mode 100644
index 0000000..e72f85c
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/AggregateState.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group;
+
+import java.io.Serializable;
+
+/**
+ *
+ */
+public class AggregateState implements Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ public Object state = null;
+
+ public AggregateState() {
+ state = null;
+ }
+
+ public AggregateState(Object obj) {
+ state = obj;
+ }
+
+ public void reset() {
+ state = null;
+ }
+
+ public void close() {
+ state = null;
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/DeserializedPreclusteredGroupOperator.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/DeserializedPreclusteredGroupOperator.java
index 4ac72f6..370f6d0 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/DeserializedPreclusteredGroupOperator.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/DeserializedPreclusteredGroupOperator.java
@@ -90,11 +90,6 @@
}
@Override
- public void fail() throws HyracksDataException {
- writer.fail();
- }
-
- @Override
public void setDataWriter(int index, IOpenableDataWriter<Object[]> writer) {
if (index != 0) {
throw new IllegalArgumentException();
@@ -121,4 +116,10 @@
}
return 0;
}
+
+ @Override
+ public void fail() throws HyracksDataException {
+ // TODO Auto-generated method stub
+
+ }
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java
index 5f78421..1cab78d 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ExternalGroupOperatorDescriptor.java
@@ -37,7 +37,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -46,8 +45,6 @@
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
import edu.uci.ics.hyracks.dataflow.common.io.RunFileWriter;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IAggregatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IAggregatorDescriptorFactory;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractTaskState;
@@ -56,7 +53,11 @@
import edu.uci.ics.hyracks.dataflow.std.util.ReferenceEntry;
import edu.uci.ics.hyracks.dataflow.std.util.ReferencedPriorityQueue;
+/**
+ *
+ */
public class ExternalGroupOperatorDescriptor extends AbstractOperatorDescriptor {
+
private static final int AGGREGATE_ACTIVITY_ID = 0;
private static final int MERGE_ACTIVITY_ID = 1;
@@ -65,15 +66,17 @@
private final int[] keyFields;
private final IBinaryComparatorFactory[] comparatorFactories;
private final INormalizedKeyComputerFactory firstNormalizerFactory;
+
private final IAggregatorDescriptorFactory aggregatorFactory;
- private final IAggregatorDescriptorFactory mergeFactory;
+ private final IAggregatorDescriptorFactory mergerFactory;
+
private final int framesLimit;
private final ISpillableTableFactory spillableTableFactory;
private final boolean isOutputSorted;
public ExternalGroupOperatorDescriptor(JobSpecification spec, int[] keyFields, int framesLimit,
IBinaryComparatorFactory[] comparatorFactories, INormalizedKeyComputerFactory firstNormalizerFactory,
- IAggregatorDescriptorFactory aggregatorFactory, IAggregatorDescriptorFactory mergeFactory,
+ IAggregatorDescriptorFactory aggregatorFactory, IAggregatorDescriptorFactory mergerFactory,
RecordDescriptor recordDescriptor, ISpillableTableFactory spillableTableFactory, boolean isOutputSorted) {
super(spec, 1, 1);
this.framesLimit = framesLimit;
@@ -84,9 +87,8 @@
*/
throw new IllegalStateException("frame limit should at least be 2, but it is " + framesLimit + "!");
}
-
this.aggregatorFactory = aggregatorFactory;
- this.mergeFactory = mergeFactory;
+ this.mergerFactory = mergerFactory;
this.keyFields = keyFields;
this.comparatorFactories = comparatorFactories;
this.firstNormalizerFactory = firstNormalizerFactory;
@@ -94,13 +96,19 @@
this.isOutputSorted = isOutputSorted;
/**
- * Set the record descriptor. Note that since
- * this operator is a unary operator,
- * only the first record descriptor is used here.
+ * Set the record descriptor. Note that since this operator is a unary
+ * operator, only the first record descriptor is used here.
*/
recordDescriptors[0] = recordDescriptor;
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor#contributeActivities
+ * (edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder)
+ */
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
AggregateActivity aggregateAct = new AggregateActivity(new ActivityId(getOperatorId(), AGGREGATE_ACTIVITY_ID));
@@ -146,7 +154,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
throws HyracksDataException {
final FrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(),
@@ -186,7 +194,7 @@
@Override
public void fail() throws HyracksDataException {
-
+ throw new HyracksDataException("failed");
}
@Override
@@ -201,7 +209,7 @@
state.gTable = null;
}
}
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
private void flushFramesToRun() throws HyracksDataException {
@@ -238,15 +246,23 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions)
throws HyracksDataException {
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
}
- final IAggregatorDescriptor currentWorkingAggregator = mergeFactory.createAggregator(ctx,
- recordDescriptors[0], recordDescriptors[0], keyFields);
+
+ int[] keyFieldsInPartialResults = new int[keyFields.length];
+ for (int i = 0; i < keyFieldsInPartialResults.length; i++) {
+ keyFieldsInPartialResults[i] = i;
+ }
+
+ final IAggregatorDescriptor aggregator = mergerFactory.createAggregator(ctx, recordDescriptors[0],
+ recordDescriptors[0], keyFields, keyFieldsInPartialResults);
+ final AggregateState aggregateState = aggregator.createAggregateStates();
+
final int[] storedKeys = new int[keyFields.length];
/**
* Get the list of the fields in the stored records.
@@ -254,9 +270,7 @@
for (int i = 0; i < keyFields.length; ++i) {
storedKeys[i] = i;
}
- /**
- * Tuple builder
- */
+
final ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(recordDescriptors[0].getFields().length);
IOperatorNodePushable op = new AbstractUnaryOutputSourceOperatorNodePushable() {
@@ -269,11 +283,15 @@
* Output frame.
*/
private ByteBuffer outFrame, writerFrame;
+ private final FrameTupleAppender outAppender = new FrameTupleAppender(ctx.getFrameSize());
+ private FrameTupleAppender writerAppender;
private LinkedList<RunFileReader> runs;
private AggregateActivityState aggState;
+ private ArrayTupleBuilder finalTupleBuilder;
+
/**
* how many frames to be read ahead once
*/
@@ -281,14 +299,12 @@
private int[] currentFrameIndexInRun;
private int[] currentRunFrames;
- private final FrameTupleAppender outFrameAppender = new FrameTupleAppender(ctx.getFrameSize());
+
private final FrameTupleAccessor outFrameAccessor = new FrameTupleAccessor(ctx.getFrameSize(),
recordDescriptors[0]);
- private ArrayTupleBuilder finalTupleBuilder;
- private FrameTupleAppender writerFrameAppender;
public void initialize() throws HyracksDataException {
- aggState = (AggregateActivityState) env.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ aggState = (AggregateActivityState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
AGGREGATE_ACTIVITY_ID), partition));
runs = aggState.runs;
writer.open();
@@ -300,11 +316,16 @@
gTable.sortFrames();
gTable.flushFrames(writer, false);
}
+ gTable = null;
+ aggState = null;
+ System.gc();
} else {
+ aggState = null;
+ System.gc();
runs = new LinkedList<RunFileReader>(runs);
inFrames = new ArrayList<ByteBuffer>();
outFrame = ctx.allocateFrame();
- outFrameAppender.reset(outFrame, true);
+ outAppender.reset(outFrame, true);
outFrameAccessor.reset(outFrame);
while (runs.size() > 0) {
try {
@@ -319,6 +340,7 @@
writer.fail();
throw new HyracksDataException(e);
} finally {
+ aggregateState.close();
writer.close();
}
}
@@ -347,8 +369,8 @@
currentFrameIndexInRun = new int[runNumber];
currentRunFrames = new int[runNumber];
/**
- * Create file readers for each input run file, only
- * for the ones fit into the inFrames
+ * Create file readers for each input run file, only for
+ * the ones fit into the inFrames
*/
RunFileReader[] runFileReaders = new RunFileReader[runNumber];
FrameTupleAccessor[] tupleAccessors = new FrameTupleAccessor[inFrames.size()];
@@ -400,47 +422,50 @@
if (currentTupleInOutFrame < 0
|| compareFrameTuples(fta, tupleIndex, outFrameAccessor, currentTupleInOutFrame) != 0) {
/**
- * Initialize the first output record
- * Reset the tuple builder
+ * Initialize the first output record Reset the
+ * tuple builder
*/
+
tupleBuilder.reset();
- for (int i = 0; i < keyFields.length; i++) {
- tupleBuilder.addField(fta, tupleIndex, i);
+
+ for(int k = 0; k < storedKeys.length; k++){
+ tupleBuilder.addField(fta, tupleIndex, storedKeys[k]);
}
- currentWorkingAggregator.init(fta, tupleIndex, tupleBuilder);
- if (!outFrameAppender.append(tupleBuilder.getFieldEndOffsets(),
+ aggregator.init(tupleBuilder, fta, tupleIndex, aggregateState);
+
+ if (!outAppender.appendSkipEmptyField(tupleBuilder.getFieldEndOffsets(),
tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
flushOutFrame(writer, finalPass);
- if (!outFrameAppender.append(tupleBuilder.getFieldEndOffsets(),
- tupleBuilder.getByteArray(), 0, tupleBuilder.getSize()))
+ if (!outAppender.appendSkipEmptyField(tupleBuilder.getFieldEndOffsets(),
+ tupleBuilder.getByteArray(), 0, tupleBuilder.getSize())) {
throw new HyracksDataException(
- "Failed to append an aggregation result to the output frame.");
+ "The partial result is too large to be initialized in a frame.");
+ }
}
+
} else {
/**
* if new tuple is in the same group of the
- * current aggregator
- * do merge and output to the outFrame
+ * current aggregator do merge and output to the
+ * outFrame
*/
- int tupleOffset = outFrameAccessor.getTupleStartOffset(currentTupleInOutFrame);
- int fieldOffset = outFrameAccessor.getFieldStartOffset(currentTupleInOutFrame,
- keyFields.length);
- int fieldLength = outFrameAccessor.getFieldLength(currentTupleInOutFrame,
- keyFields.length);
- currentWorkingAggregator.aggregate(fta, tupleIndex, outFrameAccessor.getBuffer()
- .array(), tupleOffset + outFrameAccessor.getFieldSlotsLength() + fieldOffset,
- fieldLength);
+
+ aggregator.aggregate(fta, tupleIndex, outFrameAccessor, currentTupleInOutFrame,
+ aggregateState);
+
}
tupleIndices[runIndex]++;
setNextTopTuple(runIndex, tupleIndices, runFileReaders, tupleAccessors, topTuples);
}
- if (outFrameAppender.getTupleCount() > 0) {
+ if (outAppender.getTupleCount() > 0) {
flushOutFrame(writer, finalPass);
+ outAppender.reset(outFrame, true);
}
- currentWorkingAggregator.close();
+ aggregator.close();
+
runs.subList(0, runNumber).clear();
/**
* insert the new run file into the beginning of the run
@@ -457,42 +482,56 @@
}
private void flushOutFrame(IFrameWriter writer, boolean isFinal) throws HyracksDataException {
+
if (finalTupleBuilder == null) {
finalTupleBuilder = new ArrayTupleBuilder(recordDescriptors[0].getFields().length);
}
+
if (writerFrame == null) {
writerFrame = ctx.allocateFrame();
}
- if (writerFrameAppender == null) {
- writerFrameAppender = new FrameTupleAppender(ctx.getFrameSize());
- writerFrameAppender.reset(writerFrame, true);
- }
- outFrameAccessor.reset(outFrame);
- for (int i = 0; i < outFrameAccessor.getTupleCount(); i++) {
- finalTupleBuilder.reset();
- for (int j = 0; j < keyFields.length; j++) {
- finalTupleBuilder.addField(outFrameAccessor, i, j);
- }
- if (isFinal)
- currentWorkingAggregator.outputResult(outFrameAccessor, i, finalTupleBuilder);
- else
- currentWorkingAggregator.outputPartialResult(outFrameAccessor, i, finalTupleBuilder);
- if (!writerFrameAppender.append(finalTupleBuilder.getFieldEndOffsets(),
+ if (writerAppender == null) {
+ writerAppender = new FrameTupleAppender(ctx.getFrameSize());
+ writerAppender.reset(writerFrame, true);
+ }
+
+ outFrameAccessor.reset(outFrame);
+
+ for (int i = 0; i < outFrameAccessor.getTupleCount(); i++) {
+
+ finalTupleBuilder.reset();
+
+ for (int k = 0; k < storedKeys.length; k++) {
+ finalTupleBuilder.addField(outFrameAccessor, i, storedKeys[k]);
+ }
+
+ if (isFinal) {
+
+ aggregator.outputFinalResult(finalTupleBuilder, outFrameAccessor, i, aggregateState);
+
+ } else {
+
+ aggregator.outputPartialResult(finalTupleBuilder, outFrameAccessor, i, aggregateState);
+ }
+
+ if (!writerAppender.appendSkipEmptyField(finalTupleBuilder.getFieldEndOffsets(),
finalTupleBuilder.getByteArray(), 0, finalTupleBuilder.getSize())) {
FrameUtils.flushFrame(writerFrame, writer);
- writerFrameAppender.reset(writerFrame, true);
- if (!writerFrameAppender.append(finalTupleBuilder.getFieldEndOffsets(),
- finalTupleBuilder.getByteArray(), 0, finalTupleBuilder.getSize()))
+ writerAppender.reset(writerFrame, true);
+ if (!writerAppender.appendSkipEmptyField(finalTupleBuilder.getFieldEndOffsets(),
+ finalTupleBuilder.getByteArray(), 0, finalTupleBuilder.getSize())) {
throw new HyracksDataException(
- "Failed to write final aggregation result to a writer frame!");
+ "Aggregation output is too large to be fit into a frame.");
+ }
}
}
- if (writerFrameAppender.getTupleCount() > 0) {
+ if (writerAppender.getTupleCount() > 0) {
FrameUtils.flushFrame(writerFrame, writer);
- writerFrameAppender.reset(writerFrame, true);
+ writerAppender.reset(writerFrame, true);
}
- outFrameAppender.reset(outFrame, true);
+
+ outAppender.reset(outFrame, true);
}
private void setNextTopTuple(int runIndex, int[] tupleIndices, RunFileReader[] runCursors,
@@ -525,22 +564,20 @@
* If all tuples in the targeting frame have been
* checked.
*/
- int frameOffset = runIndex * runFrameLimit;
tupleIndices[runIndex] = 0;
- currentFrameIndexInRun[runIndex] = frameOffset;
+ currentFrameIndexInRun[runIndex] = runStart;
/**
* read in batch
*/
currentRunFrames[runIndex] = 0;
- for (int j = 0; j < runFrameLimit; j++, frameOffset++) {
- ByteBuffer buffer = tupleAccessors[frameOffset].getBuffer();
- if (runCursors[runIndex].nextFrame(buffer)) {
- tupleAccessors[frameOffset].reset(buffer);
- if (tupleAccessors[frameOffset].getTupleCount() > 0) {
- existNext = true;
- } else {
- throw new IllegalStateException("illegal: empty run file");
- }
+ for (int j = 0; j < runFrameLimit; j++) {
+ int frameIndex = currentFrameIndexInRun[runIndex]
+ + j;
+ if (runCursors[runIndex].nextFrame(inFrames
+ .get(frameIndex))) {
+ tupleAccessors[frameIndex].reset(inFrames
+ .get(frameIndex));
+ existNext = true;
currentRunFrames[runIndex]++;
} else {
break;
@@ -571,6 +608,10 @@
if (runCursors[index] != null) {
runCursors[index].close();
runCursors[index] = null;
+ int frameOffset = index * runFrameLimit;
+ for (int j = 0; j < runFrameLimit; j++) {
+ tupleAccessor[frameOffset + j] = null;
+ }
}
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/FrameToolsForGroupers.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/FrameToolsForGroupers.java
new file mode 100644
index 0000000..2346160
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/FrameToolsForGroupers.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.FrameHelper;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+
+/**
+ *
+ */
+public class FrameToolsForGroupers {
+
+ public static void writeFields(byte[] buf, int offset, int length, ArrayTupleBuilder tupleBuilder)
+ throws HyracksDataException {
+ writeFields(buf, offset, length, tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0,
+ tupleBuilder.getSize());
+ }
+
+ public static void writeFields(byte[] buf, int offset, int length, int[] fieldsOffset, byte[] data, int dataOffset,
+ int dataLength) throws HyracksDataException {
+ if (dataLength + 4 * fieldsOffset.length > length) {
+ throw new HyracksDataException("Out of buffer bound: try to write too much data (" + dataLength
+ + ") to the given bound (" + length + ").");
+ }
+
+ ByteBuffer buffer = ByteBuffer.wrap(buf, offset, length);
+ for (int i = 0; i < fieldsOffset.length; i++) {
+ buffer.putInt(fieldsOffset[i]);
+ }
+ buffer.put(data, dataOffset, dataLength);
+ }
+
+ public static void updateFrameMetaForNewTuple(ByteBuffer buffer, int addedTupleLength) throws HyracksDataException {
+ int currentTupleCount = buffer.getInt(FrameHelper.getTupleCountOffset(buffer.capacity()));
+ int currentTupleEndOffset = buffer.getInt(FrameHelper.getTupleCountOffset(buffer.capacity()) - 4
+ * currentTupleCount);
+ int newTupleEndOffset = currentTupleEndOffset + addedTupleLength;
+
+ // update tuple end offset
+ buffer.putInt(FrameHelper.getTupleCountOffset(buffer.capacity()) - 4 * (currentTupleCount + 1),
+ newTupleEndOffset);
+ // Update the tuple count
+ buffer.putInt(FrameHelper.getTupleCountOffset(buffer.capacity()), currentTupleCount + 1);
+ }
+
+ public static void updateFrameMetaForNewTuple(ByteBuffer buffer, int addedTupleLength, boolean isReset)
+ throws HyracksDataException {
+ int currentTupleCount;
+ int currentTupleEndOffset;
+ if (isReset) {
+ currentTupleCount = 0;
+ currentTupleEndOffset = 0;
+ } else {
+ currentTupleCount = buffer.getInt(FrameHelper.getTupleCountOffset(buffer.capacity()));
+ currentTupleEndOffset = buffer.getInt(FrameHelper.getTupleCountOffset(buffer.capacity()) - 4
+ * currentTupleCount);
+ }
+ int newTupleEndOffset = currentTupleEndOffset + addedTupleLength;
+
+ // update tuple end offset
+ buffer.putInt(FrameHelper.getTupleCountOffset(buffer.capacity()) - 4 * (currentTupleCount + 1),
+ newTupleEndOffset);
+ // Update the tuple count
+ buffer.putInt(FrameHelper.getTupleCountOffset(buffer.capacity()), currentTupleCount + 1);
+ }
+
+ public static boolean isFrameOverflowing(ByteBuffer buffer, int length, boolean isReset)
+ throws HyracksDataException {
+
+ int currentTupleCount = buffer.getInt(FrameHelper.getTupleCountOffset(buffer.capacity()));
+ if (currentTupleCount == 0 || isReset) {
+ return length + 4 + 4 > buffer.capacity();
+ }
+ int currentTupleEndOffset = buffer.getInt(FrameHelper.getTupleCountOffset(buffer.capacity()) - 4
+ * currentTupleCount);
+ return currentTupleEndOffset + length + 4 + (currentTupleCount + 1) * 4 > buffer.capacity();
+ }
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/GroupingHashTable.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/GroupingHashTable.java
index 5733c11..43bb0ae 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/GroupingHashTable.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/GroupingHashTable.java
@@ -28,13 +28,15 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
class GroupingHashTable {
/**
- * The pointers in the link store 3 int values for each entry in the hashtable: (bufferIdx, tIndex, accumulatorIdx).
+ * The pointers in the link store 3 int values for each entry in the
+ * hashtable: (bufferIdx, tIndex, accumulatorIdx).
*
* @author vinayakb
*/
@@ -59,55 +61,82 @@
}
}
- private static final int INIT_ACCUMULATORS_SIZE = 8;
+ private static final int INIT_AGG_STATE_SIZE = 8;
private final IHyracksTaskContext ctx;
- private final FrameTupleAppender appender;
+
private final List<ByteBuffer> buffers;
private final Link[] table;
- private IAccumulatingAggregator[] accumulators;
+ /**
+ * Aggregate states: a list of states for all groups maintained in the main
+ * memory.
+ */
+ private AggregateState[] aggregateStates;
private int accumulatorSize;
private int lastBIndex;
- private final int[] fields;
private final int[] storedKeys;
+ private final int[] keys;
private final IBinaryComparator[] comparators;
private final FrameTuplePairComparator ftpc;
private final ITuplePartitionComputer tpc;
- private final IAccumulatingAggregatorFactory aggregatorFactory;
- private final RecordDescriptor inRecordDescriptor;
- private final RecordDescriptor outRecordDescriptor;
+ private final IAggregatorDescriptor aggregator;
+
+ private final FrameTupleAppender appender;
private final FrameTupleAccessor storedKeysAccessor;
+ private final ArrayTupleBuilder stateTupleBuilder, outputTupleBuilder;
+
GroupingHashTable(IHyracksTaskContext ctx, int[] fields, IBinaryComparatorFactory[] comparatorFactories,
- ITuplePartitionComputerFactory tpcf, IAccumulatingAggregatorFactory aggregatorFactory,
- RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor, int tableSize) {
+ ITuplePartitionComputerFactory tpcf, IAggregatorDescriptorFactory aggregatorFactory,
+ RecordDescriptor inRecordDescriptor, RecordDescriptor outRecordDescriptor, int tableSize)
+ throws HyracksDataException {
this.ctx = ctx;
- appender = new FrameTupleAppender(ctx.getFrameSize());
+
buffers = new ArrayList<ByteBuffer>();
table = new Link[tableSize];
- accumulators = new IAccumulatingAggregator[INIT_ACCUMULATORS_SIZE];
- accumulatorSize = 0;
- this.fields = fields;
+
+ keys = fields;
storedKeys = new int[fields.length];
+ @SuppressWarnings("rawtypes")
ISerializerDeserializer[] storedKeySerDeser = new ISerializerDeserializer[fields.length];
for (int i = 0; i < fields.length; ++i) {
storedKeys[i] = i;
storedKeySerDeser[i] = inRecordDescriptor.getFields()[fields[i]];
}
+
comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
}
ftpc = new FrameTuplePairComparator(fields, storedKeys, comparators);
tpc = tpcf.createPartitioner();
- this.aggregatorFactory = aggregatorFactory;
- this.inRecordDescriptor = inRecordDescriptor;
- this.outRecordDescriptor = outRecordDescriptor;
+
+ int[] keyFieldsInPartialResults = new int[fields.length];
+ for (int i = 0; i < keyFieldsInPartialResults.length; i++) {
+ keyFieldsInPartialResults[i] = i;
+ }
+
+ this.aggregator = aggregatorFactory.createAggregator(ctx, inRecordDescriptor, outRecordDescriptor, fields,
+ keyFieldsInPartialResults);
+
+ this.aggregateStates = new AggregateState[INIT_AGG_STATE_SIZE];
+ accumulatorSize = 0;
+
RecordDescriptor storedKeysRecordDescriptor = new RecordDescriptor(storedKeySerDeser);
storedKeysAccessor = new FrameTupleAccessor(ctx.getFrameSize(), storedKeysRecordDescriptor);
lastBIndex = -1;
+
+ appender = new FrameTupleAppender(ctx.getFrameSize());
+
addNewBuffer();
+
+ if (fields.length < outRecordDescriptor.getFields().length) {
+ stateTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
+ } else {
+ stateTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length + 1);
+ }
+ outputTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
}
private void addNewBuffer() {
@@ -119,57 +148,64 @@
++lastBIndex;
}
- private void flushFrame(FrameTupleAppender appender, IFrameWriter writer) throws HyracksDataException {
- ByteBuffer frame = appender.getBuffer();
- frame.position(0);
- frame.limit(frame.capacity());
- writer.nextFrame(appender.getBuffer());
- appender.reset(appender.getBuffer(), true);
- }
-
- void insert(FrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
+ void insert(FrameTupleAccessor accessor, int tIndex) throws Exception {
int entry = tpc.partition(accessor, tIndex, table.length);
Link link = table[entry];
if (link == null) {
link = table[entry] = new Link();
}
- IAccumulatingAggregator aggregator = null;
+ int saIndex = -1;
for (int i = 0; i < link.size; i += 3) {
int sbIndex = link.pointers[i];
int stIndex = link.pointers[i + 1];
- int saIndex = link.pointers[i + 2];
storedKeysAccessor.reset(buffers.get(sbIndex));
int c = ftpc.compare(accessor, tIndex, storedKeysAccessor, stIndex);
if (c == 0) {
- aggregator = accumulators[saIndex];
+ saIndex = link.pointers[i + 2];
break;
}
}
- if (aggregator == null) {
+ if (saIndex < 0) {
// Did not find the key. Insert a new entry.
- if (!appender.appendProjection(accessor, tIndex, fields)) {
+ saIndex = accumulatorSize++;
+ // Add keys
+
+ // Add aggregation fields
+ AggregateState newState = aggregator.createAggregateStates();
+
+ stateTupleBuilder.reset();
+ for (int k = 0; k < keys.length; k++) {
+ stateTupleBuilder.addField(accessor, tIndex, keys[k]);
+ }
+
+ aggregator.init(stateTupleBuilder, accessor, tIndex, newState);
+
+ if (!appender.appendSkipEmptyField(stateTupleBuilder.getFieldEndOffsets(),
+ stateTupleBuilder.getByteArray(), 0, stateTupleBuilder.getSize())) {
addNewBuffer();
- if (!appender.appendProjection(accessor, tIndex, fields)) {
- throw new IllegalStateException();
+ if (!appender.appendSkipEmptyField(stateTupleBuilder.getFieldEndOffsets(),
+ stateTupleBuilder.getByteArray(), 0, stateTupleBuilder.getSize())) {
+ throw new HyracksDataException("Cannot init the aggregate state in a single frame.");
}
}
- int sbIndex = lastBIndex;
- int stIndex = appender.getTupleCount() - 1;
- if (accumulatorSize >= accumulators.length) {
- accumulators = Arrays.copyOf(accumulators, accumulators.length * 2);
+
+ if (accumulatorSize >= aggregateStates.length) {
+ aggregateStates = Arrays.copyOf(aggregateStates, aggregateStates.length * 2);
}
- int saIndex = accumulatorSize++;
- aggregator = accumulators[saIndex] = aggregatorFactory.createAggregator(ctx, inRecordDescriptor,
- outRecordDescriptor);
- aggregator.init(accessor, tIndex);
- link.add(sbIndex, stIndex, saIndex);
+
+ aggregateStates[saIndex] = newState;
+
+ link.add(lastBIndex, appender.getTupleCount() - 1, saIndex);
+
+ } else {
+ aggregator.aggregate(accessor, tIndex, null, 0, aggregateStates[saIndex]);
}
- aggregator.accumulate(accessor, tIndex);
}
void write(IFrameWriter writer) throws HyracksDataException {
ByteBuffer buffer = ctx.allocateFrame();
appender.reset(buffer, true);
+
for (int i = 0; i < table.length; ++i) {
Link link = table[i];
if (link != null) {
@@ -179,15 +215,37 @@
int aIndex = link.pointers[j + 2];
ByteBuffer keyBuffer = buffers.get(bIndex);
storedKeysAccessor.reset(keyBuffer);
- IAccumulatingAggregator aggregator = accumulators[aIndex];
- while (!aggregator.output(appender, storedKeysAccessor, tIndex, storedKeys)) {
- flushFrame(appender, writer);
+
+ // copy keys
+ outputTupleBuilder.reset();
+ for (int k = 0; k < storedKeys.length; k++) {
+ outputTupleBuilder.addField(storedKeysAccessor, tIndex, storedKeys[k]);
}
+
+ aggregator.outputFinalResult(outputTupleBuilder, storedKeysAccessor, tIndex,
+ aggregateStates[aIndex]);
+
+ if (!appender.appendSkipEmptyField(outputTupleBuilder.getFieldEndOffsets(),
+ outputTupleBuilder.getByteArray(), 0, outputTupleBuilder.getSize())) {
+ writer.nextFrame(buffer);
+ appender.reset(buffer, true);
+ if (!appender.appendSkipEmptyField(outputTupleBuilder.getFieldEndOffsets(),
+ outputTupleBuilder.getByteArray(), 0, outputTupleBuilder.getSize())) {
+ throw new HyracksDataException("Cannot write the aggregation output into a frame.");
+ }
+ }
+
}
}
}
if (appender.getTupleCount() != 0) {
- flushFrame(appender, writer);
+ writer.nextFrame(buffer);
+ }
+ }
+
+ void close() throws HyracksDataException {
+ for (AggregateState aState : aggregateStates) {
+ aState.close();
}
}
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashGroupOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashGroupOperatorDescriptor.java
index 625ee25..49443d1 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashGroupOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashGroupOperatorDescriptor.java
@@ -29,7 +29,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
@@ -39,7 +38,11 @@
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
+/**
+ *
+ */
public class HashGroupOperatorDescriptor extends AbstractOperatorDescriptor {
+
private static final int HASH_BUILD_ACTIVITY_ID = 0;
private static final int OUTPUT_ACTIVITY_ID = 1;
@@ -49,21 +52,30 @@
private final int[] keys;
private final ITuplePartitionComputerFactory tpcf;
private final IBinaryComparatorFactory[] comparatorFactories;
- private final IAccumulatingAggregatorFactory aggregatorFactory;
+
+ private final IAggregatorDescriptorFactory aggregatorFactory;
+
private final int tableSize;
public HashGroupOperatorDescriptor(JobSpecification spec, int[] keys, ITuplePartitionComputerFactory tpcf,
- IBinaryComparatorFactory[] comparatorFactories, IAccumulatingAggregatorFactory aggregatorFactory,
- RecordDescriptor recordDescriptor, int tableSize) {
+ IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory,
+ RecordDescriptor outRecordDescriptor, int tableSize) {
super(spec, 1, 1);
this.keys = keys;
this.tpcf = tpcf;
this.comparatorFactories = comparatorFactories;
this.aggregatorFactory = aggregatorFactory;
- recordDescriptors[0] = recordDescriptor;
+ recordDescriptors[0] = outRecordDescriptor;
this.tableSize = tableSize;
}
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor#contributeActivities
+ * (edu.uci.ics.hyracks.api.dataflow.IActivityGraphBuilder)
+ */
@Override
public void contributeActivities(IActivityGraphBuilder builder) {
HashBuildActivity ha = new HashBuildActivity(new ActivityId(odId, HASH_BUILD_ACTIVITY_ID));
@@ -106,7 +118,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
final FrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(),
recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
@@ -127,17 +139,23 @@
accessor.reset(buffer);
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; ++i) {
- state.table.insert(accessor, i);
+ try {
+ state.table.insert(accessor, i);
+ } catch (Exception e) {
+ System.out.println(e.toString());
+ throw new HyracksDataException(e);
+ }
}
}
@Override
public void close() throws HyracksDataException {
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@Override
public void fail() throws HyracksDataException {
+ throw new HyracksDataException("HashGroupOperator is failed.");
}
};
}
@@ -151,12 +169,12 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
return new AbstractUnaryOutputSourceOperatorNodePushable() {
@Override
public void initialize() throws HyracksDataException {
- HashBuildActivityState buildState = (HashBuildActivityState) env.getTaskState(new TaskId(
+ HashBuildActivityState buildState = (HashBuildActivityState) ctx.getTaskState(new TaskId(
new ActivityId(getOperatorId(), HASH_BUILD_ACTIVITY_ID), partition));
GroupingHashTable table = buildState.table;
writer.open();
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableGroupingTableFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableGroupingTableFactory.java
deleted file mode 100644
index af5d16b..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableGroupingTableFactory.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.group;
-
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputer;
-import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
-import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IAggregatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.structures.ISerializableTable;
-import edu.uci.ics.hyracks.dataflow.std.structures.SerializableHashTable;
-import edu.uci.ics.hyracks.dataflow.std.structures.TuplePointer;
-
-public class HashSpillableGroupingTableFactory implements ISpillableTableFactory {
- private static final long serialVersionUID = 1L;
- private final ITuplePartitionComputerFactory tpcf;
- private final int tableSize;
-
- public HashSpillableGroupingTableFactory(ITuplePartitionComputerFactory tpcf, int tableSize) {
- this.tpcf = tpcf;
- this.tableSize = tableSize;
- }
-
- @Override
- public ISpillableTable buildSpillableTable(final IHyracksTaskContext ctx, final int[] keyFields,
- final IBinaryComparatorFactory[] comparatorFactories,
- final INormalizedKeyComputerFactory firstKeyNormalizerFactory,
- final IAggregatorDescriptorFactory aggregateDescriptorFactory, final RecordDescriptor inRecordDescriptor,
- final RecordDescriptor outRecordDescriptor, final int framesLimit) throws HyracksDataException {
- final int[] storedKeys = new int[keyFields.length];
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] storedKeySerDeser = new ISerializerDeserializer[keyFields.length];
- for (int i = 0; i < keyFields.length; i++) {
- storedKeys[i] = i;
- storedKeySerDeser[i] = inRecordDescriptor.getFields()[keyFields[i]];
- }
-
- RecordDescriptor internalRecordDescriptor = outRecordDescriptor;
- final FrameTupleAccessor storedKeysAccessor1;
- final FrameTupleAccessor storedKeysAccessor2;
- if (keyFields.length >= outRecordDescriptor.getFields().length) {
- // for the case of zero-aggregations
- ISerializerDeserializer<?>[] fields = outRecordDescriptor.getFields();
- ITypeTrait[] types = outRecordDescriptor.getTypeTraits();
- ISerializerDeserializer<?>[] newFields = new ISerializerDeserializer[fields.length + 1];
- for (int i = 0; i < fields.length; i++)
- newFields[i] = fields[i];
- ITypeTrait[] newTypes = null;
- if (types != null) {
- newTypes = new ITypeTrait[types.length + 1];
- for (int i = 0; i < types.length; i++)
- newTypes[i] = types[i];
- }
- internalRecordDescriptor = new RecordDescriptor(newFields, newTypes);
- }
- storedKeysAccessor1 = new FrameTupleAccessor(ctx.getFrameSize(), internalRecordDescriptor);
- storedKeysAccessor2 = new FrameTupleAccessor(ctx.getFrameSize(), internalRecordDescriptor);
-
- final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
- for (int i = 0; i < comparatorFactories.length; ++i) {
- comparators[i] = comparatorFactories[i].createBinaryComparator();
- }
-
- final FrameTuplePairComparator ftpcPartial = new FrameTuplePairComparator(keyFields, storedKeys, comparators);
- final FrameTuplePairComparator ftpcTuple = new FrameTuplePairComparator(storedKeys, storedKeys, comparators);
- final FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- final ITuplePartitionComputer tpc = tpcf.createPartitioner();
- final ByteBuffer outFrame = ctx.allocateFrame();
-
- final ArrayTupleBuilder internalTupleBuilder;
- if (keyFields.length < outRecordDescriptor.getFields().length)
- internalTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
- else
- internalTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length + 1);
- final ArrayTupleBuilder outputTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
- final INormalizedKeyComputer nkc = firstKeyNormalizerFactory == null ? null : firstKeyNormalizerFactory
- .createNormalizedKeyComputer();
-
- return new ISpillableTable() {
- private int dataFrameCount;
- private final ISerializableTable table = new SerializableHashTable(tableSize, ctx);;
- private final TuplePointer storedTuplePointer = new TuplePointer();
- private final List<ByteBuffer> frames = new ArrayList<ByteBuffer>();
- private int groupSize = 0;
- private IAggregatorDescriptor aggregator = aggregateDescriptorFactory.createAggregator(ctx,
- inRecordDescriptor, outRecordDescriptor, keyFields);
-
- /**
- * A tuple is "pointed" to by 3 entries in the tPointers array. [0]
- * = Frame index in the "Frames" list, [1] = Tuple index in the
- * frame, [2] = Poor man's normalized key for the tuple.
- */
- private int[] tPointers;
-
- @Override
- public void reset() {
- groupSize = 0;
- dataFrameCount = -1;
- tPointers = null;
- table.reset();
- aggregator.close();
- }
-
- @Override
- public boolean insert(FrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
- if (dataFrameCount < 0)
- nextAvailableFrame();
- int entry = tpc.partition(accessor, tIndex, tableSize);
- boolean foundGroup = false;
- int offset = 0;
- do {
- table.getTuplePointer(entry, offset++, storedTuplePointer);
- if (storedTuplePointer.frameIndex < 0)
- break;
- storedKeysAccessor1.reset(frames.get(storedTuplePointer.frameIndex));
- int c = ftpcPartial.compare(accessor, tIndex, storedKeysAccessor1, storedTuplePointer.tupleIndex);
- if (c == 0) {
- foundGroup = true;
- break;
- }
- } while (true);
-
- if (!foundGroup) {
- /**
- * If no matching group is found, create a new aggregator
- * Create a tuple for the new group
- */
- internalTupleBuilder.reset();
- for (int i = 0; i < keyFields.length; i++) {
- internalTupleBuilder.addField(accessor, tIndex, keyFields[i]);
- }
- aggregator.init(accessor, tIndex, internalTupleBuilder);
- if (!appender.append(internalTupleBuilder.getFieldEndOffsets(),
- internalTupleBuilder.getByteArray(), 0, internalTupleBuilder.getSize())) {
- if (!nextAvailableFrame()) {
- return false;
- } else {
- if (!appender.append(internalTupleBuilder.getFieldEndOffsets(),
- internalTupleBuilder.getByteArray(), 0, internalTupleBuilder.getSize())) {
- throw new IllegalStateException("Failed to init an aggregator");
- }
- }
- }
-
- storedTuplePointer.frameIndex = dataFrameCount;
- storedTuplePointer.tupleIndex = appender.getTupleCount() - 1;
- table.insert(entry, storedTuplePointer);
- groupSize++;
- } else {
- // If there is a matching found, do aggregation directly
- int tupleOffset = storedKeysAccessor1.getTupleStartOffset(storedTuplePointer.tupleIndex);
- int aggFieldOffset = storedKeysAccessor1.getFieldStartOffset(storedTuplePointer.tupleIndex,
- keyFields.length);
- int tupleLength = storedKeysAccessor1.getFieldLength(storedTuplePointer.tupleIndex,
- keyFields.length);
- aggregator.aggregate(accessor, tIndex, storedKeysAccessor1.getBuffer().array(), tupleOffset
- + storedKeysAccessor1.getFieldSlotsLength() + aggFieldOffset, tupleLength);
- }
- return true;
- }
-
- @Override
- public List<ByteBuffer> getFrames() {
- return frames;
- }
-
- @Override
- public int getFrameCount() {
- return dataFrameCount;
- }
-
- @Override
- public void flushFrames(IFrameWriter writer, boolean isPartial) throws HyracksDataException {
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- writer.open();
- appender.reset(outFrame, true);
- if (tPointers == null) {
- // Not sorted
- for (int i = 0; i < tableSize; ++i) {
- int entry = i;
- int offset = 0;
- do {
- table.getTuplePointer(entry, offset++, storedTuplePointer);
- if (storedTuplePointer.frameIndex < 0)
- break;
- int bIndex = storedTuplePointer.frameIndex;
- int tIndex = storedTuplePointer.tupleIndex;
- storedKeysAccessor1.reset(frames.get(bIndex));
- // Reset the tuple for the partial result
- outputTupleBuilder.reset();
- for (int k = 0; k < keyFields.length; k++) {
- outputTupleBuilder.addField(storedKeysAccessor1, tIndex, k);
- }
- if (isPartial)
- aggregator.outputPartialResult(storedKeysAccessor1, tIndex, outputTupleBuilder);
- else
- aggregator.outputResult(storedKeysAccessor1, tIndex, outputTupleBuilder);
- while (!appender.append(outputTupleBuilder.getFieldEndOffsets(),
- outputTupleBuilder.getByteArray(), 0, outputTupleBuilder.getSize())) {
- FrameUtils.flushFrame(outFrame, writer);
- appender.reset(outFrame, true);
- }
- } while (true);
- }
- if (appender.getTupleCount() != 0) {
- FrameUtils.flushFrame(outFrame, writer);
- }
- aggregator.close();
- return;
- }
- int n = tPointers.length / 3;
- for (int ptr = 0; ptr < n; ptr++) {
- int tableIndex = tPointers[ptr * 3];
- int rowIndex = tPointers[ptr * 3 + 1];
- table.getTuplePointer(tableIndex, rowIndex, storedTuplePointer);
- int frameIndex = storedTuplePointer.frameIndex;
- int tupleIndex = storedTuplePointer.tupleIndex;
- // Get the frame containing the value
- ByteBuffer buffer = frames.get(frameIndex);
- storedKeysAccessor1.reset(buffer);
-
- outputTupleBuilder.reset();
- for (int k = 0; k < keyFields.length; k++) {
- outputTupleBuilder.addField(storedKeysAccessor1, tupleIndex, k);
- }
- if (isPartial)
- aggregator.outputPartialResult(storedKeysAccessor1, tupleIndex, outputTupleBuilder);
- else
- aggregator.outputResult(storedKeysAccessor1, tupleIndex, outputTupleBuilder);
- if (!appender.append(outputTupleBuilder.getFieldEndOffsets(), outputTupleBuilder.getByteArray(), 0,
- outputTupleBuilder.getSize())) {
- FrameUtils.flushFrame(outFrame, writer);
- appender.reset(outFrame, true);
- if (!appender.append(outputTupleBuilder.getFieldEndOffsets(),
- outputTupleBuilder.getByteArray(), 0, outputTupleBuilder.getSize())) {
- throw new IllegalStateException();
- }
- }
- }
- if (appender.getTupleCount() > 0) {
- FrameUtils.flushFrame(outFrame, writer);
- }
- aggregator.close();
- }
-
- /**
- * Set the working frame to the next available frame in the frame
- * list. There are two cases:<br>
- * 1) If the next frame is not initialized, allocate a new frame. 2)
- * When frames are already created, they are recycled.
- *
- * @return Whether a new frame is added successfully.
- */
- private boolean nextAvailableFrame() {
- // Return false if the number of frames is equal to the limit.
- if (dataFrameCount + 1 >= framesLimit)
- return false;
-
- if (frames.size() < framesLimit) {
- // Insert a new frame
- ByteBuffer frame = ctx.allocateFrame();
- frame.position(0);
- frame.limit(frame.capacity());
- frames.add(frame);
- appender.reset(frame, true);
- dataFrameCount = frames.size() - 1;
- } else {
- // Reuse an old frame
- dataFrameCount++;
- ByteBuffer frame = frames.get(dataFrameCount);
- frame.position(0);
- frame.limit(frame.capacity());
- appender.reset(frame, true);
- }
- return true;
- }
-
- @Override
- public void sortFrames() {
- int sfIdx = storedKeys[0];
- int totalTCount = table.getTupleCount();
- tPointers = new int[totalTCount * 3];
- int ptr = 0;
-
- for (int i = 0; i < tableSize; i++) {
- int entry = i;
- int offset = 0;
- do {
- table.getTuplePointer(entry, offset, storedTuplePointer);
- if (storedTuplePointer.frameIndex < 0)
- break;
- tPointers[ptr * 3] = entry;
- tPointers[ptr * 3 + 1] = offset;
- table.getTuplePointer(entry, offset, storedTuplePointer);
- int fIndex = storedTuplePointer.frameIndex;
- int tIndex = storedTuplePointer.tupleIndex;
- storedKeysAccessor1.reset(frames.get(fIndex));
- int tStart = storedKeysAccessor1.getTupleStartOffset(tIndex);
- int f0StartRel = storedKeysAccessor1.getFieldStartOffset(tIndex, sfIdx);
- int f0EndRel = storedKeysAccessor1.getFieldEndOffset(tIndex, sfIdx);
- int f0Start = f0StartRel + tStart + storedKeysAccessor1.getFieldSlotsLength();
- tPointers[ptr * 3 + 2] = nkc == null ? 0 : nkc.normalize(storedKeysAccessor1.getBuffer()
- .array(), f0Start, f0EndRel - f0StartRel);
- ptr++;
- offset++;
- } while (true);
- }
- /**
- * Sort using quick sort
- */
- if (tPointers.length > 0) {
- sort(tPointers, 0, totalTCount);
- }
- }
-
- private void sort(int[] tPointers, int offset, int length) {
- int m = offset + (length >> 1);
- int mTable = tPointers[m * 3];
- int mRow = tPointers[m * 3 + 1];
- int mNormKey = tPointers[m * 3 + 2];
-
- table.getTuplePointer(mTable, mRow, storedTuplePointer);
- int mFrame = storedTuplePointer.frameIndex;
- int mTuple = storedTuplePointer.tupleIndex;
- storedKeysAccessor1.reset(frames.get(mFrame));
-
- int a = offset;
- int b = a;
- int c = offset + length - 1;
- int d = c;
- while (true) {
- while (b <= c) {
- int bTable = tPointers[b * 3];
- int bRow = tPointers[b * 3 + 1];
- int bNormKey = tPointers[b * 3 + 2];
- int cmp = 0;
- if (bNormKey != mNormKey) {
- cmp = ((((long) bNormKey) & 0xffffffffL) < (((long) mNormKey) & 0xffffffffL)) ? -1 : 1;
- } else {
- table.getTuplePointer(bTable, bRow, storedTuplePointer);
- int bFrame = storedTuplePointer.frameIndex;
- int bTuple = storedTuplePointer.tupleIndex;
- storedKeysAccessor2.reset(frames.get(bFrame));
- cmp = ftpcTuple.compare(storedKeysAccessor2, bTuple, storedKeysAccessor1, mTuple);
- }
- if (cmp > 0) {
- break;
- }
- if (cmp == 0) {
- swap(tPointers, a++, b);
- }
- ++b;
- }
- while (c >= b) {
- int cTable = tPointers[c * 3];
- int cRow = tPointers[c * 3 + 1];
- int cNormKey = tPointers[c * 3 + 2];
- int cmp = 0;
- if (cNormKey != mNormKey) {
- cmp = ((((long) cNormKey) & 0xffffffffL) < (((long) mNormKey) & 0xffffffffL)) ? -1 : 1;
- } else {
- table.getTuplePointer(cTable, cRow, storedTuplePointer);
- int cFrame = storedTuplePointer.frameIndex;
- int cTuple = storedTuplePointer.tupleIndex;
- storedKeysAccessor2.reset(frames.get(cFrame));
- cmp = ftpcTuple.compare(storedKeysAccessor2, cTuple, storedKeysAccessor1, mTuple);
- }
- if (cmp < 0) {
- break;
- }
- if (cmp == 0) {
- swap(tPointers, c, d--);
- }
- --c;
- }
- if (b > c)
- break;
- swap(tPointers, b++, c--);
- }
-
- int s;
- int n = offset + length;
- s = Math.min(a - offset, b - a);
- vecswap(tPointers, offset, b - s, s);
- s = Math.min(d - c, n - d - 1);
- vecswap(tPointers, b, n - s, s);
-
- if ((s = b - a) > 1) {
- sort(tPointers, offset, s);
- }
- if ((s = d - c) > 1) {
- sort(tPointers, n - s, s);
- }
- }
-
- private void swap(int x[], int a, int b) {
- for (int i = 0; i < 3; ++i) {
- int t = x[a * 3 + i];
- x[a * 3 + i] = x[b * 3 + i];
- x[b * 3 + i] = t;
- }
- }
-
- private void vecswap(int x[], int a, int b, int n) {
- for (int i = 0; i < n; i++, a++, b++) {
- swap(x, a, b);
- }
- }
-
- @Override
- public void close() {
- groupSize = 0;
- dataFrameCount = -1;
- tPointers = null;
- table.close();
- frames.clear();
- }
- };
- }
-}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableTableFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableTableFactory.java
new file mode 100644
index 0000000..b0e092d
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/HashSpillableTableFactory.java
@@ -0,0 +1,509 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import edu.uci.ics.hyracks.api.comm.IFrameWriter;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.std.structures.ISerializableTable;
+import edu.uci.ics.hyracks.dataflow.std.structures.SerializableHashTable;
+import edu.uci.ics.hyracks.dataflow.std.structures.TuplePointer;
+
+/**
+ *
+ */
+public class HashSpillableTableFactory implements ISpillableTableFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final ITuplePartitionComputerFactory tpcf;
+ private final int tableSize;
+
+ public HashSpillableTableFactory(ITuplePartitionComputerFactory tpcf, int tableSize) {
+ this.tpcf = tpcf;
+ this.tableSize = tableSize;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.dataflow.std.aggregations.ISpillableTableFactory#
+ * buildSpillableTable(edu.uci.ics.hyracks.api.context.IHyracksTaskContext,
+ * int[], edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory[],
+ * edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory,
+ * edu.
+ * uci.ics.hyracks.dataflow.std.aggregations.IFieldAggregateDescriptorFactory
+ * [], edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor, int)
+ */
+ @Override
+ public ISpillableTable buildSpillableTable(final IHyracksTaskContext ctx, final int[] keyFields,
+ IBinaryComparatorFactory[] comparatorFactories, INormalizedKeyComputerFactory firstKeyNormalizerFactory,
+ IAggregatorDescriptorFactory aggregateFactory, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor, final int framesLimit) throws HyracksDataException {
+ final int[] storedKeys = new int[keyFields.length];
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] storedKeySerDeser = new ISerializerDeserializer[keyFields.length];
+ for (int i = 0; i < keyFields.length; i++) {
+ storedKeys[i] = i;
+ storedKeySerDeser[i] = inRecordDescriptor.getFields()[keyFields[i]];
+ }
+
+ RecordDescriptor internalRecordDescriptor = outRecordDescriptor;
+ final FrameTupleAccessor storedKeysAccessor1;
+ final FrameTupleAccessor storedKeysAccessor2;
+ if (keyFields.length >= outRecordDescriptor.getFields().length) {
+ // for the case of zero-aggregations
+ ISerializerDeserializer<?>[] fields = outRecordDescriptor.getFields();
+ ITypeTraits[] types = outRecordDescriptor.getTypeTraits();
+ ISerializerDeserializer<?>[] newFields = new ISerializerDeserializer[fields.length + 1];
+ for (int i = 0; i < fields.length; i++)
+ newFields[i] = fields[i];
+ ITypeTraits[] newTypes = null;
+ if (types != null) {
+ newTypes = new ITypeTraits[types.length + 1];
+ for (int i = 0; i < types.length; i++)
+ newTypes[i] = types[i];
+ }
+ internalRecordDescriptor = new RecordDescriptor(newFields, newTypes);
+ }
+ storedKeysAccessor1 = new FrameTupleAccessor(ctx.getFrameSize(), internalRecordDescriptor);
+ storedKeysAccessor2 = new FrameTupleAccessor(ctx.getFrameSize(), internalRecordDescriptor);
+
+ final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
+ for (int i = 0; i < comparatorFactories.length; ++i) {
+ comparators[i] = comparatorFactories[i].createBinaryComparator();
+ }
+
+ final FrameTuplePairComparator ftpcPartial = new FrameTuplePairComparator(keyFields, storedKeys, comparators);
+
+ final FrameTuplePairComparator ftpcTuple = new FrameTuplePairComparator(storedKeys, storedKeys, comparators);
+
+ final ITuplePartitionComputer tpc = tpcf.createPartitioner();
+
+ final INormalizedKeyComputer nkc = firstKeyNormalizerFactory == null ? null : firstKeyNormalizerFactory
+ .createNormalizedKeyComputer();
+
+ int[] keyFieldsInPartialResults = new int[keyFields.length];
+ for (int i = 0; i < keyFieldsInPartialResults.length; i++) {
+ keyFieldsInPartialResults[i] = i;
+ }
+
+ final IAggregatorDescriptor aggregator = aggregateFactory.createAggregator(ctx, inRecordDescriptor,
+ outRecordDescriptor, keyFields, keyFieldsInPartialResults);
+
+ final AggregateState aggregateState = aggregator.createAggregateStates();
+
+ final ArrayTupleBuilder stateTupleBuilder;
+ if (keyFields.length < outRecordDescriptor.getFields().length) {
+ stateTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
+ } else {
+ stateTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length + 1);
+ }
+
+ final ArrayTupleBuilder outputTupleBuilder = new ArrayTupleBuilder(outRecordDescriptor.getFields().length);
+
+ return new ISpillableTable() {
+
+ private int lastBufIndex;
+
+ private ByteBuffer outputFrame;
+ private FrameTupleAppender outputAppender;
+
+ private FrameTupleAppender stateAppender = new FrameTupleAppender(ctx.getFrameSize());
+
+ private final ISerializableTable table = new SerializableHashTable(tableSize, ctx);
+ private final TuplePointer storedTuplePointer = new TuplePointer();
+ private final List<ByteBuffer> frames = new ArrayList<ByteBuffer>();
+
+ /**
+ * A tuple is "pointed" to by 3 entries in the tPointers array. [0]
+ * = Frame index in the "Frames" list, [1] = Tuple index in the
+ * frame, [2] = Poor man's normalized key for the tuple.
+ */
+ private int[] tPointers;
+
+ @Override
+ public void sortFrames() {
+ int sfIdx = storedKeys[0];
+ int totalTCount = table.getTupleCount();
+ tPointers = new int[totalTCount * 3];
+ int ptr = 0;
+
+ for (int i = 0; i < tableSize; i++) {
+ int entry = i;
+ int offset = 0;
+ do {
+ table.getTuplePointer(entry, offset, storedTuplePointer);
+ if (storedTuplePointer.frameIndex < 0)
+ break;
+ tPointers[ptr * 3] = entry;
+ tPointers[ptr * 3 + 1] = offset;
+ table.getTuplePointer(entry, offset, storedTuplePointer);
+ int fIndex = storedTuplePointer.frameIndex;
+ int tIndex = storedTuplePointer.tupleIndex;
+ storedKeysAccessor1.reset(frames.get(fIndex));
+ int tStart = storedKeysAccessor1.getTupleStartOffset(tIndex);
+ int f0StartRel = storedKeysAccessor1.getFieldStartOffset(tIndex, sfIdx);
+ int f0EndRel = storedKeysAccessor1.getFieldEndOffset(tIndex, sfIdx);
+ int f0Start = f0StartRel + tStart + storedKeysAccessor1.getFieldSlotsLength();
+ tPointers[ptr * 3 + 2] = nkc == null ? 0 : nkc.normalize(storedKeysAccessor1.getBuffer()
+ .array(), f0Start, f0EndRel - f0StartRel);
+ ptr++;
+ offset++;
+ } while (true);
+ }
+ /**
+ * Sort using quick sort
+ */
+ if (tPointers.length > 0) {
+ sort(tPointers, 0, totalTCount);
+ }
+ }
+
+ @Override
+ public void reset() {
+ lastBufIndex = -1;
+ tPointers = null;
+ table.reset();
+ aggregator.reset();
+ }
+
+ @Override
+ public boolean insert(FrameTupleAccessor accessor, int tIndex) throws HyracksDataException {
+ if (lastBufIndex < 0)
+ nextAvailableFrame();
+ int entry = tpc.partition(accessor, tIndex, tableSize);
+ boolean foundGroup = false;
+ int offset = 0;
+ do {
+ table.getTuplePointer(entry, offset++, storedTuplePointer);
+ if (storedTuplePointer.frameIndex < 0)
+ break;
+ storedKeysAccessor1.reset(frames.get(storedTuplePointer.frameIndex));
+ int c = ftpcPartial.compare(accessor, tIndex, storedKeysAccessor1, storedTuplePointer.tupleIndex);
+ if (c == 0) {
+ foundGroup = true;
+ break;
+ }
+ } while (true);
+
+ if (!foundGroup) {
+
+ stateTupleBuilder.reset();
+
+ for (int k = 0; k < keyFields.length; k++) {
+ stateTupleBuilder.addField(accessor, tIndex, keyFields[k]);
+ }
+
+ aggregator.init(stateTupleBuilder, accessor, tIndex, aggregateState);
+ if (!stateAppender.appendSkipEmptyField(stateTupleBuilder.getFieldEndOffsets(),
+ stateTupleBuilder.getByteArray(), 0, stateTupleBuilder.getSize())) {
+ if (!nextAvailableFrame()) {
+ return false;
+ }
+ if (!stateAppender.appendSkipEmptyField(stateTupleBuilder.getFieldEndOffsets(),
+ stateTupleBuilder.getByteArray(), 0, stateTupleBuilder.getSize())) {
+ throw new HyracksDataException("Cannot init external aggregate state in a frame.");
+ }
+ }
+
+ storedTuplePointer.frameIndex = lastBufIndex;
+ storedTuplePointer.tupleIndex = stateAppender.getTupleCount() - 1;
+ table.insert(entry, storedTuplePointer);
+ } else {
+
+ aggregator.aggregate(accessor, tIndex, storedKeysAccessor1, storedTuplePointer.tupleIndex,
+ aggregateState);
+
+ }
+ return true;
+ }
+
+ @Override
+ public List<ByteBuffer> getFrames() {
+ return frames;
+ }
+
+ @Override
+ public int getFrameCount() {
+ return lastBufIndex;
+ }
+
+ @Override
+ public void flushFrames(IFrameWriter writer, boolean isPartial) throws HyracksDataException {
+ if (outputFrame == null) {
+ outputFrame = ctx.allocateFrame();
+ }
+
+ if (outputAppender == null) {
+ outputAppender = new FrameTupleAppender(outputFrame.capacity());
+ }
+
+ outputAppender.reset(outputFrame, true);
+
+ writer.open();
+
+ if (tPointers == null) {
+ // Not sorted
+ for (int i = 0; i < tableSize; ++i) {
+ int entry = i;
+ int offset = 0;
+ do {
+ table.getTuplePointer(entry, offset++, storedTuplePointer);
+ if (storedTuplePointer.frameIndex < 0)
+ break;
+ int bIndex = storedTuplePointer.frameIndex;
+ int tIndex = storedTuplePointer.tupleIndex;
+
+ storedKeysAccessor1.reset(frames.get(bIndex));
+
+ outputTupleBuilder.reset();
+ for (int k = 0; k < storedKeys.length; k++) {
+ outputTupleBuilder.addField(storedKeysAccessor1, tIndex, storedKeys[k]);
+ }
+
+ if (isPartial) {
+
+ aggregator.outputPartialResult(outputTupleBuilder, storedKeysAccessor1, tIndex,
+ aggregateState);
+
+ } else {
+
+ aggregator.outputFinalResult(outputTupleBuilder, storedKeysAccessor1, tIndex,
+ aggregateState);
+ }
+
+ if (!outputAppender.appendSkipEmptyField(outputTupleBuilder.getFieldEndOffsets(),
+ outputTupleBuilder.getByteArray(), 0, outputTupleBuilder.getSize())) {
+ FrameUtils.flushFrame(outputFrame, writer);
+ outputAppender.reset(outputFrame, true);
+ if (!outputAppender.appendSkipEmptyField(outputTupleBuilder.getFieldEndOffsets(),
+ outputTupleBuilder.getByteArray(), 0, outputTupleBuilder.getSize())) {
+ throw new HyracksDataException(
+ "The output item is too large to be fit into a frame.");
+ }
+ }
+
+ } while (true);
+ }
+ if (outputAppender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(outputFrame, writer);
+ outputAppender.reset(outputFrame, true);
+ }
+ aggregator.close();
+ return;
+ }
+ int n = tPointers.length / 3;
+ for (int ptr = 0; ptr < n; ptr++) {
+ int tableIndex = tPointers[ptr * 3];
+ int rowIndex = tPointers[ptr * 3 + 1];
+ table.getTuplePointer(tableIndex, rowIndex, storedTuplePointer);
+ int frameIndex = storedTuplePointer.frameIndex;
+ int tupleIndex = storedTuplePointer.tupleIndex;
+ // Get the frame containing the value
+ ByteBuffer buffer = frames.get(frameIndex);
+ storedKeysAccessor1.reset(buffer);
+
+ outputTupleBuilder.reset();
+ for (int k = 0; k < storedKeys.length; k++) {
+ outputTupleBuilder.addField(storedKeysAccessor1, tupleIndex, storedKeys[k]);
+ }
+
+ if (isPartial) {
+
+ aggregator.outputPartialResult(outputTupleBuilder, storedKeysAccessor1, tupleIndex,
+ aggregateState);
+
+ } else {
+
+ aggregator.outputFinalResult(outputTupleBuilder, storedKeysAccessor1, tupleIndex,
+ aggregateState);
+ }
+
+ if (!outputAppender.appendSkipEmptyField(outputTupleBuilder.getFieldEndOffsets(),
+ outputTupleBuilder.getByteArray(), 0, outputTupleBuilder.getSize())) {
+ FrameUtils.flushFrame(outputFrame, writer);
+ outputAppender.reset(outputFrame, true);
+ if (!outputAppender.appendSkipEmptyField(outputTupleBuilder.getFieldEndOffsets(),
+ outputTupleBuilder.getByteArray(), 0, outputTupleBuilder.getSize())) {
+ throw new HyracksDataException("The output item is too large to be fit into a frame.");
+ }
+ }
+ }
+ if (outputAppender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(outputFrame, writer);
+ outputAppender.reset(outputFrame, true);
+ }
+ aggregator.close();
+ }
+
+ @Override
+ public void close() {
+ lastBufIndex = -1;
+ tPointers = null;
+ table.close();
+ frames.clear();
+ aggregateState.close();
+ }
+
+ /**
+ * Set the working frame to the next available frame in the frame
+ * list. There are two cases:<br>
+ * 1) If the next frame is not initialized, allocate a new frame. 2)
+ * When frames are already created, they are recycled.
+ *
+ * @return Whether a new frame is added successfully.
+ */
+ private boolean nextAvailableFrame() {
+ // Return false if the number of frames is equal to the limit.
+ if (lastBufIndex + 1 >= framesLimit)
+ return false;
+
+ if (frames.size() < framesLimit) {
+ // Insert a new frame
+ ByteBuffer frame = ctx.allocateFrame();
+ frame.position(0);
+ frame.limit(frame.capacity());
+ frames.add(frame);
+ stateAppender.reset(frame, true);
+ lastBufIndex = frames.size() - 1;
+ } else {
+ // Reuse an old frame
+ lastBufIndex++;
+ ByteBuffer frame = frames.get(lastBufIndex);
+ frame.position(0);
+ frame.limit(frame.capacity());
+ stateAppender.reset(frame, true);
+ }
+ return true;
+ }
+
+ private void sort(int[] tPointers, int offset, int length) {
+ int m = offset + (length >> 1);
+ int mTable = tPointers[m * 3];
+ int mRow = tPointers[m * 3 + 1];
+ int mNormKey = tPointers[m * 3 + 2];
+
+ table.getTuplePointer(mTable, mRow, storedTuplePointer);
+ int mFrame = storedTuplePointer.frameIndex;
+ int mTuple = storedTuplePointer.tupleIndex;
+ storedKeysAccessor1.reset(frames.get(mFrame));
+
+ int a = offset;
+ int b = a;
+ int c = offset + length - 1;
+ int d = c;
+ while (true) {
+ while (b <= c) {
+ int bTable = tPointers[b * 3];
+ int bRow = tPointers[b * 3 + 1];
+ int bNormKey = tPointers[b * 3 + 2];
+ int cmp = 0;
+ if (bNormKey != mNormKey) {
+ cmp = ((((long) bNormKey) & 0xffffffffL) < (((long) mNormKey) & 0xffffffffL)) ? -1 : 1;
+ } else {
+ table.getTuplePointer(bTable, bRow, storedTuplePointer);
+ int bFrame = storedTuplePointer.frameIndex;
+ int bTuple = storedTuplePointer.tupleIndex;
+ storedKeysAccessor2.reset(frames.get(bFrame));
+ cmp = ftpcTuple.compare(storedKeysAccessor2, bTuple, storedKeysAccessor1, mTuple);
+ }
+ if (cmp > 0) {
+ break;
+ }
+ if (cmp == 0) {
+ swap(tPointers, a++, b);
+ }
+ ++b;
+ }
+ while (c >= b) {
+ int cTable = tPointers[c * 3];
+ int cRow = tPointers[c * 3 + 1];
+ int cNormKey = tPointers[c * 3 + 2];
+ int cmp = 0;
+ if (cNormKey != mNormKey) {
+ cmp = ((((long) cNormKey) & 0xffffffffL) < (((long) mNormKey) & 0xffffffffL)) ? -1 : 1;
+ } else {
+ table.getTuplePointer(cTable, cRow, storedTuplePointer);
+ int cFrame = storedTuplePointer.frameIndex;
+ int cTuple = storedTuplePointer.tupleIndex;
+ storedKeysAccessor2.reset(frames.get(cFrame));
+ cmp = ftpcTuple.compare(storedKeysAccessor2, cTuple, storedKeysAccessor1, mTuple);
+ }
+ if (cmp < 0) {
+ break;
+ }
+ if (cmp == 0) {
+ swap(tPointers, c, d--);
+ }
+ --c;
+ }
+ if (b > c)
+ break;
+ swap(tPointers, b++, c--);
+ }
+
+ int s;
+ int n = offset + length;
+ s = Math.min(a - offset, b - a);
+ vecswap(tPointers, offset, b - s, s);
+ s = Math.min(d - c, n - d - 1);
+ vecswap(tPointers, b, n - s, s);
+
+ if ((s = b - a) > 1) {
+ sort(tPointers, offset, s);
+ }
+ if ((s = d - c) > 1) {
+ sort(tPointers, n - s, s);
+ }
+ }
+
+ private void swap(int x[], int a, int b) {
+ for (int i = 0; i < 3; ++i) {
+ int t = x[a * 3 + i];
+ x[a * 3 + i] = x[b * 3 + i];
+ x[b * 3 + i] = t;
+ }
+ }
+
+ private void vecswap(int x[], int a, int b, int n) {
+ for (int i = 0; i < n; i++, a++, b++) {
+ swap(x, a, b);
+ }
+ }
+
+ };
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAccumulatingAggregator.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAccumulatingAggregator.java
deleted file mode 100644
index b0da898..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAccumulatingAggregator.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.group;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-
-public interface IAccumulatingAggregator {
- /**
- * Called once per aggregator before calling accumulate for the first time.
- *
- * @param accessor
- * - Accessor to the data tuple.
- * @param tIndex
- * - Index of the tuple in the accessor.
- * @throws HyracksDataException
- */
- public void init(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException;
-
- /**
- * Called once per tuple that belongs to this group.
- *
- * @param accessor
- * - Accessor to data tuple.
- * @param tIndex
- * - Index of tuple in the accessor.
- * @throws HyracksDataException
- */
- public void accumulate(IFrameTupleAccessor accessor, int tIndex) throws HyracksDataException;
-
- /**
- * Called finally to emit output. This method is called until it returns true. The method is free to
- * write out output to the provided appender until there is no more space and return false. It is the
- * caller's responsibility to flush and make room in the appender before this method is called again.
- *
- * @param appender
- * - Appender to write output to.
- * @param accessor
- * - Accessor to access the key.
- * @param tIndex
- * - Tuple index of the key in the accessor.
- * @param keyFieldIndexes
- * - Field indexes of the key field.
- * @return true if all output is written, false if the appender is full.
- * @throws HyracksDataException
- */
- public boolean output(FrameTupleAppender appender, IFrameTupleAccessor accessor, int tIndex, int[] keyFieldIndexes)
- throws HyracksDataException;
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAccumulatingAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAccumulatingAggregatorFactory.java
deleted file mode 100644
index 978f671..0000000
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAccumulatingAggregatorFactory.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.dataflow.std.group;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-
-public interface IAccumulatingAggregatorFactory extends Serializable {
- IAccumulatingAggregator createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDesc,
- RecordDescriptor outRecordDescriptor) throws HyracksDataException;
-}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAggregatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAggregatorDescriptor.java
new file mode 100644
index 0000000..2cf978c
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAggregatorDescriptor.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+
+/**
+ *
+ */
+public interface IAggregatorDescriptor {
+
+ /**
+ * Create an aggregate state
+ *
+ * @return
+ */
+ public AggregateState createAggregateStates();
+
+ /**
+ * Initialize the state based on the input tuple.
+ *
+ * @param accessor
+ * @param tIndex
+ * @param fieldOutput
+ * The data output for the frame containing the state. This may
+ * be null, if the state is maintained as a java object
+ * @param state
+ * The state to be initialized.
+ * @throws HyracksDataException
+ */
+ public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex, AggregateState state)
+ throws HyracksDataException;
+
+ /**
+ * Reset the aggregator. The corresponding aggregate state should be reset
+ * too. Note that here the frame is not an input argument, since it can be
+ * reset outside of the aggregator (simply reset the starting index of the
+ * buffer).
+ *
+ * @param state
+ */
+ public void reset();
+
+ /**
+ * Aggregate the value. Aggregate state should be updated correspondingly.
+ *
+ * @param accessor
+ * @param tIndex
+ * @param data
+ * The buffer containing the state, if frame-based-state is used.
+ * This means that it can be null if java-object-based-state is
+ * used.
+ * @param offset
+ * @param state
+ * The aggregate state.
+ * @throws HyracksDataException
+ */
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
+ int stateTupleIndex, AggregateState state) throws HyracksDataException;
+
+ /**
+ * Output the partial aggregation result.
+ *
+ * @param fieldOutput
+ * The data output for the output frame
+ * @param data
+ * The buffer containing the aggregation state
+ * @param offset
+ * @param state
+ * The aggregation state.
+ * @throws HyracksDataException
+ */
+ public void outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException;
+
+ /**
+ * Output the final aggregation result.
+ *
+ * @param fieldOutput
+ * The data output for the output frame
+ * @param data
+ * The buffer containing the aggregation state
+ * @param offset
+ * @param state
+ * The aggregation state.
+ * @throws HyracksDataException
+ */
+ public void outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException;
+
+ public void close();
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAggregatorDescriptorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAggregatorDescriptorFactory.java
new file mode 100644
index 0000000..9ff915d
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IAggregatorDescriptorFactory.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ *
+ */
+public interface IAggregatorDescriptorFactory extends Serializable {
+
+ IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor, int[] keyFields, final int[] keyFieldsInPartialResults)
+ throws HyracksDataException;
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IFieldAggregateDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IFieldAggregateDescriptor.java
new file mode 100644
index 0000000..a34fa73
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IFieldAggregateDescriptor.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group;
+
+import java.io.DataOutput;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ *
+ */
+public interface IFieldAggregateDescriptor {
+
+ /**
+ * Initialize the state based on the input tuple.
+ *
+ * @param accessor
+ * @param tIndex
+ * @param fieldOutput
+ * The data output for the frame containing the state. This may
+ * be null, if the state is maintained as a java object.
+ *
+ * Note that we have an assumption that the initialization of the
+ * binary state (if any) inserts the state fields into the buffer
+ * in a appending fashion. This means that an arbitrary initial
+ * size of the state can be accquired.
+ * @param state
+ * The state to be initialized.
+ * @throws HyracksDataException
+ */
+ public void init(IFrameTupleAccessor accessor, int tIndex, DataOutput fieldOutput, AggregateState state)
+ throws HyracksDataException;
+
+ /**
+ * Reset the aggregator. The corresponding aggregate state should be reset
+ * too. Note that here the frame is not an input argument, since it can be
+ * reset outside of the aggregator (simply reset the starting index of the
+ * buffer).
+ *
+ * @param state
+ */
+ public void reset();
+
+ /**
+ * Aggregate the value. Aggregate state should be updated correspondingly.
+ *
+ * @param accessor
+ * @param tIndex
+ * @param data
+ * The buffer containing the state, if frame-based-state is used.
+ * This means that it can be null if java-object-based-state is
+ * used.
+ *
+ * Here the length of binary state can be obtains from the state
+ * parameter, and if the content to be filled into that is over-
+ * flowing (larger than the reversed space), error should be
+ * emit.
+ * @param offset
+ * @param state
+ * The aggregate state.
+ * @throws HyracksDataException
+ */
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException;
+
+ /**
+ * Output the partial aggregation result.
+ *
+ * @param fieldOutput
+ * The data output for the output frame
+ * @param data
+ * The buffer containing the aggregation state
+ * @param offset
+ * @param state
+ * The aggregation state.
+ * @throws HyracksDataException
+ */
+ public void outputPartialResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException;
+
+ /**
+ * Output the final aggregation result.
+ *
+ * @param fieldOutput
+ * The data output for the output frame
+ * @param data
+ * The buffer containing the aggregation state
+ * @param offset
+ * @param state
+ * The aggregation state.
+ * @throws HyracksDataException
+ */
+ public void outputFinalResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException;
+
+ public boolean needsBinaryState();
+
+ public boolean needsObjectState();
+
+ public AggregateState createState();
+
+ /**
+ * Close the field aggregator
+ */
+ public void close();
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IFieldAggregateDescriptorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IFieldAggregateDescriptorFactory.java
new file mode 100644
index 0000000..6f50597
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/IFieldAggregateDescriptorFactory.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ *
+ */
+public interface IFieldAggregateDescriptorFactory extends Serializable {
+
+ public IFieldAggregateDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor) throws HyracksDataException;
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ISpillableTable.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ISpillableTable.java
index 7bd8cd8..ff80a23 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ISpillableTable.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ISpillableTable.java
@@ -21,9 +21,6 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-/**
- * @author jarodwen
- */
public interface ISpillableTable {
public void close();
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ISpillableTableFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ISpillableTableFactory.java
index 0ff1d1d..de9fac5 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ISpillableTableFactory.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/ISpillableTableFactory.java
@@ -21,15 +21,11 @@
import edu.uci.ics.hyracks.api.dataflow.value.INormalizedKeyComputerFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IAggregatorDescriptorFactory;
-/**
- * @author jarodwen
- */
public interface ISpillableTableFactory extends Serializable {
ISpillableTable buildSpillableTable(IHyracksTaskContext ctx, int[] keyFields,
IBinaryComparatorFactory[] comparatorFactories, INormalizedKeyComputerFactory normalizedKeyComputerFactory,
- IAggregatorDescriptorFactory aggregatorFactory, RecordDescriptor inRecordDescriptor,
+ IAggregatorDescriptorFactory aggregateFactory, RecordDescriptor inRecordDescriptor,
RecordDescriptor outRecordDescriptor, int framesLimit) throws HyracksDataException;
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupOperatorDescriptor.java
index a865456..c718ee4 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupOperatorDescriptor.java
@@ -23,7 +23,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
@@ -33,12 +32,12 @@
public class PreclusteredGroupOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
private final int[] groupFields;
private final IBinaryComparatorFactory[] comparatorFactories;
- private final IAccumulatingAggregatorFactory aggregatorFactory;
+ private final IAggregatorDescriptorFactory aggregatorFactory;
private static final long serialVersionUID = 1L;
public PreclusteredGroupOperatorDescriptor(JobSpecification spec, int[] groupFields,
- IBinaryComparatorFactory[] comparatorFactories, IAccumulatingAggregatorFactory aggregatorFactory,
+ IBinaryComparatorFactory[] comparatorFactories, IAggregatorDescriptorFactory aggregatorFactory,
RecordDescriptor recordDescriptor) {
super(spec, 1, 1);
this.groupFields = groupFields;
@@ -48,15 +47,15 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
}
final RecordDescriptor inRecordDesc = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
- final IAccumulatingAggregator aggregator = aggregatorFactory.createAggregator(ctx, inRecordDesc,
- recordDescriptors[0]);
+ final IAggregatorDescriptor aggregator = aggregatorFactory.createAggregator(ctx, inRecordDesc,
+ recordDescriptors[0], groupFields, groupFields);
final ByteBuffer copyFrame = ctx.allocateFrame();
final FrameTupleAccessor copyFrameAccessor = new FrameTupleAccessor(ctx.getFrameSize(), inRecordDesc);
copyFrameAccessor.reset(copyFrame);
@@ -68,7 +67,8 @@
@Override
public void open() throws HyracksDataException {
- pgw = new PreclusteredGroupWriter(ctx, groupFields, comparators, aggregator, inRecordDesc, writer);
+ pgw = new PreclusteredGroupWriter(ctx, groupFields, comparators, aggregator, inRecordDesc,
+ recordDescriptors[0], writer);
pgw.open();
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupWriter.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupWriter.java
index df7f9f9..89955c0 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupWriter.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/PreclusteredGroupWriter.java
@@ -21,6 +21,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
@@ -28,28 +29,37 @@
public class PreclusteredGroupWriter implements IFrameWriter {
private final int[] groupFields;
private final IBinaryComparator[] comparators;
- private final IAccumulatingAggregator aggregator;
+ private final IAggregatorDescriptor aggregator;
+ private final AggregateState aggregateState;
private final IFrameWriter writer;
private final ByteBuffer copyFrame;
private final FrameTupleAccessor inFrameAccessor;
private final FrameTupleAccessor copyFrameAccessor;
+
private final ByteBuffer outFrame;
private final FrameTupleAppender appender;
+ private final ArrayTupleBuilder tupleBuilder;
+
private boolean first;
public PreclusteredGroupWriter(IHyracksTaskContext ctx, int[] groupFields, IBinaryComparator[] comparators,
- IAccumulatingAggregator aggregator, RecordDescriptor inRecordDesc, IFrameWriter writer) {
+ IAggregatorDescriptor aggregator, RecordDescriptor inRecordDesc, RecordDescriptor outRecordDesc,
+ IFrameWriter writer) {
this.groupFields = groupFields;
this.comparators = comparators;
this.aggregator = aggregator;
+ this.aggregateState = aggregator.createAggregateStates();
this.writer = writer;
copyFrame = ctx.allocateFrame();
inFrameAccessor = new FrameTupleAccessor(ctx.getFrameSize(), inRecordDesc);
copyFrameAccessor = new FrameTupleAccessor(ctx.getFrameSize(), inRecordDesc);
copyFrameAccessor.reset(copyFrame);
+
outFrame = ctx.allocateFrame();
appender = new FrameTupleAppender(ctx.getFrameSize());
appender.reset(outFrame, true);
+
+ tupleBuilder = new ArrayTupleBuilder(outRecordDesc.getFields().length);
}
@Override
@@ -64,16 +74,23 @@
int nTuples = inFrameAccessor.getTupleCount();
for (int i = 0; i < nTuples; ++i) {
if (first) {
- aggregator.init(inFrameAccessor, i);
+
+ tupleBuilder.reset();
+ for (int j = 0; j < groupFields.length; j++) {
+ tupleBuilder.addField(inFrameAccessor, i, groupFields[j]);
+ }
+ aggregator.init(tupleBuilder, inFrameAccessor, i, aggregateState);
+
first = false;
+
} else {
if (i == 0) {
switchGroupIfRequired(copyFrameAccessor, copyFrameAccessor.getTupleCount() - 1, inFrameAccessor, i);
} else {
switchGroupIfRequired(inFrameAccessor, i - 1, inFrameAccessor, i);
}
+
}
- aggregator.accumulate(inFrameAccessor, i);
}
FrameUtils.copy(buffer, copyFrame);
}
@@ -82,19 +99,36 @@
FrameTupleAccessor currTupleAccessor, int currTupleIndex) throws HyracksDataException {
if (!sameGroup(prevTupleAccessor, prevTupleIndex, currTupleAccessor, currTupleIndex)) {
writeOutput(prevTupleAccessor, prevTupleIndex);
- aggregator.init(currTupleAccessor, currTupleIndex);
+
+ tupleBuilder.reset();
+ for (int j = 0; j < groupFields.length; j++) {
+ tupleBuilder.addField(currTupleAccessor, currTupleIndex, groupFields[j]);
+ }
+ aggregator.init(tupleBuilder, currTupleAccessor, currTupleIndex, aggregateState);
+ } else {
+ aggregator.aggregate(currTupleAccessor, currTupleIndex, null, 0, aggregateState);
}
}
private void writeOutput(final FrameTupleAccessor lastTupleAccessor, int lastTupleIndex)
throws HyracksDataException {
- if (!aggregator.output(appender, lastTupleAccessor, lastTupleIndex, groupFields)) {
- FrameUtils.flushFrame(appender.getBuffer(), writer);
- appender.reset(appender.getBuffer(), true);
- if (!aggregator.output(appender, lastTupleAccessor, lastTupleIndex, groupFields)) {
- throw new IllegalStateException();
+
+ tupleBuilder.reset();
+ for (int j = 0; j < groupFields.length; j++) {
+ tupleBuilder.addField(lastTupleAccessor, lastTupleIndex, groupFields[j]);
+ }
+ aggregator.outputFinalResult(tupleBuilder, lastTupleAccessor, lastTupleIndex, aggregateState);
+
+ if (!appender.appendSkipEmptyField(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0,
+ tupleBuilder.getSize())) {
+ FrameUtils.flushFrame(outFrame, writer);
+ appender.reset(outFrame, true);
+ if (!appender.appendSkipEmptyField(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray(), 0,
+ tupleBuilder.getSize())) {
+ throw new HyracksDataException("The output cannot be fit into a frame.");
}
}
+
}
private boolean sameGroup(FrameTupleAccessor a1, int t1Idx, FrameTupleAccessor a2, int t2Idx) {
@@ -121,9 +155,10 @@
if (!first) {
writeOutput(copyFrameAccessor, copyFrameAccessor.getTupleCount() - 1);
if (appender.getTupleCount() > 0) {
- FrameUtils.flushFrame(appender.getBuffer(), writer);
+ FrameUtils.flushFrame(outFrame, writer);
}
}
+ aggregateState.close();
writer.close();
}
}
\ No newline at end of file
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/AvgFieldGroupAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/AvgFieldGroupAggregatorFactory.java
new file mode 100644
index 0000000..2e781b5
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/AvgFieldGroupAggregatorFactory.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.aggregators;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+
+/**
+ *
+ */
+public class AvgFieldGroupAggregatorFactory implements IFieldAggregateDescriptorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int aggField;
+
+ private final boolean useObjectState;
+
+ public AvgFieldGroupAggregatorFactory(int aggField, boolean useObjectState) {
+ this.aggField = aggField;
+ this.useObjectState = useObjectState;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see edu.uci.ics.hyracks.dataflow.std.aggregations.
+ * IFieldAggregateDescriptorFactory
+ * #createAggregator(edu.uci.ics.hyracks.api.context.IHyracksTaskContext,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor)
+ */
+ @Override
+ public IFieldAggregateDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor) throws HyracksDataException {
+
+ return new IFieldAggregateDescriptor() {
+
+ @Override
+ public void reset() {
+ }
+
+ @Override
+ public void outputPartialResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ int sum, count;
+ if (!useObjectState) {
+ sum = IntegerSerializerDeserializer.getInt(data, offset);
+ count = IntegerSerializerDeserializer.getInt(data, offset + 4);
+ } else {
+ Integer[] fields = (Integer[]) state.state;
+ sum = fields[0];
+ count = fields[1];
+ }
+ try {
+ fieldOutput.writeInt(sum);
+ fieldOutput.writeInt(count);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public void outputFinalResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ int sum, count;
+ if (!useObjectState) {
+ sum = IntegerSerializerDeserializer.getInt(data, offset);
+ count = IntegerSerializerDeserializer.getInt(data, offset + 4);
+ } else {
+ Integer[] fields = (Integer[]) state.state;
+ sum = fields[0];
+ count = fields[1];
+ }
+ try {
+ fieldOutput.writeFloat((float) sum / count);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public void init(IFrameTupleAccessor accessor, int tIndex, DataOutput fieldOutput, AggregateState state)
+ throws HyracksDataException {
+ int sum = 0;
+ int count = 0;
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+ sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
+ count += 1;
+ if (!useObjectState) {
+ try {
+ fieldOutput.writeInt(sum);
+ fieldOutput.writeInt(count);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when initializing the aggregator.");
+ }
+ } else {
+ state.state = new Integer[] { sum, count };
+ }
+ }
+
+ @Override
+ public void close() {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset,
+ AggregateState state) throws HyracksDataException {
+ int sum = 0, count = 0;
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+ sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
+ count += 1;
+ if (!useObjectState) {
+ ByteBuffer buf = ByteBuffer.wrap(data);
+ sum += buf.getInt(offset);
+ count += buf.getInt(offset + 4);
+ buf.putInt(offset, sum);
+ buf.putInt(offset + 4, count);
+ } else {
+ Integer[] fields = (Integer[]) state.state;
+ sum += fields[0];
+ count += fields[1];
+ state.state = new Integer[] { sum, count };
+ }
+ }
+
+ @Override
+ public boolean needsObjectState() {
+ return useObjectState;
+ }
+
+ @Override
+ public boolean needsBinaryState() {
+ return !useObjectState;
+ }
+
+ @Override
+ public AggregateState createState() {
+ return new AggregateState(new Integer[] { 0, 0 });
+ }
+
+ };
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/AvgFieldMergeAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/AvgFieldMergeAggregatorFactory.java
new file mode 100644
index 0000000..cc5c1e1
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/AvgFieldMergeAggregatorFactory.java
@@ -0,0 +1,173 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.aggregators;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+
+/**
+ *
+ */
+public class AvgFieldMergeAggregatorFactory implements IFieldAggregateDescriptorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int aggField;
+
+ private final boolean useObjectState;
+
+ public AvgFieldMergeAggregatorFactory(int aggField, boolean useObjectState) {
+ this.aggField = aggField;
+ this.useObjectState = useObjectState;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see edu.uci.ics.hyracks.dataflow.std.aggregations.
+ * IFieldAggregateDescriptorFactory
+ * #createAggregator(edu.uci.ics.hyracks.api.context.IHyracksTaskContext,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor)
+ */
+ @Override
+ public IFieldAggregateDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor) throws HyracksDataException {
+
+ return new IFieldAggregateDescriptor() {
+
+ @Override
+ public void reset() {
+ }
+
+ @Override
+ public void outputPartialResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ int sum, count;
+ if (!useObjectState) {
+ sum = IntegerSerializerDeserializer.getInt(data, offset);
+ count = IntegerSerializerDeserializer.getInt(data, offset + 4);
+ } else {
+ Integer[] fields = (Integer[]) state.state;
+ sum = fields[0];
+ count = fields[1];
+ }
+ try {
+ fieldOutput.writeInt(sum);
+ fieldOutput.writeInt(count);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public void outputFinalResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ int sum, count;
+ if (!useObjectState) {
+ sum = IntegerSerializerDeserializer.getInt(data, offset);
+ count = IntegerSerializerDeserializer.getInt(data, offset + 4);
+ } else {
+ Integer[] fields = (Integer[]) state.state;
+ sum = fields[0];
+ count = fields[1];
+ }
+ try {
+ fieldOutput.writeFloat((float) sum / count);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public void close() {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset,
+ AggregateState state) throws HyracksDataException {
+ int sum = 0, count = 0;
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+ sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
+ count += 1;
+ if (!useObjectState) {
+ ByteBuffer buf = ByteBuffer.wrap(data);
+ sum += buf.getInt(offset);
+ count += buf.getInt(offset + 4);
+ buf.putInt(offset, sum);
+ buf.putInt(offset + 4, count);
+ } else {
+ Integer[] fields = (Integer[]) state.state;
+ sum += fields[0];
+ count += fields[1];
+ state.state = new Integer[] { sum, count };
+ }
+ }
+
+ @Override
+ public boolean needsObjectState() {
+ return useObjectState;
+ }
+
+ @Override
+ public boolean needsBinaryState() {
+ return !useObjectState;
+ }
+
+ @Override
+ public AggregateState createState() {
+ return new AggregateState(new Integer[] { 0, 0 });
+ }
+
+ @Override
+ public void init(IFrameTupleAccessor accessor, int tIndex, DataOutput fieldOutput, AggregateState state)
+ throws HyracksDataException {
+ int sum = 0;
+ int count = 0;
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+ sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
+ count += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldStart + 4);
+ if (!useObjectState) {
+ try {
+ fieldOutput.writeInt(sum);
+ fieldOutput.writeInt(count);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when initializing the aggregator.");
+ }
+ } else {
+ state.state = new Integer[] { sum, count };
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/CountFieldAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/CountFieldAggregatorFactory.java
new file mode 100644
index 0000000..9bfec8e
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/CountFieldAggregatorFactory.java
@@ -0,0 +1,141 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.aggregators;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+
+/**
+ *
+ */
+public class CountFieldAggregatorFactory implements IFieldAggregateDescriptorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final boolean useObjectState;
+
+ public CountFieldAggregatorFactory(boolean useObjectState) {
+ this.useObjectState = useObjectState;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see edu.uci.ics.hyracks.dataflow.std.aggregations.
+ * IFieldAggregateDescriptorFactory
+ * #createAggregator(edu.uci.ics.hyracks.api.context.IHyracksTaskContext,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor)
+ */
+ @Override
+ public IFieldAggregateDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor) throws HyracksDataException {
+ return new IFieldAggregateDescriptor() {
+
+ @Override
+ public void reset() {
+ }
+
+ @Override
+ public void outputPartialResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ int count;
+ if (!useObjectState) {
+ count = IntegerSerializerDeserializer.getInt(data, offset);
+ } else {
+ count = (Integer) state.state;
+ }
+ try {
+ fieldOutput.writeInt(count);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public void outputFinalResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ int count;
+ if (!useObjectState) {
+ count = IntegerSerializerDeserializer.getInt(data, offset);
+ } else {
+ count = (Integer) state.state;
+ }
+ try {
+ fieldOutput.writeInt(count);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public void init(IFrameTupleAccessor accessor, int tIndex, DataOutput fieldOutput, AggregateState state)
+ throws HyracksDataException {
+ int count = 1;
+ if (!useObjectState) {
+ try {
+ fieldOutput.writeInt(count);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when initializing the aggregator.");
+ }
+ } else {
+ state.state = count;
+ }
+ }
+
+ public boolean needsObjectState() {
+ return useObjectState;
+ }
+
+ public boolean needsBinaryState() {
+ return !useObjectState;
+ }
+
+ public AggregateState createState() {
+ return new AggregateState(new Integer(0));
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset,
+ AggregateState state) throws HyracksDataException {
+ int count = 1;
+ if (!useObjectState) {
+ ByteBuffer buf = ByteBuffer.wrap(data);
+ count += buf.getInt(offset);
+ buf.putInt(offset, count);
+ } else {
+ count += (Integer) state.state;
+ state.state = count;
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/FloatSumFieldAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/FloatSumFieldAggregatorFactory.java
new file mode 100644
index 0000000..f8b1d74
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/FloatSumFieldAggregatorFactory.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.aggregators;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+
+/**
+ *
+ */
+public class FloatSumFieldAggregatorFactory implements
+ IFieldAggregateDescriptorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int aggField;
+
+ private final boolean useObjectState;
+
+ public FloatSumFieldAggregatorFactory(int aggField, boolean useObjState){
+ this.aggField = aggField;
+ this.useObjectState = useObjState;
+ }
+
+ /* (non-Javadoc)
+ * @see edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory#createAggregator(edu.uci.ics.hyracks.api.context.IHyracksTaskContext, edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor, edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor)
+ */
+ @Override
+ public IFieldAggregateDescriptor createAggregator(IHyracksTaskContext ctx,
+ RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor) throws HyracksDataException {
+ return new IFieldAggregateDescriptor() {
+
+ @Override
+ public void reset() {
+
+ }
+
+ @Override
+ public void outputPartialResult(DataOutput fieldOutput, byte[] data,
+ int offset, AggregateState state) throws HyracksDataException {
+ float sum;
+ if (!useObjectState) {
+ sum = FloatSerializerDeserializer.getFloat(data, offset);
+ } else {
+ sum = (Float) state.state;
+ }
+ try {
+ fieldOutput.writeFloat(sum);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public void outputFinalResult(DataOutput fieldOutput, byte[] data,
+ int offset, AggregateState state) throws HyracksDataException {
+ float sum;
+ if (!useObjectState) {
+ sum = FloatSerializerDeserializer.getFloat(data, offset);
+ } else {
+ sum = (Float) state.state;
+ }
+ try {
+ fieldOutput.writeFloat(sum);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public boolean needsObjectState() {
+ return useObjectState;
+ }
+
+ @Override
+ public boolean needsBinaryState() {
+ return !useObjectState;
+ }
+
+ @Override
+ public void init(IFrameTupleAccessor accessor, int tIndex,
+ DataOutput fieldOutput, AggregateState state)
+ throws HyracksDataException {
+ float sum = 0;
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+
+ sum += FloatSerializerDeserializer.getFloat(accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
+
+ if (!useObjectState) {
+ try {
+ fieldOutput.writeFloat(sum);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when initializing the aggregator.");
+ }
+ } else {
+ state.state = sum;
+ }
+ }
+
+ @Override
+ public AggregateState createState() {
+ return new AggregateState(new Float(0.0));
+ }
+
+ @Override
+ public void close() {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex,
+ byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ float sum = 0;
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+ sum += FloatSerializerDeserializer.getFloat(accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
+
+ if (!useObjectState) {
+ ByteBuffer buf = ByteBuffer.wrap(data);
+ sum += buf.getFloat(offset);
+ buf.putFloat(offset, sum);
+ } else {
+ sum += (Float) state.state;
+ state.state = sum;
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/IntSumFieldAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/IntSumFieldAggregatorFactory.java
new file mode 100644
index 0000000..7d85deb
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/IntSumFieldAggregatorFactory.java
@@ -0,0 +1,157 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.aggregators;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+
+/**
+ *
+ */
+public class IntSumFieldAggregatorFactory implements IFieldAggregateDescriptorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int aggField;
+
+ private final boolean useObjectState;
+
+ public IntSumFieldAggregatorFactory(int aggField, boolean useObjState) {
+ this.aggField = aggField;
+ this.useObjectState = useObjState;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see edu.uci.ics.hyracks.dataflow.std.aggregations.
+ * IFieldAggregateDescriptorFactory
+ * #createAggregator(edu.uci.ics.hyracks.api.context.IHyracksTaskContext,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor)
+ */
+ @Override
+ public IFieldAggregateDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor) throws HyracksDataException {
+
+ return new IFieldAggregateDescriptor() {
+
+ @Override
+ public void reset() {
+ }
+
+ @Override
+ public void outputPartialResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ int sum;
+ if (!useObjectState) {
+ sum = IntegerSerializerDeserializer.getInt(data, offset);
+ } else {
+ sum = (Integer) state.state;
+ }
+ try {
+ fieldOutput.writeInt(sum);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public void outputFinalResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ int sum;
+ if (!useObjectState) {
+ sum = IntegerSerializerDeserializer.getInt(data, offset);
+ } else {
+ sum = (Integer) state.state;
+ }
+ try {
+ fieldOutput.writeInt(sum);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when writing aggregation to the output buffer.");
+ }
+ }
+
+ @Override
+ public void init(IFrameTupleAccessor accessor, int tIndex, DataOutput fieldOutput, AggregateState state)
+ throws HyracksDataException {
+
+ int sum = 0;
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+
+ sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
+
+ if (!useObjectState) {
+ try {
+ fieldOutput.writeInt(sum);
+ } catch (IOException e) {
+ throw new HyracksDataException("I/O exception when initializing the aggregator.");
+ }
+ } else {
+ state.state = sum;
+ }
+ }
+
+ public boolean needsObjectState() {
+ return useObjectState;
+ }
+
+ public boolean needsBinaryState() {
+ return !useObjectState;
+ }
+
+ public AggregateState createState() {
+ return new AggregateState(new Integer(0));
+ }
+
+ @Override
+ public void close() {
+
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset,
+ AggregateState state) throws HyracksDataException {
+ int sum = 0;
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+ sum += IntegerSerializerDeserializer.getInt(accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldStart);
+
+ if (!useObjectState) {
+ ByteBuffer buf = ByteBuffer.wrap(data);
+ sum += buf.getInt(offset);
+ buf.putInt(offset, sum);
+ } else {
+ sum += (Integer) state.state;
+ state.state = sum;
+ }
+ }
+ };
+ }
+
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/MinMaxStringFieldAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/MinMaxStringFieldAggregatorFactory.java
new file mode 100644
index 0000000..94ebcbd
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/MinMaxStringFieldAggregatorFactory.java
@@ -0,0 +1,199 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.aggregators;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+
+/**
+ *
+ */
+public class MinMaxStringFieldAggregatorFactory implements IFieldAggregateDescriptorFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int aggField;
+
+ private final boolean isMax;
+
+ private final boolean hasBinaryState;
+
+ public MinMaxStringFieldAggregatorFactory(int aggField, boolean isMax, boolean hasBinaryState) {
+ this.aggField = aggField;
+ this.isMax = isMax;
+ this.hasBinaryState = hasBinaryState;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.dataflow.std.aggregators.IAggregatorDescriptorFactory
+ * #createAggregator(edu.uci.ics.hyracks.api.context.IHyracksTaskContext,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor, int[])
+ */
+ @Override
+ public IFieldAggregateDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor) throws HyracksDataException {
+ return new IFieldAggregateDescriptor() {
+
+ @Override
+ public void reset() {
+ }
+
+ @Override
+ public void outputPartialResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ try {
+ if (hasBinaryState) {
+ int stateIdx = IntegerSerializerDeserializer.getInt(data, offset);
+ Object[] storedState = (Object[]) state.state;
+ fieldOutput.writeUTF((String) storedState[stateIdx]);
+ } else {
+ fieldOutput.writeUTF((String) state.state);
+ }
+ } catch (IOException e) {
+ throw new HyracksDataException(
+ "I/O exception when writing a string to the output writer in MinMaxStringAggregatorFactory.");
+ }
+ }
+
+ @Override
+ public void outputFinalResult(DataOutput fieldOutput, byte[] data, int offset, AggregateState state)
+ throws HyracksDataException {
+ try {
+ if (hasBinaryState) {
+ int stateIdx = IntegerSerializerDeserializer.getInt(data, offset);
+ Object[] storedState = (Object[]) state.state;
+ fieldOutput.writeUTF((String) storedState[stateIdx]);
+ } else {
+ fieldOutput.writeUTF((String) state.state);
+ }
+ } catch (IOException e) {
+ throw new HyracksDataException(
+ "I/O exception when writing a string to the output writer in MinMaxStringAggregatorFactory.");
+ }
+ }
+
+ @Override
+ public void init(IFrameTupleAccessor accessor, int tIndex, DataOutput fieldOutput, AggregateState state)
+ throws HyracksDataException {
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+ int fieldLength = accessor.getFieldLength(tIndex, aggField);
+ String strField = UTF8StringSerializerDeserializer.INSTANCE.deserialize(new DataInputStream(
+ new ByteArrayInputStream(accessor.getBuffer().array(), tupleOffset
+ + accessor.getFieldSlotsLength() + fieldStart, fieldLength)));
+ if (hasBinaryState) {
+ // Object-binary-state
+ Object[] storedState;
+ if (state.state == null) {
+ storedState = new Object[8];
+ storedState[0] = new Integer(0);
+ state.state = storedState;
+ } else {
+ storedState = (Object[]) state.state;
+ }
+ int stateCount = (Integer) (storedState[0]);
+ if (stateCount + 1 >= storedState.length) {
+ storedState = Arrays.copyOf(storedState, storedState.length * 2);
+ state.state = storedState;
+ }
+
+ stateCount++;
+ storedState[0] = stateCount;
+ storedState[stateCount] = strField;
+ try {
+ fieldOutput.writeInt(stateCount);
+ } catch (IOException e) {
+ throw new HyracksDataException(e.fillInStackTrace());
+ }
+ } else {
+ // Only object-state
+ state.state = strField;
+ }
+ }
+
+ @Override
+ public void close() {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, byte[] data, int offset,
+ AggregateState state) throws HyracksDataException {
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ int fieldStart = accessor.getFieldStartOffset(tIndex, aggField);
+ int fieldLength = accessor.getFieldLength(tIndex, aggField);
+ String strField = UTF8StringSerializerDeserializer.INSTANCE.deserialize(new DataInputStream(
+ new ByteArrayInputStream(accessor.getBuffer().array(), tupleOffset
+ + accessor.getFieldSlotsLength() + fieldStart, fieldLength)));
+ if (hasBinaryState) {
+ int stateIdx = IntegerSerializerDeserializer.getInt(data, offset);
+
+ Object[] storedState = (Object[]) state.state;
+
+ if (isMax) {
+ if (strField.length() > ((String) (storedState[stateIdx])).length()) {
+ storedState[stateIdx] = strField;
+ }
+ } else {
+ if (strField.length() < ((String) (storedState[stateIdx])).length()) {
+ storedState[stateIdx] = strField;
+ }
+ }
+ } else {
+ if (isMax) {
+ if (strField.length() > ((String) (state.state)).length()) {
+ state.state = strField;
+ }
+ } else {
+ if (strField.length() < ((String) (state.state)).length()) {
+ state.state = strField;
+ }
+ }
+ }
+ }
+
+ public boolean needsObjectState() {
+ return true;
+ }
+
+ public boolean needsBinaryState() {
+ return hasBinaryState;
+ }
+
+ public AggregateState createState() {
+ return new AggregateState();
+ }
+
+ };
+ }
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/MultiFieldsAggregatorFactory.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/MultiFieldsAggregatorFactory.java
new file mode 100644
index 0000000..6c436a9
--- /dev/null
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/group/aggregators/MultiFieldsAggregatorFactory.java
@@ -0,0 +1,170 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.dataflow.std.group.aggregators;
+
+import java.io.DataOutput;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.std.group.AggregateState;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IAggregatorDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+
+/**
+ *
+ */
+public class MultiFieldsAggregatorFactory implements IAggregatorDescriptorFactory {
+
+ private static final long serialVersionUID = 1L;
+ private final IFieldAggregateDescriptorFactory[] aggregatorFactories;
+ private int[] keys;
+
+ public MultiFieldsAggregatorFactory(int[] keys, IFieldAggregateDescriptorFactory[] aggregatorFactories) {
+ this.keys = keys;
+ this.aggregatorFactories = aggregatorFactories;
+ }
+
+ public MultiFieldsAggregatorFactory(IFieldAggregateDescriptorFactory[] aggregatorFactories) {
+ this.aggregatorFactories = aggregatorFactories;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * edu.uci.ics.hyracks.dataflow.std.aggregations.IAggregatorDescriptorFactory
+ * #createAggregator(edu.uci.ics.hyracks.api.context.IHyracksTaskContext,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor,
+ * edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor)
+ */
+ @Override
+ public IAggregatorDescriptor createAggregator(IHyracksTaskContext ctx, RecordDescriptor inRecordDescriptor,
+ RecordDescriptor outRecordDescriptor, final int[] keyFields, final int[] keyFieldsInPartialResults)
+ throws HyracksDataException {
+
+ final IFieldAggregateDescriptor[] aggregators = new IFieldAggregateDescriptor[aggregatorFactories.length];
+ for (int i = 0; i < aggregators.length; i++) {
+ aggregators[i] = aggregatorFactories[i].createAggregator(ctx, inRecordDescriptor, outRecordDescriptor);
+ }
+
+ if (this.keys == null) {
+ this.keys = keyFields;
+ }
+
+ return new IAggregatorDescriptor() {
+
+ @Override
+ public void reset() {
+ for (int i = 0; i < aggregators.length; i++) {
+ aggregators[i].reset();
+ }
+ }
+
+ @Override
+ public void outputPartialResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ DataOutput dos = tupleBuilder.getDataOutput();
+
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ for (int i = 0; i < aggregators.length; i++) {
+ int fieldOffset = accessor.getFieldStartOffset(tIndex, keys.length + i);
+ aggregators[i].outputPartialResult(dos, accessor.getBuffer().array(),
+ fieldOffset + accessor.getFieldSlotsLength() + tupleOffset,
+ ((AggregateState[]) state.state)[i]);
+ tupleBuilder.addFieldEndOffset();
+ }
+
+ }
+
+ @Override
+ public void outputFinalResult(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ DataOutput dos = tupleBuilder.getDataOutput();
+
+ int tupleOffset = accessor.getTupleStartOffset(tIndex);
+ for (int i = 0; i < aggregators.length; i++) {
+ if (aggregators[i].needsBinaryState()) {
+ int fieldOffset = accessor.getFieldStartOffset(tIndex, keys.length + i);
+ aggregators[i].outputFinalResult(dos, accessor.getBuffer().array(),
+ tupleOffset + accessor.getFieldSlotsLength() + fieldOffset,
+ ((AggregateState[]) state.state)[i]);
+ } else {
+ aggregators[i].outputFinalResult(dos, null, 0, ((AggregateState[]) state.state)[i]);
+ }
+ tupleBuilder.addFieldEndOffset();
+ }
+ }
+
+ @Override
+ public void init(ArrayTupleBuilder tupleBuilder, IFrameTupleAccessor accessor, int tIndex,
+ AggregateState state) throws HyracksDataException {
+ DataOutput dos = tupleBuilder.getDataOutput();
+
+ for (int i = 0; i < aggregators.length; i++) {
+ aggregators[i].init(accessor, tIndex, dos, ((AggregateState[]) state.state)[i]);
+ if (aggregators[i].needsBinaryState()) {
+ tupleBuilder.addFieldEndOffset();
+ }
+ }
+ }
+
+ @Override
+ public AggregateState createAggregateStates() {
+ AggregateState[] states = new AggregateState[aggregators.length];
+ for (int i = 0; i < states.length; i++) {
+ states[i] = aggregators[i].createState();
+ }
+ return new AggregateState(states);
+ }
+
+ @Override
+ public void close() {
+ for (int i = 0; i < aggregators.length; i++) {
+ aggregators[i].close();
+ }
+ }
+
+ @Override
+ public void aggregate(IFrameTupleAccessor accessor, int tIndex, IFrameTupleAccessor stateAccessor,
+ int stateTupleIndex, AggregateState state) throws HyracksDataException {
+ if (stateAccessor != null) {
+ int stateTupleOffset = stateAccessor.getTupleStartOffset(stateTupleIndex);
+ int fieldIndex = 0;
+ for (int i = 0; i < aggregators.length; i++) {
+ if (aggregators[i].needsBinaryState()) {
+ int stateFieldOffset = stateAccessor.getFieldStartOffset(stateTupleIndex, keys.length
+ + fieldIndex);
+ aggregators[i].aggregate(accessor, tIndex, stateAccessor.getBuffer().array(),
+ stateTupleOffset + stateAccessor.getFieldSlotsLength() + stateFieldOffset,
+ ((AggregateState[]) state.state)[i]);
+ fieldIndex++;
+ } else {
+ aggregators[i].aggregate(accessor, tIndex, null, 0, ((AggregateState[]) state.state)[i]);
+ }
+ }
+ } else {
+ for (int i = 0; i < aggregators.length; i++) {
+ aggregators[i].aggregate(accessor, tIndex, null, 0, ((AggregateState[]) state.state)[i]);
+ }
+ }
+ }
+ };
+ }
+}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java
index e0035e7..773e1e5 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/GraceHashJoinOperatorDescriptor.java
@@ -34,7 +34,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
@@ -160,7 +159,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
@@ -191,7 +190,7 @@
closeWriter(i);
}
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
private void closeWriter(int i) throws HyracksDataException {
@@ -221,16 +220,15 @@
int entry = hpc.partition(accessor0, i, numPartitions);
ByteBuffer outbuf = outbufs[entry];
- appender.reset(outbuf, true);
- while (true) {
- if (appender.append(accessor0, i)) {
- break;
- } else {
- // buffer is full, ie. we cannot fit the tuple
- // into the buffer -- write it to disk
- write(entry, outbuf);
- outbuf.clear();
- appender.reset(outbuf, true);
+ appender.reset(outbuf, false);
+ if (!appender.append(accessor0, i)) {
+ // buffer is full, ie. we cannot fit the tuple
+ // into the buffer -- write it to disk
+ write(entry, outbuf);
+ outbuf.clear();
+ appender.reset(outbuf, true);
+ if (!appender.append(accessor0, i)) {
+ throw new HyracksDataException("Item too big to fit in frame");
}
}
}
@@ -263,7 +261,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
final IBinaryComparator[] comparators = new IBinaryComparator[comparatorFactories.length];
for (int i = 0; i < comparatorFactories.length; ++i) {
@@ -287,9 +285,9 @@
@Override
public void initialize() throws HyracksDataException {
- HashPartitionTaskState rState = (HashPartitionTaskState) env.getTaskState(new TaskId(
+ HashPartitionTaskState rState = (HashPartitionTaskState) ctx.getTaskState(new TaskId(
new ActivityId(getOperatorId(), RPARTITION_ACTIVITY_ID), partition));
- HashPartitionTaskState sState = (HashPartitionTaskState) env.getTaskState(new TaskId(
+ HashPartitionTaskState sState = (HashPartitionTaskState) ctx.getTaskState(new TaskId(
new ActivityId(getOperatorId(), SPARTITION_ACTIVITY_ID), partition));
buildWriters = sState.fWriters;
probeWriters = rState.fWriters;
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
index 66aa8fb..99f0020 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/HybridHashJoinOperatorDescriptor.java
@@ -35,7 +35,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
@@ -172,7 +171,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 1);
@@ -212,7 +211,7 @@
closeWriter(i);
}
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@Override
@@ -357,7 +356,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 1);
@@ -391,7 +390,7 @@
@Override
public void open() throws HyracksDataException {
- state = (BuildAndPartitionTaskState) env.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ state = (BuildAndPartitionTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
BUILD_AND_PARTITION_ACTIVITY_ID), partition));
writer.open();
buildWriters = state.fWriters;
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
index a1fc0b8..61fc0b3 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/InMemoryHashJoinOperatorDescriptor.java
@@ -33,7 +33,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
@@ -135,7 +134,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
final RecordDescriptor rd0 = recordDescProvider
.getInputRecordDescriptor(getOperatorId(), BUILD_ACTIVITY_ID);
@@ -179,7 +178,7 @@
@Override
public void close() throws HyracksDataException {
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@Override
@@ -198,14 +197,14 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
IOperatorNodePushable op = new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
private HashBuildTaskState state;
@Override
public void open() throws HyracksDataException {
- state = (HashBuildTaskState) env.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ state = (HashBuildTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
BUILD_ACTIVITY_ID), partition));
writer.open();
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoinOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoinOperatorDescriptor.java
index de1b0ea..21a828c 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoinOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/NestedLoopJoinOperatorDescriptor.java
@@ -30,7 +30,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
@@ -102,7 +101,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
final RecordDescriptor rd0 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
final RecordDescriptor rd1 = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 1);
@@ -130,7 +129,7 @@
@Override
public void close() throws HyracksDataException {
state.joiner.closeCache();
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@Override
@@ -149,7 +148,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
IOperatorNodePushable op = new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
@@ -157,7 +156,7 @@
@Override
public void open() throws HyracksDataException {
- state = (JoinCacheTaskState) env.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ state = (JoinCacheTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
JOIN_CACHE_ACTIVITY_ID), partition));
writer.open();
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
index c6c899d..87e2b2c 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/join/OptimizedHybridHashJoinOperatorDescriptor.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package edu.uci.ics.hyracks.dataflow.std.join;
import java.io.DataInput;
@@ -13,23 +27,22 @@
import edu.uci.ics.hyracks.api.dataflow.TaskId;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionGeneratorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerGeneratorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePartitionComputerFamily;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTuplePairComparator;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerGeneratorFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFamily;
import edu.uci.ics.hyracks.dataflow.common.data.partition.RepartitionComputerGeneratorFactory;
import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
@@ -42,48 +55,45 @@
/**
* @author pouria
- * This class guides the joining process, and switches between different
- joining techniques, w.r.t the implemented optimizations and skew in size of the
- partitions.
-
- - Operator overview:
- Assume we are trying to do (R Join S), with M buffers available, while we have an estimate on the size
- of R (in terms of buffers). HHJ (Hybrid Hash Join) has two main phases: Build and Probe, where in our implementation Probe phase
- can apply HHJ recursively, based on the value of M and size of R and S. HHJ phases proceed as follow:
-
- * BUILD:
- – Calculate number of partitions (Based on the size of R, fudge factor and M) [See Shapiro's paper for the detailed discussion].
- – Initialize the build phase (one frame per partition – all partitions considered resident at first)
- – Read tuples of R, frame by frame, and hash each tuple (based on a given hash function) to find
- its target partition and try to append it to that partition:
- – If target partition's buffer is full, try to allocate a new buffer for it.
- – if no free buffer is available, find the largest resident partition and spill it. Using its freed
- buffers after spilling, allocate a new buffer for the target partition.
- – Being done with R, close the build phase. (During closing we write the very last buffer of each
- spilled partition to the disk, and we do partition tuning, where we try to bring back as many buffers, belonging to
- spilled partitions as possible into memory, based on the free buffers - We will stop at the point where remaining free buffers is not enough
- for reloading an entire partition back into memory)
- – Create the hash table for the resident partitions (basically we create an in-memory hash join here)
-
- * PROBE:
- – Initialize the probe phase on S (mainly allocate one buffer per spilled partition, and one buffer
- for the whole resident partitions)
- – Read tuples of S, frame by frame and hash each tuple T to its target partition P
- – if P is a resident partition, pass T to the in-memory hash join and generate the output record,
- if any matching(s) record found
- – if P is spilled, write T to the dedicated buffer for P (on the probe side)
- – Once scanning of S is done, we try to join partition pairs (Ri, Si) of the spilled partitions:
- – if any of Ri or Si is smaller than M, then we simply use an in-memory hash join to join them
- – otherwise we apply HHJ recursively:
- – if after applying HHJ recursively, we do not gain enough size reduction (max size of the
- resulting partitions were more than 80% of the initial Ri,Si size) then we switch to
- nested loop join for joining.
- – (At each step of partition-pair joining, we consider role reversal, which means if size of Si were
- greater than Ri, then we make sure that we switch the roles of build/probe between them)
+ * This class guides the joining process, and switches between different
+ * joining techniques, w.r.t the implemented optimizations and skew in size of the
+ * partitions.
+ * - Operator overview:
+ * Assume we are trying to do (R Join S), with M buffers available, while we have an estimate on the size
+ * of R (in terms of buffers). HHJ (Hybrid Hash Join) has two main phases: Build and Probe, where in our implementation Probe phase
+ * can apply HHJ recursively, based on the value of M and size of R and S. HHJ phases proceed as follow:
+ * BUILD:
+ * – Calculate number of partitions (Based on the size of R, fudge factor and M) [See Shapiro's paper for the detailed discussion].
+ * – Initialize the build phase (one frame per partition – all partitions considered resident at first)
+ * – Read tuples of R, frame by frame, and hash each tuple (based on a given hash function) to find
+ * its target partition and try to append it to that partition:
+ * – If target partition's buffer is full, try to allocate a new buffer for it.
+ * – if no free buffer is available, find the largest resident partition and spill it. Using its freed
+ * buffers after spilling, allocate a new buffer for the target partition.
+ * – Being done with R, close the build phase. (During closing we write the very last buffer of each
+ * spilled partition to the disk, and we do partition tuning, where we try to bring back as many buffers, belonging to
+ * spilled partitions as possible into memory, based on the free buffers - We will stop at the point where remaining free buffers is not enough
+ * for reloading an entire partition back into memory)
+ * – Create the hash table for the resident partitions (basically we create an in-memory hash join here)
+ * PROBE:
+ * – Initialize the probe phase on S (mainly allocate one buffer per spilled partition, and one buffer
+ * for the whole resident partitions)
+ * – Read tuples of S, frame by frame and hash each tuple T to its target partition P
+ * – if P is a resident partition, pass T to the in-memory hash join and generate the output record,
+ * if any matching(s) record found
+ * – if P is spilled, write T to the dedicated buffer for P (on the probe side)
+ * – Once scanning of S is done, we try to join partition pairs (Ri, Si) of the spilled partitions:
+ * – if any of Ri or Si is smaller than M, then we simply use an in-memory hash join to join them
+ * – otherwise we apply HHJ recursively:
+ * – if after applying HHJ recursively, we do not gain enough size reduction (max size of the
+ * resulting partitions were more than 80% of the initial Ri,Si size) then we switch to
+ * nested loop join for joining.
+ * – (At each step of partition-pair joining, we consider role reversal, which means if size of Si were
+ * greater than Ri, then we make sure that we switch the roles of build/probe between them)
*/
public class OptimizedHybridHashJoinOperatorDescriptor extends AbstractOperatorDescriptor {
-
+
private static final int BUILD_AND_PARTITION_ACTIVITY_ID = 0;
private static final int PARTITION_AND_JOIN_ACTIVITY_ID = 1;
@@ -98,7 +108,7 @@
private final double fudgeFactor;
private final int[] probeKeys;
private final int[] buildKeys;
- private final IBinaryHashFunctionGeneratorFactory[] hashFunctionGeneratorFactories;
+ private final IBinaryHashFunctionFamily[] hashFunctionGeneratorFactories;
private final IBinaryComparatorFactory[] comparatorFactories; //For in-mem HJ
private final ITuplePairComparatorFactory tuplePairComparatorFactory0; //For NLJ in probe
private final ITuplePairComparatorFactory tuplePairComparatorFactory1; //For NLJ in probe
@@ -107,7 +117,7 @@
private final INullWriterFactory[] nullWriterFactories1;
public OptimizedHybridHashJoinOperatorDescriptor(JobSpecification spec, int memsize, int inputsize0, double factor,
- int[] keys0, int[] keys1, IBinaryHashFunctionGeneratorFactory[] hashFunctionGeneratorFactories,
+ int[] keys0, int[] keys1, IBinaryHashFunctionFamily[] hashFunctionGeneratorFactories,
IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor,
ITuplePairComparatorFactory tupPaircomparatorFactory0,
ITuplePairComparatorFactory tupPaircomparatorFactory1, boolean isLeftOuter,
@@ -130,7 +140,7 @@
}
public OptimizedHybridHashJoinOperatorDescriptor(JobSpecification spec, int memsize, int inputsize0, double factor,
- int[] keys0, int[] keys1, IBinaryHashFunctionGeneratorFactory[] hashFunctionGeneratorFactories,
+ int[] keys0, int[] keys1, IBinaryHashFunctionFamily[] hashFunctionGeneratorFactories,
IBinaryComparatorFactory[] comparatorFactories, RecordDescriptor recordDescriptor,
ITuplePairComparatorFactory tupPaircomparatorFactory0, ITuplePairComparatorFactory tupPaircomparatorFactory1)
throws HyracksDataException {
@@ -178,8 +188,9 @@
}
if (memorySize > buildSize) {
return 1; //We will switch to in-Mem HJ eventually
- }
- numberOfPartitions = (int) (Math.ceil((double) (buildSize * factor / nPartitions - memorySize) / (double) (memorySize - 1)));
+ }
+ numberOfPartitions = (int) (Math.ceil((double) (buildSize * factor / nPartitions - memorySize)
+ / (double) (memorySize - 1)));
if (numberOfPartitions <= 0) {
numberOfPartitions = 1; //becomes in-memory hash join
}
@@ -214,7 +225,7 @@
}
}
-
+
/*
* Build phase of Hybrid Hash Join:
* Creating an instance of Hybrid Hash Join, using Shapiro's formula
@@ -230,7 +241,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
final RecordDescriptor probeRd = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
@@ -252,18 +263,19 @@
private BuildAndPartitionTaskState state = new BuildAndPartitionTaskState(ctx.getJobletContext()
.getJobId(), new TaskId(getActivityId(), partition));
- ITuplePartitionComputer probeHpc = new FieldHashPartitionComputerGeneratorFactory(probeKeys,
+ ITuplePartitionComputer probeHpc = new FieldHashPartitionComputerFamily(probeKeys,
hashFunctionGeneratorFactories).createPartitioner(0);
- ITuplePartitionComputer buildHpc = new FieldHashPartitionComputerGeneratorFactory(buildKeys,
+ ITuplePartitionComputer buildHpc = new FieldHashPartitionComputerFamily(buildKeys,
hashFunctionGeneratorFactories).createPartitioner(0);
@Override
public void open() throws HyracksDataException {
- if (memsize <= 2){ //Dedicated buffers: One buffer to read and one buffer for output
+ if (memsize <= 2) { //Dedicated buffers: One buffer to read and one buffer for output
throw new HyracksDataException("not enough memory for Hybrid Hash Join");
}
state.memForJoin = memsize - 2;
- state.numOfPartitions = getNumberOfPartitions(state.memForJoin, inputsize0, fudgeFactor, nPartitions);
+ state.numOfPartitions = getNumberOfPartitions(state.memForJoin, inputsize0, fudgeFactor,
+ nPartitions);
state.hybridHJ = new OptimizedHybridHashJoin(ctx, state.memForJoin, state.numOfPartitions,
PROBE_REL, BUILD_REL, probeKeys, buildKeys, comparators, probeRd, buildRd, probeHpc,
buildHpc);
@@ -278,7 +290,7 @@
@Override
public void close() throws HyracksDataException {
state.hybridHJ.closeBuild();
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@Override
@@ -289,6 +301,7 @@
return op;
}
}
+
/*
* Probe phase of Hybrid Hash Join:
* Reading the probe side and partitioning it, resident tuples get
@@ -300,14 +313,13 @@
private class ProbeAndJoinActivityNode extends AbstractActivityNode {
private static final long serialVersionUID = 1L;
-
public ProbeAndJoinActivityNode(ActivityId id) {
super(id);
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, final int nPartitions) {
final RecordDescriptor probeRd = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
@@ -331,17 +343,17 @@
private BuildAndPartitionTaskState state;
private ByteBuffer rPartbuff = ctx.allocateFrame();
- private ITuplePartitionComputerGeneratorFactory hpcf0 = new FieldHashPartitionComputerGeneratorFactory(
- probeKeys, hashFunctionGeneratorFactories);
- private ITuplePartitionComputerGeneratorFactory hpcf1 = new FieldHashPartitionComputerGeneratorFactory(
- buildKeys, hashFunctionGeneratorFactories);
+ private ITuplePartitionComputerFamily hpcf0 = new FieldHashPartitionComputerFamily(probeKeys,
+ hashFunctionGeneratorFactories);
+ private ITuplePartitionComputerFamily hpcf1 = new FieldHashPartitionComputerFamily(buildKeys,
+ hashFunctionGeneratorFactories);
private ITuplePartitionComputer hpcRep0;
private ITuplePartitionComputer hpcRep1;
@Override
public void open() throws HyracksDataException {
- state = (BuildAndPartitionTaskState) env.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ state = (BuildAndPartitionTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
BUILD_AND_PARTITION_ACTIVITY_ID), partition));
writer.open();
@@ -366,9 +378,9 @@
BitSet partitionStatus = state.hybridHJ.getPartitinStatus();
hpcRep0 = new RepartitionComputerGeneratorFactory(state.numOfPartitions, hpcf0)
- .createPartitioner(0);
+ .createPartitioner(0);
hpcRep1 = new RepartitionComputerGeneratorFactory(state.numOfPartitions, hpcf1)
- .createPartitioner(0);
+ .createPartitioner(0);
rPartbuff.clear();
for (int pid = partitionStatus.nextSetBit(0); pid >= 0; pid = partitionStatus.nextSetBit(pid + 1)) {
@@ -376,7 +388,7 @@
RunFileReader bReader = state.hybridHJ.getBuildRFReader(pid);
RunFileReader pReader = state.hybridHJ.getProbeRFReader(pid);
- if (bReader == null || pReader == null) { //either of sides (or both) does not have any tuple, thus no need for joining (no potential match)
+ if (bReader == null || pReader == null) { //either of sides (or both) does not have any tuple, thus no need for joining (no potential match)
continue;
}
int bSize = state.hybridHJ.getBuildPartitionSizeInTup(pid);
@@ -390,42 +402,45 @@
private void joinPartitionPair(OptimizedHybridHashJoin ohhj, RunFileReader buildSideReader,
RunFileReader probeSideReader, int pid, int beforeMax, int level) throws HyracksDataException {
- ITuplePartitionComputer probeHpc = new FieldHashPartitionComputerGeneratorFactory(probeKeys,
+ ITuplePartitionComputer probeHpc = new FieldHashPartitionComputerFamily(probeKeys,
hashFunctionGeneratorFactories).createPartitioner(level);
- ITuplePartitionComputer buildHpc = new FieldHashPartitionComputerGeneratorFactory(buildKeys,
+ ITuplePartitionComputer buildHpc = new FieldHashPartitionComputerFamily(buildKeys,
hashFunctionGeneratorFactories).createPartitioner(level);
long buildPartSize = ohhj.getBuildPartitionSize(pid) / ctx.getFrameSize();
long probePartSize = ohhj.getProbePartitionSize(pid) / ctx.getFrameSize();
- //Apply in-Mem HJ if possible
+ //Apply in-Mem HJ if possible
if ((buildPartSize < state.memForJoin) || (probePartSize < state.memForJoin)) {
int tabSize = -1;
if (buildPartSize < probePartSize) {
tabSize = ohhj.getBuildPartitionSizeInTup(pid);
- if(tabSize == 0){
- throw new HyracksDataException("Trying to join an empty partition. Invalid table size for inMemoryHashJoin.");
+ if (tabSize == 0) {
+ throw new HyracksDataException(
+ "Trying to join an empty partition. Invalid table size for inMemoryHashJoin.");
}
//Build Side is smaller
applyInMemHashJoin(probeKeys, buildKeys, tabSize, probeRd, buildRd, hpcRep1, hpcRep0,
buildSideReader, probeSideReader);
-
+
} else { //Role Reversal
tabSize = ohhj.getProbePartitionSizeInTup(pid);
- if(tabSize == 0){
- throw new HyracksDataException("Trying to join an empty partition. Invalid table size for inMemoryHashJoin.");
+ if (tabSize == 0) {
+ throw new HyracksDataException(
+ "Trying to join an empty partition. Invalid table size for inMemoryHashJoin.");
}
//Probe Side is smaller
applyInMemHashJoin(buildKeys, probeKeys, tabSize, buildRd, probeRd, hpcRep0, hpcRep1,
probeSideReader, buildSideReader);
}
}
- //Apply (Recursive) HHJ
+ //Apply (Recursive) HHJ
else {
OptimizedHybridHashJoin rHHj;
if (buildPartSize < probePartSize) { //Build Side is smaller
- int n = getNumberOfPartitions(state.memForJoin, (int) buildPartSize, fudgeFactor, nPartitions);
+ int n = getNumberOfPartitions(state.memForJoin, (int) buildPartSize, fudgeFactor,
+ nPartitions);
rHHj = new OptimizedHybridHashJoin(ctx, state.memForJoin, n, PROBE_REL, BUILD_REL,
probeKeys, buildKeys, comparators, probeRd, buildRd, probeHpc, buildHpc);
@@ -485,7 +500,8 @@
}
}
} else { //Role Reversal (Probe Side is smaller)
- int n = getNumberOfPartitions(state.memForJoin, (int) probePartSize, fudgeFactor, nPartitions);
+ int n = getNumberOfPartitions(state.memForJoin, (int) probePartSize, fudgeFactor,
+ nPartitions);
rHHj = new OptimizedHybridHashJoin(ctx, state.memForJoin, n, BUILD_REL, PROBE_REL,
buildKeys, probeKeys, comparators, buildRd, probeRd, buildHpc, probeHpc);
@@ -560,7 +576,7 @@
bReader.open();
rPartbuff.clear();
while (bReader.nextFrame(rPartbuff)) {
- ByteBuffer copyBuffer = ctx.allocateFrame(); //We need to allocate a copyBuffer, because this buffer gets added to the buffers list in the InMemoryHashJoin
+ ByteBuffer copyBuffer = ctx.allocateFrame(); //We need to allocate a copyBuffer, because this buffer gets added to the buffers list in the InMemoryHashJoin
FrameUtils.copy(rPartbuff, copyBuffer);
FrameUtils.makeReadable(copyBuffer);
joiner.build(copyBuffer);
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/map/DeserializedMapperOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/map/DeserializedMapperOperatorDescriptor.java
index 3ed7c07..84caadf 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/map/DeserializedMapperOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/map/DeserializedMapperOperatorDescriptor.java
@@ -20,7 +20,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
@@ -73,7 +72,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
return new DeserializedOperatorNodePushable(ctx, new MapperOperator(),
recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java
index 48e53b5..85a6912 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/ConstantTupleSourceOperatorDescriptor.java
@@ -19,7 +19,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
@@ -41,7 +40,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
return new ConstantTupleSourceOperatorNodePushable(ctx, fieldSlots, tupleData, tupleSize);
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/LimitOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/LimitOperatorDescriptor.java
index 939935f..5d1d559 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/LimitOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/LimitOperatorDescriptor.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package edu.uci.ics.hyracks.dataflow.std.misc;
import java.nio.ByteBuffer;
@@ -7,7 +21,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
@@ -15,75 +28,69 @@
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
-public class LimitOperatorDescriptor extends
- AbstractSingleActivityOperatorDescriptor {
- private static final long serialVersionUID = 1L;
- private final int outputLimit;
+public class LimitOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+ private final int outputLimit;
- public LimitOperatorDescriptor(JobSpecification spec,
- RecordDescriptor rDesc, int outputLimit) {
- super(spec, 1, 1);
- recordDescriptors[0] = rDesc;
- this.outputLimit = outputLimit;
- }
+ public LimitOperatorDescriptor(JobSpecification spec, RecordDescriptor rDesc, int outputLimit) {
+ super(spec, 1, 1);
+ recordDescriptors[0] = rDesc;
+ this.outputLimit = outputLimit;
+ }
- @Override
- public IOperatorNodePushable createPushRuntime(
- final IHyracksTaskContext ctx, IOperatorEnvironment env,
- final IRecordDescriptorProvider recordDescProvider, int partition,
- int nPartitions) throws HyracksDataException {
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions)
+ throws HyracksDataException {
- return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
- private FrameTupleAccessor fta;
- private int currentSize;
- private boolean finished;
+ return new AbstractUnaryInputUnaryOutputOperatorNodePushable() {
+ private FrameTupleAccessor fta;
+ private int currentSize;
+ private boolean finished;
- @Override
- public void open() throws HyracksDataException {
- fta = new FrameTupleAccessor(ctx.getFrameSize(),
- recordDescriptors[0]);
- currentSize = 0;
- finished = false;
- writer.open();
- }
+ @Override
+ public void open() throws HyracksDataException {
+ fta = new FrameTupleAccessor(ctx.getFrameSize(), recordDescriptors[0]);
+ currentSize = 0;
+ finished = false;
+ writer.open();
+ }
- @Override
- public void nextFrame(ByteBuffer buffer)
- throws HyracksDataException {
- if (!finished) {
- fta.reset(buffer);
- int count = fta.getTupleCount();
- if ((currentSize + count) > outputLimit) {
- ByteBuffer b = ctx.allocateFrame();
- FrameTupleAppender partialAppender = new FrameTupleAppender(
- ctx.getFrameSize());
- partialAppender.reset(b, true);
- int copyCount = outputLimit - currentSize;
- for (int i = 0; i < copyCount; i++) {
- partialAppender.append(fta, i);
- currentSize++;
- }
- FrameUtils.makeReadable(b);
- FrameUtils.flushFrame(b, writer);
- finished = true;
- } else {
- FrameUtils.flushFrame(buffer, writer);
- currentSize += count;
- }
- }
- }
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ if (!finished) {
+ fta.reset(buffer);
+ int count = fta.getTupleCount();
+ if ((currentSize + count) > outputLimit) {
+ ByteBuffer b = ctx.allocateFrame();
+ FrameTupleAppender partialAppender = new FrameTupleAppender(ctx.getFrameSize());
+ partialAppender.reset(b, true);
+ int copyCount = outputLimit - currentSize;
+ for (int i = 0; i < copyCount; i++) {
+ partialAppender.append(fta, i);
+ currentSize++;
+ }
+ FrameUtils.makeReadable(b);
+ FrameUtils.flushFrame(b, writer);
+ finished = true;
+ } else {
+ FrameUtils.flushFrame(buffer, writer);
+ currentSize += count;
+ }
+ }
+ }
- @Override
- public void fail() throws HyracksDataException {
- writer.fail();
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
- }
+ }
- @Override
- public void close() throws HyracksDataException {
- writer.close();
- }
- };
- }
+ @Override
+ public void close() throws HyracksDataException {
+ writer.close();
+ }
+ };
+ }
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java
index 32bfc09..83160e5 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/MaterializingOperatorDescriptor.java
@@ -28,7 +28,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.io.RunFileReader;
@@ -93,7 +92,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
return new AbstractUnaryInputSinkOperatorNodePushable() {
private MaterializerTaskState state;
@@ -116,7 +115,7 @@
@Override
public void close() throws HyracksDataException {
state.out.close();
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@Override
@@ -134,13 +133,13 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
return new AbstractUnaryOutputSourceOperatorNodePushable() {
@Override
public void initialize() throws HyracksDataException {
ByteBuffer frame = ctx.allocateFrame();
- MaterializerTaskState state = (MaterializerTaskState) env.getTaskState(new TaskId(new ActivityId(
+ MaterializerTaskState state = (MaterializerTaskState) ctx.getTaskState(new TaskId(new ActivityId(
getOperatorId(), MATERIALIZER_ACTIVITY_ID), partition));
RunFileReader in = state.out.createReader();
writer.open();
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/NullSinkOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/NullSinkOperatorDescriptor.java
index 80b2212..8e0ac79 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/NullSinkOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/NullSinkOperatorDescriptor.java
@@ -20,7 +20,6 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
@@ -33,7 +32,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
return new AbstractUnaryInputSinkOperatorNodePushable() {
@Override
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/PrinterOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/PrinterOperatorDescriptor.java
index d937784..7effbb0 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/PrinterOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/PrinterOperatorDescriptor.java
@@ -19,7 +19,6 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.IOpenableDataWriterOperator;
@@ -62,7 +61,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
return new DeserializedOperatorNodePushable(ctx, new PrinterOperator(),
recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0));
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
index 8c83e3a..6874f77 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitOperatorDescriptor.java
@@ -8,7 +8,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
@@ -25,7 +24,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
final IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions)
throws HyracksDataException {
return new AbstractUnaryInputOperatorNodePushable() {
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitVectorOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitVectorOperatorDescriptor.java
index 351c8a6..3e99000 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitVectorOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/misc/SplitVectorOperatorDescriptor.java
@@ -28,7 +28,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
@@ -73,7 +72,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
IOpenableDataWriterOperator op = new IOpenableDataWriterOperator() {
private CollectTaskState state;
@@ -92,7 +91,7 @@
@Override
public void close() throws HyracksDataException {
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@Override
@@ -118,7 +117,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
IOpenableDataWriterOperator op = new IOpenableDataWriterOperator() {
private IOpenableDataWriter<Object[]> writer;
@@ -135,7 +134,7 @@
@Override
public void open() throws HyracksDataException {
- state = (CollectTaskState) env.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ state = (CollectTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
COLLECT_ACTIVITY_ID), partition));
}
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTMemMgr.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTMemMgr.java
index eda567a..d07ba51 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTMemMgr.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTMemMgr.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package edu.uci.ics.hyracks.dataflow.std.sort;
import java.nio.ByteBuffer;
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTNodeUtil.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTNodeUtil.java
index 3a1a629..2b9f7c9 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTNodeUtil.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/BSTNodeUtil.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package edu.uci.ics.hyracks.dataflow.std.sort;
import java.nio.ByteBuffer;
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
index 073ff28..0effefb 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/ExternalSortOperatorDescriptor.java
@@ -32,7 +32,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
@@ -115,7 +114,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
IOperatorNodePushable op = new AbstractUnaryInputSinkOperatorNodePushable() {
private ExternalSortRunGenerator runGen;
@@ -139,7 +138,7 @@
runGen.close();
state.runs = runGen.getRuns();
state.frameSorter = runGen.getFrameSorter();
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@Override
@@ -159,12 +158,12 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
IOperatorNodePushable op = new AbstractUnaryOutputSourceOperatorNodePushable() {
@Override
public void initialize() throws HyracksDataException {
- SortTaskState state = (SortTaskState) env.getTaskState(new TaskId(new ActivityId(getOperatorId(),
+ SortTaskState state = (SortTaskState) ctx.getTaskState(new TaskId(new ActivityId(getOperatorId(),
SORT_ACTIVITY_ID), partition));
List<IFrameReader> runs = state.runs;
FrameSorter frameSorter = state.frameSorter;
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/InMemorySortOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/InMemorySortOperatorDescriptor.java
index eafe738..30bebe9 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/InMemorySortOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/InMemorySortOperatorDescriptor.java
@@ -29,7 +29,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
@@ -104,7 +103,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
IOperatorNodePushable op = new AbstractUnaryInputSinkOperatorNodePushable() {
private SortTaskState state;
@@ -125,7 +124,7 @@
@Override
public void close() throws HyracksDataException {
state.frameSorter.sortFrames();
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@Override
@@ -144,14 +143,14 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
IOperatorNodePushable op = new AbstractUnaryOutputSourceOperatorNodePushable() {
@Override
public void initialize() throws HyracksDataException {
writer.open();
try {
- SortTaskState state = (SortTaskState) env.getTaskState(new TaskId(new ActivityId(
+ SortTaskState state = (SortTaskState) ctx.getTaskState(new TaskId(new ActivityId(
getOperatorId(), SORT_ACTIVITY_ID), partition));
state.frameSorter.flushFrames(writer);
} catch (Exception e) {
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortOperatorDescriptor.java
index 6b73c54..a8653d1 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortOperatorDescriptor.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package edu.uci.ics.hyracks.dataflow.std.sort;
import java.io.DataInput;
@@ -18,7 +32,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
@@ -123,7 +136,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
final IRunGenerator runGen;
if (outputLimit == NO_LIMIT) {
@@ -152,7 +165,7 @@
new TaskId(getActivityId(), partition));
runGen.close();
state.runs = runGen.getRuns();
- env.setTaskState(state);
+ ctx.setTaskState(state);
}
@@ -173,12 +186,12 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, final int partition, int nPartitions) {
IOperatorNodePushable op = new AbstractUnaryOutputSourceOperatorNodePushable() {
@Override
public void initialize() throws HyracksDataException {
- OptimizedSortTaskState state = (OptimizedSortTaskState) env.getTaskState(new TaskId(new ActivityId(
+ OptimizedSortTaskState state = (OptimizedSortTaskState) ctx.getTaskState(new TaskId(new ActivityId(
getOperatorId(), SORT_ACTIVITY_ID), partition));
List<IFrameReader> runs = state.runs;
@@ -187,7 +200,7 @@
for (int i = 0; i < comparatorFactories.length; ++i) {
comparators[i] = comparatorFactories[i].createBinaryComparator();
}
-
+
int necessaryFrames = Math.min(runs.size() + 2, memSize);
ExternalSortRunMerger merger = new ExternalSortRunMerger(ctx, outputLimit, runs, sortFields,
comparators, recordDescriptors[0], necessaryFrames, writer);
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortRunGenerator.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortRunGenerator.java
index 8e1e3a7..f01d886 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortRunGenerator.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortRunGenerator.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package edu.uci.ics.hyracks.dataflow.std.sort;
import java.nio.ByteBuffer;
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortRunGeneratorWithLimit.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortRunGeneratorWithLimit.java
index 3ccba02..25442b3 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortRunGeneratorWithLimit.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/sort/OptimizedExternalSortRunGeneratorWithLimit.java
@@ -1,3 +1,17 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package edu.uci.ics.hyracks.dataflow.std.sort;
import java.nio.ByteBuffer;
diff --git a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/union/UnionAllOperatorDescriptor.java b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/union/UnionAllOperatorDescriptor.java
index 373ed48..f55eb3d 100644
--- a/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/union/UnionAllOperatorDescriptor.java
+++ b/hyracks-dataflow-std/src/main/java/edu/uci/ics/hyracks/dataflow/std/union/UnionAllOperatorDescriptor.java
@@ -24,7 +24,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractActivityNode;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
@@ -56,7 +55,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions)
throws HyracksDataException {
RecordDescriptor inRecordDesc = recordDescProvider.getInputRecordDescriptor(getOperatorId(), 0);
diff --git a/hyracks-dataflow-std/src/test/java/edu/uci/ics/hyracks/dataflow/std/test/util/SelectionTreeTest.java b/hyracks-dataflow-std/src/test/java/edu/uci/ics/hyracks/dataflow/std/test/util/SelectionTreeTest.java
index dc5b599..8f467a0 100644
--- a/hyracks-dataflow-std/src/test/java/edu/uci/ics/hyracks/dataflow/std/test/util/SelectionTreeTest.java
+++ b/hyracks-dataflow-std/src/test/java/edu/uci/ics/hyracks/dataflow/std/test/util/SelectionTreeTest.java
@@ -33,7 +33,6 @@
int last = Integer.MIN_VALUE;
while ((e = tree.peek()) != null) {
MergeEntry me = (MergeEntry) e;
- System.err.print(me.i + " ");
if (me.i < last) {
Assert.fail();
}
@@ -67,4 +66,4 @@
return true;
}
}
-}
+}
\ No newline at end of file
diff --git a/hyracks-documentation/.classpath b/hyracks-documentation/.classpath
deleted file mode 100644
index d0bec0f..0000000
--- a/hyracks-documentation/.classpath
+++ /dev/null
@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-documentation/.project b/hyracks-documentation/.project
deleted file mode 100644
index 64d4505..0000000
--- a/hyracks-documentation/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-documentation</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-documentation/.settings/org.eclipse.jdt.core.prefs b/hyracks-documentation/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index d368ea9..0000000
--- a/hyracks-documentation/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Thu Dec 15 06:58:53 PST 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
-org.eclipse.jdt.core.compiler.compliance=1.5
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.5
diff --git a/hyracks-documentation/.settings/org.maven.ide.eclipse.prefs b/hyracks-documentation/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 1b662c1..0000000
--- a/hyracks-documentation/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Oct 19 13:07:01 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-documentation/pom.xml b/hyracks-documentation/pom.xml
index 522011a..47d997b 100644
--- a/hyracks-documentation/pom.xml
+++ b/hyracks-documentation/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-documentation</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
diff --git a/hyracks-examples/.project b/hyracks-examples/.project
deleted file mode 100644
index 7fb5a5d..0000000
--- a/hyracks-examples/.project
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-examples</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 1b2848c..0000000
--- a/hyracks-examples/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Sun Aug 29 19:38:09 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/btree-example/.project b/hyracks-examples/btree-example/.project
deleted file mode 100644
index ef68a5a..0000000
--- a/hyracks-examples/btree-example/.project
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>btree-example</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/btree-example/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/btree-example/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 6733f8d..0000000
--- a/hyracks-examples/btree-example/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Wed Oct 06 08:06:48 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/btree-example/btreeapp/.classpath b/hyracks-examples/btree-example/btreeapp/.classpath
deleted file mode 100644
index d0bec0f..0000000
--- a/hyracks-examples/btree-example/btreeapp/.classpath
+++ /dev/null
@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/btree-example/btreeapp/.project b/hyracks-examples/btree-example/btreeapp/.project
deleted file mode 100644
index 897fb85..0000000
--- a/hyracks-examples/btree-example/btreeapp/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>btreeapp</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/btree-example/btreeapp/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/btree-example/btreeapp/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 0526f68..0000000
--- a/hyracks-examples/btree-example/btreeapp/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Thu Dec 15 06:58:55 PST 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
-org.eclipse.jdt.core.compiler.compliance=1.5
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.5
diff --git a/hyracks-examples/btree-example/btreeapp/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/btree-example/btreeapp/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 6733f8d..0000000
--- a/hyracks-examples/btree-example/btreeapp/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Wed Oct 06 08:06:48 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/btree-example/btreeapp/pom.xml b/hyracks-examples/btree-example/btreeapp/pom.xml
index e3f0590..d5239bd 100644
--- a/hyracks-examples/btree-example/btreeapp/pom.xml
+++ b/hyracks-examples/btree-example/btreeapp/pom.xml
@@ -2,15 +2,44 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.btree</groupId>
<artifactId>btreeapp</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>btree-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <versionRange>[1.0.0,)</versionRange>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -51,7 +80,7 @@
<dependency>
<groupId>edu.uci.ics.hyracks.examples.btree</groupId>
<artifactId>btreehelper</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
diff --git a/hyracks-examples/btree-example/btreeclient/.classpath b/hyracks-examples/btree-example/btreeclient/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-examples/btree-example/btreeclient/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/btree-example/btreeclient/.project b/hyracks-examples/btree-example/btreeclient/.project
deleted file mode 100644
index 8e16966..0000000
--- a/hyracks-examples/btree-example/btreeclient/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>btreeclient</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/btree-example/btreeclient/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/btree-example/btreeclient/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 7cf8ad6..0000000
--- a/hyracks-examples/btree-example/btreeclient/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,264 +0,0 @@
-#Fri May 20 19:34:07 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=48
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=48
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=true
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true
-org.eclipse.jdt.core.formatter.comment.format_line_comments=true
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=true
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_member=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=120
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
diff --git a/hyracks-examples/btree-example/btreeclient/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/btree-example/btreeclient/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 6733f8d..0000000
--- a/hyracks-examples/btree-example/btreeclient/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Wed Oct 06 08:06:48 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/btree-example/btreeclient/pom.xml b/hyracks-examples/btree-example/btreeclient/pom.xml
index dea042a..e0930b2 100644
--- a/hyracks-examples/btree-example/btreeclient/pom.xml
+++ b/hyracks-examples/btree-example/btreeclient/pom.xml
@@ -2,31 +2,29 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.btree</groupId>
<artifactId>btreeclient</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>btree-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks.examples.btree</groupId>
<artifactId>btreehelper</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
index 05b16c8..5a6d8cd 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
@@ -18,21 +18,21 @@
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
@@ -41,15 +41,15 @@
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
import edu.uci.ics.hyracks.examples.btree.helper.DataGenOperatorDescriptor;
+import edu.uci.ics.hyracks.examples.btree.helper.IndexRegistryProvider;
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
-import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexInsertUpdateDeleteOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
@@ -63,7 +63,7 @@
public String host;
@Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
- public int port = 1099;
+ public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
public String app;
@@ -86,7 +86,7 @@
CmdLineParser parser = new CmdLineParser(options);
parser.parseArgument(args);
- IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+ IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
JobSpecification job = createJob(options);
@@ -128,17 +128,21 @@
// run data generator on first nodecontroller given
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, dataGen, splitNCs[0]);
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = TreeIndexRegistryProvider.INSTANCE;
+ IIndexRegistryProvider<IIndex> indexRegistryProvider = IndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
// prepare insertion into primary index
// tuples to be put into B-Tree shall have 4 fields
int primaryFieldCount = 4;
- ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- primaryTypeTraits[0] = new TypeTrait(4);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(4);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ primaryTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
+
+ // comparator factories for primary index
+ IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[1];
+ primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
@@ -149,27 +153,29 @@
// tuple
int[] primaryFieldPermutation = { 2, 1, 3, 4 }; // map field 2 of input
// tuple to field 0 of
- // B-Tree tuple, etc.
- // comparator factories for primary index
- IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[1];
- primaryComparatorFactories[0] = IntegerBinaryComparatorFactory.INSTANCE;
+ // B-Tree tuple, etc.
IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
- ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
// create operator descriptor
TreeIndexInsertUpdateDeleteOperatorDescriptor primaryInsert = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
- spec, recDesc, storageManager, treeIndexRegistryProvider, primarySplitProvider,
+ spec, recDesc, storageManager, indexRegistryProvider, primarySplitProvider,
primaryInteriorFrameFactory, primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories,
- null, primaryFieldPermutation, IndexOp.INSERT, opHelperFactory);
+ primaryFieldPermutation, IndexOp.INSERT, dataflowHelperFactory);
JobHelper.createPartitionConstraint(spec, primaryInsert, splitNCs);
// prepare insertion into secondary index
// tuples to be put into B-Tree shall have 2 fields
int secondaryFieldCount = 2;
- ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
- secondaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryTypeTraits[1] = new TypeTrait(4);
+ ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount];
+ secondaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ secondaryTypeTraits[1] = IntegerPointable.TYPE_TRAITS;
+
+ // comparator factories for secondary index
+ IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[2];
+ secondaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ secondaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
@@ -180,17 +186,13 @@
// the B-Tree expects its keyfields to be at the front of its input
// tuple
int[] secondaryFieldPermutation = { 1, 2 };
- // comparator factories for primary index
- IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[2];
- secondaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
- secondaryComparatorFactories[1] = IntegerBinaryComparatorFactory.INSTANCE;
IFileSplitProvider secondarySplitProvider = JobHelper.createFileSplitProvider(splitNCs,
options.secondaryBTreeName);
// create operator descriptor
TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsert = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
- spec, recDesc, storageManager, treeIndexRegistryProvider, secondarySplitProvider,
+ spec, recDesc, storageManager, indexRegistryProvider, secondarySplitProvider,
secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, null, secondaryFieldPermutation, IndexOp.INSERT, opHelperFactory);
+ secondaryComparatorFactories, secondaryFieldPermutation, IndexOp.INSERT, dataflowHelperFactory);
JobHelper.createPartitionConstraint(spec, secondaryInsert, splitNCs);
// end the insert pipeline at this sink operator
@@ -200,7 +202,7 @@
// distribute the records from the datagen via hashing to the bulk load
// ops
IBinaryHashFunctionFactory[] hashFactories = new IBinaryHashFunctionFactory[1];
- hashFactories[0] = UTF8StringBinaryHashFunctionFactory.INSTANCE;
+ hashFactories[0] = PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY);
IConnectorDescriptor hashConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 }, hashFactories));
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
index 8019080..0b1cd6c 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
@@ -17,20 +17,21 @@
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
@@ -39,15 +40,15 @@
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.examples.btree.helper.DataGenOperatorDescriptor;
+import edu.uci.ics.hyracks.examples.btree.helper.IndexRegistryProvider;
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
-import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -60,7 +61,7 @@
public String host;
@Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
- public int port = 1099;
+ public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
public String app;
@@ -83,7 +84,7 @@
CmdLineParser parser = new CmdLineParser(options);
parser.parseArgument(args);
- IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+ IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
JobSpecification job = createJob(options);
@@ -130,24 +131,24 @@
int[] sortFields = { 2 };
// comparators for sort fields
IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[1];
- comparatorFactories[0] = IntegerBinaryComparatorFactory.INSTANCE;
+ comparatorFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, options.sbSize, sortFields,
comparatorFactories, recDesc);
JobHelper.createPartitionConstraint(spec, sorter, splitNCs);
// tuples to be put into B-Tree shall have 4 fields
int fieldCount = 4;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- typeTraits[2] = new TypeTrait(4);
- typeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
// create factories and providers for B-Tree
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = TreeIndexRegistryProvider.INSTANCE;
+ IIndexRegistryProvider<IIndex> indexRegistryProvider = IndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
// the B-Tree expects its keyfields to be at the front of its input
@@ -156,16 +157,16 @@
// to field 0 of B-Tree tuple,
// etc.
IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
- ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, treeIndexRegistryProvider, btreeSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, comparatorFactories, null, fieldPermutation, 0.7f, opHelperFactory);
+ storageManager, indexRegistryProvider, btreeSplitProvider, interiorFrameFactory, leafFrameFactory,
+ typeTraits, comparatorFactories, fieldPermutation, 0.7f, dataflowHelperFactory);
JobHelper.createPartitionConstraint(spec, btreeBulkLoad, splitNCs);
// distribute the records from the datagen via hashing to the bulk load
// ops
IBinaryHashFunctionFactory[] hashFactories = new IBinaryHashFunctionFactory[1];
- hashFactories[0] = UTF8StringBinaryHashFunctionFactory.INSTANCE;
+ hashFactories[0] = PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY);
IConnectorDescriptor hashConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 }, hashFactories));
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexEnlistFilesExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexEnlistFilesExample.java
deleted file mode 100644
index b75f64d..0000000
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexEnlistFilesExample.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.examples.btree.client;
-
-import org.kohsuke.args4j.CmdLineParser;
-import org.kohsuke.args4j.Option;
-
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
-import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
-import edu.uci.ics.hyracks.api.job.JobId;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
-import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexFileEnlistmentOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-
-// This example will enlist existing files as primary index
-
-public class PrimaryIndexEnlistFilesExample {
- private static class Options {
- @Option(name = "-host", usage = "Hyracks Cluster Controller Host name", required = true)
- public String host;
-
- @Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
- public int port = 1099;
-
- @Option(name = "-app", usage = "Hyracks Application name", required = true)
- public String app;
-
- @Option(name = "-target-ncs", usage = "Comma separated list of node-controller names to use", required = true)
- public String ncs;
-
- @Option(name = "-btreename", usage = "B-Tree file name", required = true)
- public String btreeName;
- }
-
- public static void main(String[] args) throws Exception {
- Options options = new Options();
- CmdLineParser parser = new CmdLineParser(options);
- parser.parseArgument(args);
-
- IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
-
- JobSpecification job = createJob(options);
-
- long start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job);
- hcc.start(jobId);
- hcc.waitForCompletion(jobId);
- long end = System.currentTimeMillis();
- System.err.println(start + " " + end + " " + (end - start));
- }
-
- private static JobSpecification createJob(Options options) {
-
- JobSpecification spec = new JobSpecification();
-
- String[] splitNCs = options.ncs.split(",");
-
- // schema of tuples in existing files (see PrimaryIndexBulkLoadExample)
- RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
-
- int fieldCount = 4;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- typeTraits[2] = new TypeTrait(4);
- typeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
-
- // create factories and providers for B-Tree
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = TreeIndexRegistryProvider.INSTANCE;
- IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
-
- IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[1];
- comparatorFactories[0] = IntegerBinaryComparatorFactory.INSTANCE;
-
- IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
- ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
- TreeIndexFileEnlistmentOperatorDescriptor fileEnlistmentOp = new TreeIndexFileEnlistmentOperatorDescriptor(
- spec, recDesc, storageManager, treeIndexRegistryProvider, btreeSplitProvider, interiorFrameFactory,
- leafFrameFactory, typeTraits, comparatorFactories, null, opHelperFactory);
- JobHelper.createPartitionConstraint(spec, fileEnlistmentOp, splitNCs);
-
- spec.addRoot(fileEnlistmentOp);
-
- return spec;
- }
-}
\ No newline at end of file
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
index fbf7f9d..aa73a62 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
@@ -19,34 +19,35 @@
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.examples.btree.helper.IndexRegistryProvider;
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
-import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -59,7 +60,7 @@
public String host;
@Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
- public int port = 1099;
+ public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
public String app;
@@ -76,7 +77,7 @@
CmdLineParser parser = new CmdLineParser(options);
parser.parseArgument(args);
- IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+ IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
JobSpecification job = createJob(options);
@@ -95,17 +96,21 @@
String[] splitNCs = options.ncs.split(",");
int fieldCount = 4;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- typeTraits[2] = new TypeTrait(4);
- typeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
+
+ // comparators for btree
+ IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[1];
+ comparatorFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// create factories and providers for B-Tree
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = TreeIndexRegistryProvider.INSTANCE;
+ IIndexRegistryProvider<IIndex> indexRegistryProvider = IndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
// schema of tuples coming out of primary index
@@ -113,10 +118,6 @@
IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE, });
- // comparators for btree
- IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[1];
- comparatorFactories[0] = IntegerBinaryComparatorFactory.INSTANCE;
-
// build tuple containing low and high search keys
ArrayTupleBuilder tb = new ArrayTupleBuilder(comparatorFactories.length * 2); // high
// key
@@ -146,10 +147,10 @@
// into search op
IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.btreeName);
- ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
BTreeSearchOperatorDescriptor btreeSearchOp = new BTreeSearchOperatorDescriptor(spec, recDesc, storageManager,
- treeIndexRegistryProvider, btreeSplitProvider, interiorFrameFactory, leafFrameFactory, typeTraits,
- comparatorFactories, true, lowKeyFields, highKeyFields, true, true, opHelperFactory);
+ indexRegistryProvider, btreeSplitProvider, interiorFrameFactory, leafFrameFactory, typeTraits,
+ comparatorFactories, true, lowKeyFields, highKeyFields, true, true, dataflowHelperFactory);
JobHelper.createPartitionConstraint(spec, btreeSearchOp, splitNCs);
// have each node print the results of its respective B-Tree
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
index 21c00a1..ad9ad3a 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
@@ -17,31 +17,31 @@
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+import edu.uci.ics.hyracks.examples.btree.helper.IndexRegistryProvider;
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
-import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDiskOrderScanOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
@@ -56,7 +56,7 @@
public String host;
@Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
- public int port = 1099;
+ public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
public String app;
@@ -79,7 +79,7 @@
CmdLineParser parser = new CmdLineParser(options);
parser.parseArgument(args);
- IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+ IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
JobSpecification job = createJob(options);
@@ -97,7 +97,7 @@
String[] splitNCs = options.ncs.split(",");
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = TreeIndexRegistryProvider.INSTANCE;
+ IIndexRegistryProvider<IIndex> indexRegistryProvider = IndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
// schema of tuples that we are retrieving from the primary index
@@ -111,11 +111,16 @@
IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
int primaryFieldCount = 4;
- ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- primaryTypeTraits[0] = new TypeTrait(4);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(4);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ primaryTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
+
+ // comparators for sort fields and BTree fields
+ IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[2];
+ comparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ comparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// create factories and providers for primary B-Tree
TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
@@ -124,28 +129,24 @@
// use a disk-order scan to read primary index
IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
- ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
TreeIndexDiskOrderScanOperatorDescriptor btreeScanOp = new TreeIndexDiskOrderScanOperatorDescriptor(spec,
- recDesc, storageManager, treeIndexRegistryProvider, primarySplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, opHelperFactory);
+ recDesc, storageManager, indexRegistryProvider, primarySplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, dataflowHelperFactory);
JobHelper.createPartitionConstraint(spec, btreeScanOp, splitNCs);
// sort the tuples as preparation for bulk load into secondary index
// fields to sort on
int[] sortFields = { 1, 0 };
- // comparators for sort fields
- IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[2];
- comparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
- comparatorFactories[1] = IntegerBinaryComparatorFactory.INSTANCE;
ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, options.sbSize, sortFields,
comparatorFactories, recDesc);
JobHelper.createPartitionConstraint(spec, sorter, splitNCs);
// tuples to be put into B-Tree shall have 2 fields
int secondaryFieldCount = 2;
- ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
- secondaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryTypeTraits[1] = new TypeTrait(4);
+ ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount];
+ secondaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ secondaryTypeTraits[1] = IntegerPointable.TYPE_TRAITS;
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
@@ -158,9 +159,9 @@
int[] fieldPermutation = { 1, 0 };
IFileSplitProvider btreeSplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.secondaryBTreeName);
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, treeIndexRegistryProvider, btreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, comparatorFactories, null, fieldPermutation, 0.7f,
- opHelperFactory);
+ storageManager, indexRegistryProvider, btreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits, comparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
JobHelper.createPartitionConstraint(spec, btreeBulkLoad, splitNCs);
// connect the ops
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
index 5eec60c..2719397 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
@@ -19,34 +19,35 @@
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.examples.btree.helper.IndexRegistryProvider;
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
-import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -59,7 +60,7 @@
public String host;
@Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
- public int port = 1099;
+ public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
public String app;
@@ -79,7 +80,7 @@
CmdLineParser parser = new CmdLineParser(options);
parser.parseArgument(args);
- IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+ IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
JobSpecification job = createJob(options);
@@ -97,7 +98,7 @@
String[] splitNCs = options.ncs.split(",");
- IIndexRegistryProvider<ITreeIndex> btreeRegistryProvider = TreeIndexRegistryProvider.INSTANCE;
+ IIndexRegistryProvider<IIndex> indexRegistryProvider = IndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
// schema of tuples coming out of secondary index
@@ -105,9 +106,18 @@
UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
int secondaryFieldCount = 2;
- ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
- secondaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryTypeTraits[1] = new TypeTrait(4);
+ ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount];
+ secondaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ secondaryTypeTraits[1] = IntegerPointable.TYPE_TRAITS;
+
+ // comparators for sort fields and BTree fields
+ IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[2];
+ secondaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ secondaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // comparators for primary index
+ IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[1];
+ primaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
@@ -121,11 +131,11 @@
IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE, });
int primaryFieldCount = 4;
- ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- primaryTypeTraits[0] = new TypeTrait(4);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(4);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ primaryTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
@@ -136,14 +146,14 @@
// non-unique key
// i.e. we will have a range condition on the first field only (implying
// [-infinity, +infinity] for the second field)
- IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[1];
- comparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ IBinaryComparatorFactory[] searchComparatorFactories = new IBinaryComparatorFactory[1];
+ searchComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
// build tuple containing low and high search keys
- ArrayTupleBuilder tb = new ArrayTupleBuilder(comparatorFactories.length * 2); // low
- // and
- // high
- // key
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(searchComparatorFactories.length * 2); // low
+ // and
+ // high
+ // key
DataOutput dos = tb.getDataOutput();
tb.reset();
@@ -171,11 +181,11 @@
IFileSplitProvider secondarySplitProvider = JobHelper.createFileSplitProvider(splitNCs,
options.secondaryBTreeName);
- ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
BTreeSearchOperatorDescriptor secondarySearchOp = new BTreeSearchOperatorDescriptor(spec, secondaryRecDesc,
- storageManager, btreeRegistryProvider, secondarySplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, comparatorFactories, true, secondaryLowKeyFields,
- secondaryHighKeyFields, true, true, opHelperFactory);
+ storageManager, indexRegistryProvider, secondarySplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits, searchComparatorFactories, true, secondaryLowKeyFields,
+ secondaryHighKeyFields, true, true, dataflowHelperFactory);
JobHelper.createPartitionConstraint(spec, secondarySearchOp, splitNCs);
// secondary index will output tuples with [UTF8String, Integer]
@@ -189,9 +199,9 @@
IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, btreeRegistryProvider, primarySplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, comparatorFactories, true, primaryLowKeyFields,
- primaryHighKeyFields, true, true, opHelperFactory);
+ storageManager, indexRegistryProvider, primarySplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, primaryLowKeyFields,
+ primaryHighKeyFields, true, true, dataflowHelperFactory);
JobHelper.createPartitionConstraint(spec, primarySearchOp, splitNCs);
// have each node print the results of its respective B-Tree
diff --git a/hyracks-examples/btree-example/btreehelper/.classpath b/hyracks-examples/btree-example/btreehelper/.classpath
deleted file mode 100644
index fb2f7c1..0000000
--- a/hyracks-examples/btree-example/btreehelper/.classpath
+++ /dev/null
@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry excluding="**" kind="src" output="target/classes" path="src/main/resources"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/btree-example/btreehelper/.project b/hyracks-examples/btree-example/btreehelper/.project
deleted file mode 100644
index c65470b..0000000
--- a/hyracks-examples/btree-example/btreehelper/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>btreehelper</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/btree-example/btreehelper/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/btree-example/btreehelper/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index dfac000..0000000
--- a/hyracks-examples/btree-example/btreehelper/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:05 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/btree-example/btreehelper/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/btree-example/btreehelper/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 6733f8d..0000000
--- a/hyracks-examples/btree-example/btreehelper/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Wed Oct 06 08:06:48 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/btree-example/btreehelper/pom.xml b/hyracks-examples/btree-example/btreehelper/pom.xml
index f790b5a..046d762 100644
--- a/hyracks-examples/btree-example/btreehelper/pom.xml
+++ b/hyracks-examples/btree-example/btreehelper/pom.xml
@@ -2,33 +2,36 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.btree</groupId>
<artifactId>btreehelper</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>btree-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
index 1581383..36e315d 100644
--- a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
+++ b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/DataGenOperatorDescriptor.java
@@ -25,7 +25,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
@@ -59,7 +58,7 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
final ByteBuffer outputFrame = ctx.allocateFrame();
@@ -83,7 +82,7 @@
appender.reset(outputFrame, true);
for (int i = 0; i < numRecords; i++) {
tb.reset();
- for (int j = 0; j < recDesc.getFields().length; j++) {
+ for (int j = 0; j < recDesc.getFieldCount(); j++) {
genField(tb, j);
}
diff --git a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/IndexRegistryProvider.java b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/IndexRegistryProvider.java
new file mode 100644
index 0000000..d1f40d8
--- /dev/null
+++ b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/IndexRegistryProvider.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.examples.btree.helper;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
+
+public class IndexRegistryProvider implements IIndexRegistryProvider<IIndex> {
+ private static final long serialVersionUID = 1L;
+
+ public static final IndexRegistryProvider INSTANCE = new IndexRegistryProvider();
+
+ private IndexRegistryProvider() {
+ }
+
+ @Override
+ public IndexRegistry<IIndex> getRegistry(IHyracksTaskContext ctx) {
+ return RuntimeContext.get(ctx).getIndexRegistry();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java
index 4a452a8..3336d6d 100644
--- a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java
+++ b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/RuntimeContext.java
@@ -17,7 +17,7 @@
import edu.uci.ics.hyracks.api.application.INCApplicationContext;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
import edu.uci.ics.hyracks.storage.common.buffercache.BufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
@@ -30,7 +30,7 @@
import edu.uci.ics.hyracks.storage.common.smi.TransientFileMapManager;
public class RuntimeContext {
- private IndexRegistry<ITreeIndex> treeIndexRegistry;
+ private IndexRegistry<IIndex> indexRegistry;
private IBufferCache bufferCache;
private IFileMapManager fileMapManager;
@@ -39,7 +39,7 @@
ICacheMemoryAllocator allocator = new HeapBufferAllocator();
IPageReplacementStrategy prs = new ClockPageReplacementStrategy();
bufferCache = new BufferCache(appCtx.getRootContext().getIOManager(), allocator, prs, fileMapManager, 32768, 50, 100);
- treeIndexRegistry = new IndexRegistry<ITreeIndex>();
+ indexRegistry = new IndexRegistry<IIndex>();
}
public void close() {
@@ -54,8 +54,8 @@
return fileMapManager;
}
- public IndexRegistry<ITreeIndex> getTreeIndexRegistry() {
- return treeIndexRegistry;
+ public IndexRegistry<IIndex> getIndexRegistry() {
+ return indexRegistry;
}
public static RuntimeContext get(IHyracksTaskContext ctx) {
diff --git a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/TreeIndexRegistryProvider.java b/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/TreeIndexRegistryProvider.java
deleted file mode 100644
index f8a7bf1..0000000
--- a/hyracks-examples/btree-example/btreehelper/src/main/java/edu/uci/ics/hyracks/examples/btree/helper/TreeIndexRegistryProvider.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.examples.btree.helper;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
-
-public class TreeIndexRegistryProvider implements IIndexRegistryProvider<ITreeIndex> {
- private static final long serialVersionUID = 1L;
-
- public static final TreeIndexRegistryProvider INSTANCE = new TreeIndexRegistryProvider();
-
- private TreeIndexRegistryProvider() {
- }
-
- @Override
- public IndexRegistry<ITreeIndex> getRegistry(IHyracksTaskContext ctx) {
- return RuntimeContext.get(ctx).getTreeIndexRegistry();
- }
-}
\ No newline at end of file
diff --git a/hyracks-examples/btree-example/pom.xml b/hyracks-examples/btree-example/pom.xml
index de4258d..96b3672 100644
--- a/hyracks-examples/btree-example/pom.xml
+++ b/hyracks-examples/btree-example/pom.xml
@@ -2,13 +2,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>btree-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<packaging>pom</packaging>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<modules>
diff --git a/hyracks-examples/hadoop-compat-example/.project b/hyracks-examples/hadoop-compat-example/.project
deleted file mode 100644
index 4e057cd..0000000
--- a/hyracks-examples/hadoop-compat-example/.project
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>text-example</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/hadoop-compat-example/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/hadoop-compat-example/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 4562b1a..0000000
--- a/hyracks-examples/hadoop-compat-example/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.classpath b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.classpath
deleted file mode 100644
index f2cc5f7..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.project b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.project
deleted file mode 100644
index 4f3af14..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>textapp</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 692202d..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,13 +0,0 @@
-#Thu May 19 22:55:12 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 4562b1a..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml
index 0d3ab7f..fe71db4 100644
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml
+++ b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/pom.xml
@@ -2,15 +2,44 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.compat</groupId>
<artifactId>hadoopcompatapp</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>hadoop-compat-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <versionRange>[1.0.0,)</versionRange>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -48,7 +77,7 @@
<plugin>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-maven-plugin</artifactId>
- <version>0.0.1</version>
+ <version>0.0.2</version>
<configuration>
<hyracksServerHome>${basedir}/../../../hyracks-server/target/hyracks-server-${project.version}-binary-assembly</hyracksServerHome>
<hyracksCLIHome>${basedir}/../../../hyracks-cli/target/hyracks-cli-${project.version}-binary-assembly</hyracksCLIHome>
@@ -140,13 +169,13 @@
<dependency>
<groupId>edu.uci.ics.hyracks.examples.compat</groupId>
<artifactId>hadoopcompathelper</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks.examples.compat</groupId>
<artifactId>hadoopcompatclient</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>test</scope>
</dependency>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/src/test/java/.classpath b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/src/test/java/.classpath
deleted file mode 100644
index ed53773..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/src/test/java/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" path="src"/>
- <classpathentry kind="src" path="compat-client"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
- <classpathentry kind="output" path="bin"/>
-</classpath>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/src/test/java/.project b/hyracks-examples/hadoop-compat-example/hadoopcompatapp/src/test/java/.project
deleted file mode 100644
index 7054f52..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatapp/src/test/java/.project
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>compat-app</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.classpath b/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.project b/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.project
deleted file mode 100644
index 04307d3..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>textclient</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 8599738..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 4562b1a..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/pom.xml b/hyracks-examples/hadoop-compat-example/hadoopcompatclient/pom.xml
index 5c343de..c248d55 100644
--- a/hyracks-examples/hadoop-compat-example/hadoopcompatclient/pom.xml
+++ b/hyracks-examples/hadoop-compat-example/hadoopcompatclient/pom.xml
@@ -2,25 +2,25 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.compat</groupId>
<artifactId>hadoopcompatclient</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>hadoop-compat-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks.examples.compat</groupId>
<artifactId>hadoopcompathelper</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.classpath b/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.project b/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.project
deleted file mode 100644
index 19ce234..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>texthelper</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 8599738..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 4562b1a..0000000
--- a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/pom.xml b/hyracks-examples/hadoop-compat-example/hadoopcompathelper/pom.xml
index b45f0e9..c19e56a 100644
--- a/hyracks-examples/hadoop-compat-example/hadoopcompathelper/pom.xml
+++ b/hyracks-examples/hadoop-compat-example/hadoopcompathelper/pom.xml
@@ -2,25 +2,25 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.compat</groupId>
<artifactId>hadoopcompathelper</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>hadoop-compat-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
</dependencies>
diff --git a/hyracks-examples/hadoop-compat-example/pom.xml b/hyracks-examples/hadoop-compat-example/pom.xml
index 6310d29..51e92ab 100644
--- a/hyracks-examples/hadoop-compat-example/pom.xml
+++ b/hyracks-examples/hadoop-compat-example/pom.xml
@@ -2,13 +2,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>hadoop-compat-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<packaging>pom</packaging>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<modules>
@@ -29,7 +29,7 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-hadoop-compat</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-examples/hyracks-integration-tests/.classpath b/hyracks-examples/hyracks-integration-tests/.classpath
deleted file mode 100644
index f2cc5f7..0000000
--- a/hyracks-examples/hyracks-integration-tests/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/hyracks-integration-tests/.project b/hyracks-examples/hyracks-integration-tests/.project
deleted file mode 100644
index 91bbbe6..0000000
--- a/hyracks-examples/hyracks-integration-tests/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-integration-tests</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/hyracks-integration-tests/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/hyracks-integration-tests/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index a80ec7b..0000000
--- a/hyracks-examples/hyracks-integration-tests/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:08 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/hyracks-integration-tests/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/hyracks-integration-tests/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 6b45873..0000000
--- a/hyracks-examples/hyracks-integration-tests/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Jul 29 15:30:15 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/hyracks-integration-tests/pom.xml b/hyracks-examples/hyracks-integration-tests/pom.xml
index 2006dc9..d74b384 100644
--- a/hyracks-examples/hyracks-integration-tests/pom.xml
+++ b/hyracks-examples/hyracks-integration-tests/pom.xml
@@ -2,12 +2,10 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>hyracks-integration-tests</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -21,14 +19,6 @@
<target>1.6</target>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-plugin</artifactId>
- <configuration>
- <forkMode>pertest</forkMode>
- <argLine>-enableassertions</argLine>
- </configuration>
- </plugin>
</plugins>
</build>
<dependencies>
@@ -42,51 +32,56 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-cc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-invertedindex</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-rtree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-test-support</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexScanOperatorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexScanOperatorTest.java
index ba7621d..6ef1740 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexScanOperatorTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexScanOperatorTest.java
@@ -25,15 +25,16 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -43,197 +44,157 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
-import edu.uci.ics.hyracks.test.support.TestTreeIndexRegistryProvider;
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
public class BTreePrimaryIndexScanOperatorTest extends AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20, 20);
- }
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
- private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
- private ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
+ private IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
- private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
- "ddMMyy-hhmmssSS");
- private final static String sep = System.getProperty("file.separator");
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
- // field, type and key declarations for primary index
- private int primaryFieldCount = 6;
- private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- private int primaryKeyFieldCount = 1;
- private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(
- primaryTupleWriterFactory);
+ // field, type and key declarations for primary index
+ private int primaryFieldCount = 6;
+ private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ private int primaryKeyFieldCount = 1;
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
+ private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ primaryTupleWriterFactory);
+ private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
- private static String primaryBtreeName = "primary"
- + simpleDateFormat.format(new Date());
- private static String primaryFileName = System
- .getProperty("java.io.tmpdir") + sep + primaryBtreeName;
+ private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
+ private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
- private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- primaryFileName))) });
+ private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
- @Before
- public void setup() throws Exception {
- // field, type and key declarations for primary index
- primaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ @Before
+ public void setup() throws Exception {
+ // field, type and key declarations for primary index
+ primaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
+ primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- loadPrimaryIndexTest();
- }
+ loadPrimaryIndexTest();
+ }
- public void loadPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ public void loadPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/tpch0.001/orders-part1.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
- ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/tpch0.001/orders-part1.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
- spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
- new IValueParserFactory[] {
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE }, '|'),
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- ordScanner, NC1_ID);
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
- spec,
- 1000,
- new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
- NC1_ID);
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 0 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
- int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
- TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, null, fieldPermutation, 0.7f,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeBulkLoad, NC1_ID);
+ int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
- sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
- primaryBtreeBulkLoad, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, primaryBtreeBulkLoad, 0);
- spec.addRoot(primaryBtreeBulkLoad);
- runTest(spec);
- }
+ spec.addRoot(primaryBtreeBulkLoad);
+ runTest(spec);
+ }
- @Test
- public void scanPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void scanPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build dummy tuple containing nothing
- ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
- DataOutput dos = tb.getDataOutput();
+ // build dummy tuple containing nothing
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
- spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
- tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- int[] lowKeyFields = null; // - infinity
- int[] highKeyFields = null; // + infinity
+ int[] lowKeyFields = null; // - infinity
+ int[] highKeyFields = null; // + infinity
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
- spec, primaryRecDesc, storageManager,
- treeIndexRegistryProvider, primaryBtreeSplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory,
- primaryTypeTraits, primaryComparatorFactories, true,
- lowKeyFields, highKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeSearchOp, NC1_ID);
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
+ highKeyFields, true, true, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
- NC1_ID);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
- primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- primaryBtreeSearchOp, 0, printer, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, printer, 0);
- spec.addRoot(printer);
- runTest(spec);
- }
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- @AfterClass
- public static void cleanup() throws Exception {
- File primary = new File(primaryFileName);
- primary.deleteOnExit();
- }
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primary = new File(primaryFileName);
+ primary.deleteOnExit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexSearchOperatorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexSearchOperatorTest.java
index c64424d..51dbb69 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexSearchOperatorTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexSearchOperatorTest.java
@@ -25,15 +25,16 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -43,203 +44,162 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
-import edu.uci.ics.hyracks.test.support.TestTreeIndexRegistryProvider;
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
-public class BTreePrimaryIndexSearchOperatorTest extends
- AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20, 20);
- }
+public class BTreePrimaryIndexSearchOperatorTest extends AbstractIntegrationTest {
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
- private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
- private ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
+ private IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
- private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
- "ddMMyy-hhmmssSS");
- private final static String sep = System.getProperty("file.separator");
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
- // field, type and key declarations for primary index
- private int primaryFieldCount = 6;
- private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- private int primaryKeyFieldCount = 1;
- private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(
- primaryTupleWriterFactory);
+ // field, type and key declarations for primary index
+ private int primaryFieldCount = 6;
+ private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ private int primaryKeyFieldCount = 1;
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
+ private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ primaryTupleWriterFactory);
+ private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
- private static String primaryBtreeName = "primary"
- + simpleDateFormat.format(new Date());
- private static String primaryFileName = System
- .getProperty("java.io.tmpdir") + sep + primaryBtreeName;
+ private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
+ private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
- private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- primaryFileName))) });
+ private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
- @Before
- public void setup() throws Exception {
- // field, type and key declarations for primary index
- primaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ @Before
+ public void setup() throws Exception {
+ // field, type and key declarations for primary index
+ primaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
+ primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- loadPrimaryIndexTest();
- }
+ loadPrimaryIndexTest();
+ }
- public void loadPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ public void loadPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/tpch0.001/orders-part1.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
- ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/tpch0.001/orders-part1.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
- spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
- new IValueParserFactory[] {
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE }, '|'),
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- ordScanner, NC1_ID);
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
- spec,
- 1000,
- new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
- NC1_ID);
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 0 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
- int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
- TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, null, fieldPermutation, 0.7f,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeBulkLoad, NC1_ID);
+ int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
- sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
- primaryBtreeBulkLoad, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, primaryBtreeBulkLoad, 0);
- spec.addRoot(primaryBtreeBulkLoad);
- runTest(spec);
- }
+ spec.addRoot(primaryBtreeBulkLoad);
+ runTest(spec);
+ }
- @Test
- public void searchPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void searchPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build tuple containing low and high search key
- // high key and low key
- ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
- DataOutput dos = tb.getDataOutput();
+ // build tuple containing low and high search key
+ // high key and low key
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- // low key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("100", dos);
- tb.addFieldEndOffset();
- // high key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("200", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ // low key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("100", dos);
+ tb.addFieldEndOffset();
+ // high key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("200", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
- spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
- tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- int[] lowKeyFields = { 0 };
- int[] highKeyFields = { 1 };
+ int[] lowKeyFields = { 0 };
+ int[] highKeyFields = { 1 };
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
- spec, primaryRecDesc, storageManager,
- treeIndexRegistryProvider, primaryBtreeSplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory,
- primaryTypeTraits, primaryComparatorFactories, true,
- lowKeyFields, highKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeSearchOp, NC1_ID);
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
+ highKeyFields, true, true, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
- NC1_ID);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
- primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- primaryBtreeSearchOp, 0, printer, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, printer, 0);
- spec.addRoot(printer);
- runTest(spec);
- }
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- @AfterClass
- public static void cleanup() throws Exception {
- File primary = new File(primaryFileName);
- primary.deleteOnExit();
- }
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primary = new File(primaryFileName);
+ primary.deleteOnExit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexStatsOperatorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexStatsOperatorTest.java
index e8fd60b..a804bae 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexStatsOperatorTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreePrimaryIndexStatsOperatorTest.java
@@ -24,14 +24,15 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -41,163 +42,133 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexStatsOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
-import edu.uci.ics.hyracks.test.support.TestTreeIndexRegistryProvider;
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
public class BTreePrimaryIndexStatsOperatorTest extends AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20, 20);
- }
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
- private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
- private ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
+ private IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
- private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
- "ddMMyy-hhmmssSS");
- private final static String sep = System.getProperty("file.separator");
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
- // field, type and key declarations for primary index
- private int primaryFieldCount = 6;
- private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- private int primaryKeyFieldCount = 1;
- private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(
- primaryTupleWriterFactory);
+ // field, type and key declarations for primary index
+ private int primaryFieldCount = 6;
+ private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ private int primaryKeyFieldCount = 1;
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
+ private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ primaryTupleWriterFactory);
+ private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
- private static String primaryBtreeName = "primary"
- + simpleDateFormat.format(new Date());
- private static String primaryFileName = System
- .getProperty("java.io.tmpdir") + sep + primaryBtreeName;
+ private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
+ private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
- private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- primaryFileName))) });
+ private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
- @Before
- public void setup() throws Exception {
- // field, type and key declarations for primary index
- primaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ @Before
+ public void setup() throws Exception {
+ // field, type and key declarations for primary index
+ primaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
+ primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- loadPrimaryIndexTest();
- }
+ loadPrimaryIndexTest();
+ }
- public void loadPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ public void loadPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/tpch0.001/orders-part1.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
- ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/tpch0.001/orders-part1.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
- spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
- new IValueParserFactory[] {
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE }, '|'),
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- ordScanner, NC1_ID);
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
- spec,
- 1000,
- new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
- NC1_ID);
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 0 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
- int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
- TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, null, fieldPermutation, 0.7f,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeBulkLoad, NC1_ID);
+ int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
- sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
- primaryBtreeBulkLoad, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, primaryBtreeBulkLoad, 0);
- spec.addRoot(primaryBtreeBulkLoad);
- runTest(spec);
- }
+ spec.addRoot(primaryBtreeBulkLoad);
+ runTest(spec);
+ }
- @Test
- public void showPrimaryIndexStats() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void showPrimaryIndexStats() throws Exception {
+ JobSpecification spec = new JobSpecification();
- TreeIndexStatsOperatorDescriptor primaryStatsOp = new TreeIndexStatsOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryStatsOp, NC1_ID);
+ TreeIndexStatsOperatorDescriptor primaryStatsOp = new TreeIndexStatsOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory, primaryLeafFrameFactory,
+ primaryTypeTraits, primaryComparatorFactories, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryStatsOp, NC1_ID);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
- spec.addRoot(primaryStatsOp);
- runTest(spec);
- }
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryStatsOp, 0, printer, 0);
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- @AfterClass
- public static void cleanup() throws Exception {
- File primary = new File(primaryFileName);
- primary.deleteOnExit();
- }
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primary = new File(primaryFileName);
+ primary.deleteOnExit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexInsertOperatorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexInsertOperatorTest.java
index 5c527cb..a16bbe4 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexInsertOperatorTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexInsertOperatorTest.java
@@ -25,15 +25,16 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -43,406 +44,313 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexInsertUpdateDeleteOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
-import edu.uci.ics.hyracks.test.support.TestTreeIndexRegistryProvider;
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
-public class BTreeSecondaryIndexInsertOperatorTest extends
- AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20, 20);
- }
+public class BTreeSecondaryIndexInsertOperatorTest extends AbstractIntegrationTest {
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
- private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
- private ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
+ private IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
- private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
- "ddMMyy-hhmmssSS");
- private final static String sep = System.getProperty("file.separator");
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
- // field, type and key declarations for primary index
- private int primaryFieldCount = 6;
- private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- private int primaryKeyFieldCount = 1;
- private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(
- primaryTupleWriterFactory);
+ // field, type and key declarations for primary index
+ private int primaryFieldCount = 6;
+ private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ private int primaryKeyFieldCount = 1;
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
+ private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ primaryTupleWriterFactory);
+ private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
- private static String primaryBtreeName = "primary"
- + simpleDateFormat.format(new Date());
- private static String primaryFileName = System
- .getProperty("java.io.tmpdir") + sep + primaryBtreeName;
+ private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
+ private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
- private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- primaryFileName))) });
+ private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
- // field, type and key declarations for secondary indexes
- private int secondaryFieldCount = 2;
- private ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
- private int secondaryKeyFieldCount = 2;
- private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
- private TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- secondaryTypeTraits);
- private ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- secondaryTupleWriterFactory);
- private ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(
- secondaryTupleWriterFactory);
+ // field, type and key declarations for secondary indexes
+ private int secondaryFieldCount = 2;
+ private ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount];
+ private int secondaryKeyFieldCount = 2;
+ private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
+ secondaryTypeTraits);
+ private ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ secondaryTupleWriterFactory);
+ private ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
- private static String secondaryBtreeName = "secondary"
- + simpleDateFormat.format(new Date());
- private static String secondaryFileName = System
- .getProperty("java.io.tmpdir") + sep + secondaryBtreeName;
+ private static String secondaryBtreeName = "secondary" + simpleDateFormat.format(new Date());
+ private static String secondaryFileName = System.getProperty("java.io.tmpdir") + sep + secondaryBtreeName;
- private IFileSplitProvider secondaryBtreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- secondaryFileName))) });
+ private IFileSplitProvider secondaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(secondaryFileName))) });
- private RecordDescriptor secondaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor secondaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
- @Before
- public void setup() throws Exception {
- // field, type and key declarations for primary index
- primaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ @Before
+ public void setup() throws Exception {
+ // field, type and key declarations for primary index
+ primaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
+ primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- // field, type and key declarations for secondary indexes
- secondaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
- secondaryComparatorFactories[1] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ // field, type and key declarations for secondary indexes
+ secondaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ secondaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ secondaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ secondaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- loadPrimaryIndexTest();
- loadSecondaryIndexTest();
- insertPipelineTest();
- }
+ loadPrimaryIndexTest();
+ loadSecondaryIndexTest();
+ insertPipelineTest();
+ }
- public void loadPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ public void loadPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/tpch0.001/orders-part1.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
- ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/tpch0.001/orders-part1.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
- spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
- new IValueParserFactory[] {
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE }, '|'),
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- ordScanner, NC1_ID);
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
- spec,
- 1000,
- new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
- NC1_ID);
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 0 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
- int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
- TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, null, fieldPermutation, 0.7f,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeBulkLoad, NC1_ID);
+ int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
- sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
- primaryBtreeBulkLoad, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, primaryBtreeBulkLoad, 0);
- spec.addRoot(primaryBtreeBulkLoad);
- runTest(spec);
- }
+ spec.addRoot(primaryBtreeBulkLoad);
+ runTest(spec);
+ }
- public void loadSecondaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ public void loadSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build dummy tuple containing nothing
- ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
- DataOutput dos = tb.getDataOutput();
+ // build dummy tuple containing nothing
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
- spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
- tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- int[] lowKeyFields = null; // - infinity
- int[] highKeyFields = null; // + infinity
+ int[] lowKeyFields = null; // - infinity
+ int[] highKeyFields = null; // + infinity
- // scan primary index
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
- spec, primaryRecDesc, storageManager,
- treeIndexRegistryProvider, primaryBtreeSplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory,
- primaryTypeTraits, primaryComparatorFactories, true,
- lowKeyFields, highKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeSearchOp, NC1_ID);
+ // scan primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
+ highKeyFields, true, true, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
- // sort based on secondary keys
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
- spec,
- 1000,
- new int[] { 3, 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- primaryRecDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
- NC1_ID);
+ // sort based on secondary keys
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 3, 0 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ primaryRecDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
- // load secondary index
- int[] fieldPermutation = { 3, 0 };
- TreeIndexBulkLoadOperatorDescriptor secondaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, null, fieldPermutation, 0.7f,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- secondaryBtreeBulkLoad, NC1_ID);
+ // load secondary index
+ int[] fieldPermutation = { 3, 0 };
+ TreeIndexBulkLoadOperatorDescriptor secondaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeBulkLoad, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
- primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- primaryBtreeSearchOp, 0, sorter, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
- secondaryBtreeBulkLoad, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, secondaryBtreeBulkLoad, 0);
- spec.addRoot(secondaryBtreeBulkLoad);
- runTest(spec);
- }
+ spec.addRoot(secondaryBtreeBulkLoad);
+ runTest(spec);
+ }
- public void insertPipelineTest() throws Exception {
+ public void insertPipelineTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
- ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/tpch0.001/orders-part2.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
- spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
- new IValueParserFactory[] {
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE }, '|'),
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- ordScanner, NC1_ID);
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
- // insert into primary index
- int[] primaryFieldPermutation = { 0, 1, 2, 4, 5, 7 };
- TreeIndexInsertUpdateDeleteOperatorDescriptor primaryBtreeInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
- spec, ordersDesc, storageManager, treeIndexRegistryProvider,
- primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, null, primaryFieldPermutation,
- IndexOp.INSERT, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeInsertOp, NC1_ID);
+ // insert into primary index
+ int[] primaryFieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ TreeIndexInsertUpdateDeleteOperatorDescriptor primaryBtreeInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
+ spec, ordersDesc, storageManager, indexRegistryProvider, primaryBtreeSplitProvider,
+ primaryInteriorFrameFactory, primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories,
+ primaryFieldPermutation, IndexOp.INSERT, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeInsertOp, NC1_ID);
- // first secondary index
- int[] fieldPermutationB = { 4, 0 };
- TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
- spec, ordersDesc, storageManager, treeIndexRegistryProvider,
- secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, null, fieldPermutationB,
- IndexOp.INSERT, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- secondaryInsertOp, NC1_ID);
+ // first secondary index
+ int[] fieldPermutationB = { 4, 0 };
+ TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
+ spec, ordersDesc, storageManager, indexRegistryProvider, secondaryBtreeSplitProvider,
+ secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, fieldPermutationB, IndexOp.INSERT, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryInsertOp, NC1_ID);
- NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(
- spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, nullSink,
- NC1_ID);
+ NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, nullSink, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
- primaryBtreeInsertOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, primaryBtreeInsertOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- primaryBtreeInsertOp, 0, secondaryInsertOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeInsertOp, 0, secondaryInsertOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), secondaryInsertOp,
- 0, nullSink, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), secondaryInsertOp, 0, nullSink, 0);
- spec.addRoot(nullSink);
- runTest(spec);
- }
+ spec.addRoot(nullSink);
+ runTest(spec);
+ }
- @Test
- public void searchUpdatedSecondaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void searchUpdatedSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build tuple containing search keys (only use the first key as search
- // key)
- ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
- DataOutput dos = tb.getDataOutput();
+ // build tuple containing search keys (only use the first key as search
+ // key)
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- // low key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
- tb.addFieldEndOffset();
- // high key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ // low key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
+ tb.addFieldEndOffset();
+ // high key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
- spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
- tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- int[] secondaryLowKeyFields = { 0 };
- int[] secondaryHighKeyFields = { 1 };
+ int[] secondaryLowKeyFields = { 0 };
+ int[] secondaryHighKeyFields = { 1 };
- // search secondary index
- BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
- spec, secondaryRecDesc, storageManager,
- treeIndexRegistryProvider, secondaryBtreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory,
- secondaryTypeTraits, secondaryComparatorFactories, true,
- secondaryLowKeyFields, secondaryHighKeyFields, true, true,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- secondaryBtreeSearchOp, NC1_ID);
+ // search secondary index
+ BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
+ secondaryRecDesc, storageManager, indexRegistryProvider, secondaryBtreeSplitProvider,
+ secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, true, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeSearchOp, NC1_ID);
- // second field from the tuples coming from secondary index
- int[] primaryLowKeyFields = { 1 };
- // second field from the tuples coming from secondary index
- int[] primaryHighKeyFields = { 1 };
+ // second field from the tuples coming from secondary index
+ int[] primaryLowKeyFields = { 1 };
+ // second field from the tuples coming from secondary index
+ int[] primaryHighKeyFields = { 1 };
- // search primary index
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
- spec, primaryRecDesc, storageManager,
- treeIndexRegistryProvider, primaryBtreeSplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory,
- primaryTypeTraits, primaryComparatorFactories, true,
- primaryLowKeyFields, primaryHighKeyFields, true, true,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeSearchOp, NC1_ID);
+ // search primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, primaryLowKeyFields,
+ primaryHighKeyFields, true, true, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
- NC1_ID);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
- secondaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- primaryBtreeSearchOp, 0, printer, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, secondaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, printer, 0);
- spec.addRoot(printer);
- runTest(spec);
- }
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- @AfterClass
- public static void cleanup() throws Exception {
- File primary = new File(primaryFileName);
- primary.deleteOnExit();
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primary = new File(primaryFileName);
+ primary.deleteOnExit();
- File secondary = new File(secondaryFileName);
- secondary.deleteOnExit();
- }
+ File secondary = new File(secondaryFileName);
+ secondary.deleteOnExit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexSearchOperatorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexSearchOperatorTest.java
index 60c3b13..ef1c6f6 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeSecondaryIndexSearchOperatorTest.java
@@ -25,15 +25,16 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -43,326 +44,257 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
-import edu.uci.ics.hyracks.test.support.TestTreeIndexRegistryProvider;
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
-public class BTreeSecondaryIndexSearchOperatorTest extends
- AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20, 20);
- }
+public class BTreeSecondaryIndexSearchOperatorTest extends AbstractIntegrationTest {
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
- private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
- private ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
+ private IIndexDataflowHelperFactory dataflowHelperFactory = new BTreeDataflowHelperFactory();
- private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
- "ddMMyy-hhmmssSS");
- private final static String sep = System.getProperty("file.separator");
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
- // field, type and key declarations for primary index
- private int primaryFieldCount = 6;
- private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- private int primaryKeyFieldCount = 1;
- private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(
- primaryTupleWriterFactory);
+ // field, type and key declarations for primary index
+ private int primaryFieldCount = 6;
+ private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ private int primaryKeyFieldCount = 1;
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
+ private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ primaryTupleWriterFactory);
+ private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
- private static String primaryBtreeName = "primary"
- + simpleDateFormat.format(new Date());
- private static String primaryFileName = System
- .getProperty("java.io.tmpdir") + sep + primaryBtreeName;
+ private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
+ private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
- private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- primaryFileName))) });
+ private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
- // field, type and key declarations for secondary indexes
- private int secondaryFieldCount = 2;
- private ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
- private int secondaryKeyFieldCount = 2;
- private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
- private TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- secondaryTypeTraits);
- private ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- secondaryTupleWriterFactory);
- private ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(
- secondaryTupleWriterFactory);
+ // field, type and key declarations for secondary indexes
+ private int secondaryFieldCount = 2;
+ private ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount];
+ private int secondaryKeyFieldCount = 2;
+ private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
+ secondaryTypeTraits);
+ private ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ secondaryTupleWriterFactory);
+ private ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
- private static String secondaryBtreeName = "secondary"
- + simpleDateFormat.format(new Date());
- private static String secondaryFileName = System
- .getProperty("java.io.tmpdir") + sep + secondaryBtreeName;
+ private static String secondaryBtreeName = "secondary" + simpleDateFormat.format(new Date());
+ private static String secondaryFileName = System.getProperty("java.io.tmpdir") + sep + secondaryBtreeName;
- private IFileSplitProvider secondaryBtreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- secondaryFileName))) });
+ private IFileSplitProvider secondaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(secondaryFileName))) });
- private RecordDescriptor secondaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor secondaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
- @Before
- public void setup() throws Exception {
- // field, type and key declarations for primary index
- primaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ @Before
+ public void setup() throws Exception {
+ // field, type and key declarations for primary index
+ primaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
+ primaryTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
+ primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- // field, type and key declarations for secondary indexes
- secondaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
- secondaryComparatorFactories[1] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ // field, type and key declarations for secondary indexes
+ secondaryTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ secondaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ secondaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+ secondaryComparatorFactories[1] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- loadPrimaryIndexTest();
- loadSecondaryIndexTest();
- }
+ loadPrimaryIndexTest();
+ loadSecondaryIndexTest();
+ }
- public void loadPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ public void loadPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/tpch0.001/orders-part1.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
- ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/tpch0.001/orders-part1.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
- spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
- new IValueParserFactory[] {
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE }, '|'),
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- ordScanner, NC1_ID);
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
- spec,
- 1000,
- new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
- NC1_ID);
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 0 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
- int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
- TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, null, fieldPermutation, 0.7f,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeBulkLoad, NC1_ID);
+ int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
- sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
- primaryBtreeBulkLoad, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, primaryBtreeBulkLoad, 0);
- spec.addRoot(primaryBtreeBulkLoad);
- runTest(spec);
- }
+ spec.addRoot(primaryBtreeBulkLoad);
+ runTest(spec);
+ }
- public void loadSecondaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ public void loadSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build dummy tuple containing nothing
- ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
- DataOutput dos = tb.getDataOutput();
+ // build dummy tuple containing nothing
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
- spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
- tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- int[] lowKeyFields = null; // - infinity
- int[] highKeyFields = null; // + infinity
+ int[] lowKeyFields = null; // - infinity
+ int[] highKeyFields = null; // + infinity
- // scan primary index
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
- spec, primaryRecDesc, storageManager,
- treeIndexRegistryProvider, primaryBtreeSplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory,
- primaryTypeTraits, primaryComparatorFactories, true,
- lowKeyFields, highKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeSearchOp, NC1_ID);
+ // scan primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
+ highKeyFields, true, true, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
- // sort based on secondary keys
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
- spec,
- 1000,
- new int[] { 3, 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- primaryRecDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
- NC1_ID);
+ // sort based on secondary keys
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 3, 0 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ primaryRecDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
- // load secondary index
- int[] fieldPermutation = { 3, 0 };
- TreeIndexBulkLoadOperatorDescriptor secondaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, null, fieldPermutation, 0.7f,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- secondaryBtreeBulkLoad, NC1_ID);
+ // load secondary index
+ int[] fieldPermutation = { 3, 0 };
+ TreeIndexBulkLoadOperatorDescriptor secondaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeBulkLoad, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
- primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- primaryBtreeSearchOp, 0, sorter, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
- secondaryBtreeBulkLoad, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, secondaryBtreeBulkLoad, 0);
- spec.addRoot(secondaryBtreeBulkLoad);
- runTest(spec);
- }
+ spec.addRoot(secondaryBtreeBulkLoad);
+ runTest(spec);
+ }
- @Test
- public void searchSecondaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void searchSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build tuple containing search keys (only use the first key as search
- // key)
- ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
- DataOutput dos = tb.getDataOutput();
+ // build tuple containing search keys (only use the first key as search
+ // key)
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- // low key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
- tb.addFieldEndOffset();
- // high key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ // low key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
+ tb.addFieldEndOffset();
+ // high key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
- spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
- tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- int[] secondaryLowKeyFields = { 0 };
- int[] secondaryHighKeyFields = { 1 };
+ int[] secondaryLowKeyFields = { 0 };
+ int[] secondaryHighKeyFields = { 1 };
- // search secondary index
- BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
- spec, secondaryRecDesc, storageManager,
- treeIndexRegistryProvider, secondaryBtreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory,
- secondaryTypeTraits, secondaryComparatorFactories, true,
- secondaryLowKeyFields, secondaryHighKeyFields, true, true,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- secondaryBtreeSearchOp, NC1_ID);
+ // search secondary index
+ BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
+ secondaryRecDesc, storageManager, indexRegistryProvider, secondaryBtreeSplitProvider,
+ secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, true, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeSearchOp, NC1_ID);
- int[] primaryLowKeyFields = { 1 }; // second field from the tuples
- // coming from secondary index
- int[] primaryHighKeyFields = { 1 }; // second field from the tuples
- // coming from secondary index
+ int[] primaryLowKeyFields = { 1 }; // second field from the tuples
+ // coming from secondary index
+ int[] primaryHighKeyFields = { 1 }; // second field from the tuples
+ // coming from secondary index
- // search primary index
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
- spec, primaryRecDesc, storageManager,
- treeIndexRegistryProvider, primaryBtreeSplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory,
- primaryTypeTraits, primaryComparatorFactories, true,
- primaryLowKeyFields, primaryHighKeyFields, true, true,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBtreeSearchOp, NC1_ID);
+ // search primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
+ storageManager, indexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, primaryLowKeyFields,
+ primaryHighKeyFields, true, true, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
- NC1_ID);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
- secondaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- primaryBtreeSearchOp, 0, printer, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, secondaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, printer, 0);
- spec.addRoot(printer);
- runTest(spec);
- }
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- @AfterClass
- public static void cleanup() throws Exception {
- File primary = new File(primaryFileName);
- primary.deleteOnExit();
-
- File secondary = new File(secondaryFileName);
- secondary.deleteOnExit();
- }
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primary = new File(primaryFileName);
+ primary.deleteOnExit();
+ File secondary = new File(secondaryFileName);
+ secondary.deleteOnExit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/comm/SerializationDeserializationTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/comm/SerializationDeserializationTest.java
index 8a78e49..0b4631d 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/comm/SerializationDeserializationTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/comm/SerializationDeserializationTest.java
@@ -19,6 +19,8 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import org.junit.Test;
@@ -39,6 +41,7 @@
import edu.uci.ics.hyracks.test.support.TestUtils;
public class SerializationDeserializationTest {
+ private static final Logger LOGGER = Logger.getLogger(SerializationDeserializationTest.class.getName());
private static final String DBLP_FILE = "data/dblp.txt";
private static class SerDeserRunner {
@@ -125,7 +128,9 @@
reader.open();
Object[] arr;
while ((arr = reader.readData()) != null) {
- System.err.println(arr[0] + " " + arr[1]);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(arr[0] + " " + arr[1]);
+ }
}
reader.close();
}
@@ -145,4 +150,4 @@
};
run(rDes, processor);
}
-}
+}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
index 237ab52..4ed8361 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractIntegrationTest.java
@@ -14,12 +14,21 @@
*/
package edu.uci.ics.hyracks.tests.integration;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+import org.apache.commons.io.FileUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
-import edu.uci.ics.hyracks.api.client.HyracksLocalConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
@@ -30,6 +39,8 @@
import edu.uci.ics.hyracks.control.nc.NodeControllerService;
public abstract class AbstractIntegrationTest {
+ private static final Logger LOGGER = Logger.getLogger(AbstractIntegrationTest.class.getName());
+
public static final String NC1_ID = "nc1";
public static final String NC2_ID = "nc2";
@@ -38,17 +49,36 @@
private static NodeControllerService nc2;
private static IHyracksClientConnection hcc;
+ private final List<File> outputFiles;
+
+ @Rule
+ public TemporaryFolder outputFolder = new TemporaryFolder();
+
+ public AbstractIntegrationTest() {
+ outputFiles = new ArrayList<File>();
+ }
+
@BeforeClass
public static void init() throws Exception {
CCConfig ccConfig = new CCConfig();
- ccConfig.port = 39001;
+ ccConfig.clientNetIpAddress = "127.0.0.1";
+ ccConfig.clientNetPort = 39000;
+ ccConfig.clusterNetIpAddress = "127.0.0.1";
+ ccConfig.clusterNetPort = 39001;
ccConfig.profileDumpPeriod = 10000;
+ File outDir = new File("target/ClusterController");
+ outDir.mkdirs();
+ File ccRoot = File.createTempFile(AbstractIntegrationTest.class.getName(), ".data", outDir);
+ ccRoot.delete();
+ ccRoot.mkdir();
+ ccConfig.ccRoot = ccRoot.getAbsolutePath();
cc = new ClusterControllerService(ccConfig);
cc.start();
NCConfig ncConfig1 = new NCConfig();
ncConfig1.ccHost = "localhost";
ncConfig1.ccPort = 39001;
+ ncConfig1.clusterNetIPAddress = "127.0.0.1";
ncConfig1.dataIPAddress = "127.0.0.1";
ncConfig1.nodeId = NC1_ID;
nc1 = new NodeControllerService(ncConfig1);
@@ -57,13 +87,17 @@
NCConfig ncConfig2 = new NCConfig();
ncConfig2.ccHost = "localhost";
ncConfig2.ccPort = 39001;
+ ncConfig2.clusterNetIPAddress = "127.0.0.1";
ncConfig2.dataIPAddress = "127.0.0.1";
ncConfig2.nodeId = NC2_ID;
nc2 = new NodeControllerService(ncConfig2);
nc2.start();
- hcc = new HyracksLocalConnection(cc);
+ hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
hcc.createApplication("test", null);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Starting CC in " + ccRoot.getAbsolutePath());
+ }
}
@AfterClass
@@ -75,9 +109,40 @@
protected void runTest(JobSpecification spec) throws Exception {
JobId jobId = hcc.createJob("test", spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
- System.err.println(spec.toJSON().toString(2));
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(spec.toJSON().toString(2));
+ }
hcc.start(jobId);
- System.err.print(jobId);
- cc.waitForCompletion(jobId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(jobId.toString());
+ }
+ hcc.waitForCompletion(jobId);
+ dumpOutputFiles();
+ }
+
+ private void dumpOutputFiles() {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ for (File f : outputFiles) {
+ if (f.exists() && f.isFile()) {
+ try {
+ LOGGER.info("Reading file: " + f.getAbsolutePath() + " in test: " + getClass().getName());
+ String data = FileUtils.readFileToString(f);
+ LOGGER.info(data);
+ } catch (IOException e) {
+ LOGGER.info("Error reading file: " + f.getAbsolutePath());
+ LOGGER.info(e.getMessage());
+ }
+ }
+ }
+ }
+ }
+
+ protected File createTempFile() throws IOException {
+ File tempFile = File.createTempFile(getClass().getName(), ".tmp", outputFolder.getRoot());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Output file: " + tempFile.getAbsolutePath());
+ }
+ outputFiles.add(tempFile);
+ return tempFile;
}
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
new file mode 100644
index 0000000..dec42c1
--- /dev/null
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AbstractMultiNCIntegrationTest.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.tests.integration;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.rules.TemporaryFolder;
+
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
+import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
+import edu.uci.ics.hyracks.api.job.JobFlag;
+import edu.uci.ics.hyracks.api.job.JobId;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.control.cc.ClusterControllerService;
+import edu.uci.ics.hyracks.control.common.controllers.CCConfig;
+import edu.uci.ics.hyracks.control.common.controllers.NCConfig;
+import edu.uci.ics.hyracks.control.nc.NodeControllerService;
+
+public abstract class AbstractMultiNCIntegrationTest {
+
+ private static final Logger LOGGER = Logger.getLogger(AbstractMultiNCIntegrationTest.class.getName());
+
+ public static final String[] ASTERIX_IDS = { "asterix-001", "asterix-002", "asterix-003", "asterix-004",
+ "asterix-005", "asterix-006", "asterix-007" };
+
+ private static ClusterControllerService cc;
+
+ private static NodeControllerService[] asterixNCs;
+
+ private static IHyracksClientConnection hcc;
+
+ private final List<File> outputFiles;
+
+ @Rule
+ public TemporaryFolder outputFolder = new TemporaryFolder();
+
+ public AbstractMultiNCIntegrationTest() {
+ outputFiles = new ArrayList<File>();;
+ }
+
+ @BeforeClass
+ public static void init() throws Exception {
+ CCConfig ccConfig = new CCConfig();
+ ccConfig.clientNetIpAddress = "127.0.0.1";
+ ccConfig.clientNetPort = 39000;
+ ccConfig.clusterNetIpAddress = "127.0.0.1";
+ ccConfig.clusterNetPort = 39001;
+ ccConfig.profileDumpPeriod = 10000;
+ File outDir = new File("target/ClusterController");
+ outDir.mkdirs();
+ File ccRoot = File.createTempFile(AbstractMultiNCIntegrationTest.class.getName(), ".data", outDir);
+ ccRoot.delete();
+ ccRoot.mkdir();
+ ccConfig.ccRoot = ccRoot.getAbsolutePath();
+ cc = new ClusterControllerService(ccConfig);
+ cc.start();
+
+ asterixNCs = new NodeControllerService[ASTERIX_IDS.length];
+ for (int i = 0; i < ASTERIX_IDS.length; i++) {
+ NCConfig ncConfig = new NCConfig();
+ ncConfig.ccHost = "localhost";
+ ncConfig.ccPort = 39001;
+ ncConfig.clusterNetIPAddress = "127.0.0.1";
+ ncConfig.dataIPAddress = "127.0.0.1";
+ ncConfig.nodeId = ASTERIX_IDS[i];
+ asterixNCs[i] = new NodeControllerService(ncConfig);
+ asterixNCs[i].start();
+ }
+
+ hcc = new HyracksConnection(ccConfig.clientNetIpAddress, ccConfig.clientNetPort);
+ hcc.createApplication("test", null);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Starting CC in " + ccRoot.getAbsolutePath());
+ }
+ }
+
+ @AfterClass
+ public static void deinit() throws Exception {
+ for (NodeControllerService nc : asterixNCs) {
+ nc.stop();
+ }
+ cc.stop();
+ }
+
+ protected void runTest(JobSpecification spec) throws Exception {
+ JobId jobId = hcc.createJob("test", spec, EnumSet.of(JobFlag.PROFILE_RUNTIME));
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(spec.toJSON().toString(2));
+ }
+ hcc.start(jobId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(jobId.toString());
+ }
+ hcc.waitForCompletion(jobId);
+ dumpOutputFiles();
+ }
+
+ private void dumpOutputFiles() {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ for (File f : outputFiles) {
+ if (f.exists() && f.isFile()) {
+ try {
+ LOGGER.info("Reading file: " + f.getAbsolutePath() + " in test: " + getClass().getName());
+ String data = FileUtils.readFileToString(f);
+ LOGGER.info(data);
+ } catch (IOException e) {
+ LOGGER.info("Error reading file: " + f.getAbsolutePath());
+ LOGGER.info(e.getMessage());
+ }
+ }
+ }
+ }
+ }
+
+ protected File createTempFile() throws IOException {
+ File tempFile = File.createTempFile(getClass().getName(), ".tmp", outputFolder.getRoot());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Output file: " + tempFile.getAbsolutePath());
+ }
+ outputFiles.add(tempFile);
+ return tempFile;
+ }
+
+}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AggregationTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AggregationTest.java
new file mode 100644
index 0000000..53a3f79
--- /dev/null
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/AggregationTest.java
@@ -0,0 +1,1269 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.tests.integration;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.normalizers.UTF8StringNormalizedKeyComputerFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.FloatParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableTableFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.AvgFieldGroupAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.AvgFieldMergeAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.FloatSumFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.IntSumFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MinMaxStringFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
+
+/**
+ *
+ */
+public class AggregationTest extends AbstractIntegrationTest {
+
+ final IFileSplitProvider splitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
+ "data/tpch0.001/lineitem.tbl"))) });
+
+ final RecordDescriptor desc = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
+
+ final ITupleParserFactory tupleParserFactory = new DelimitedDataTupleParserFactory(
+ new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
+ IntegerParserFactory.INSTANCE,
+ IntegerParserFactory.INSTANCE,
+ IntegerParserFactory.INSTANCE,
+ IntegerParserFactory.INSTANCE, FloatParserFactory.INSTANCE,
+ FloatParserFactory.INSTANCE, FloatParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, }, '|');
+
+ private AbstractSingleActivityOperatorDescriptor getPrinter(
+ JobSpecification spec, String prefix) throws IOException {
+
+ AbstractSingleActivityOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(
+ spec, new ConstantFileSplitProvider(new FileSplit[] {
+ new FileSplit(NC1_ID, createTempFile()
+ .getAbsolutePath()),
+ new FileSplit(NC2_ID, createTempFile()
+ .getAbsolutePath()) }), "\t");
+
+ return printer;
+ }
+
+ @Test
+ public void singleKeySumInmemGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 0 };
+ int tableSize = 8;
+
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new IntSumFieldAggregatorFactory(3, true),
+ new FloatSumFieldAggregatorFactory(5, true) }),
+ outputRec, tableSize);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "singleKeySumInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void singleKeySumPreClusterGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE});
+
+ int[] keyFields = new int[] { 0 };
+
+ PreclusteredGroupOperatorDescriptor grouper = new PreclusteredGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new IntSumFieldAggregatorFactory(3, true),
+ new FloatSumFieldAggregatorFactory(5, true)}),
+ outputRec);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "singleKeySumInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void singleKeySumExtGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE});
+
+ int[] keyFields = new int[] { 0 };
+ int frameLimits = 4;
+ int tableSize = 8;
+
+ ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ frameLimits,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new UTF8StringNormalizedKeyComputerFactory(),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, false),
+ new IntSumFieldAggregatorFactory(3, false),
+ new FloatSumFieldAggregatorFactory(5, false)}),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, false),
+ new IntSumFieldAggregatorFactory(2, false),
+ new FloatSumFieldAggregatorFactory(3, false)}),
+ outputRec,
+ new HashSpillableTableFactory(
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ tableSize), true);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "singleKeySumExtGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void singleKeyAvgInmemGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 0 };
+ int tableSize = 8;
+
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new CountFieldAggregatorFactory(true),
+ new AvgFieldGroupAggregatorFactory(1, true) }),
+ outputRec, tableSize);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "singleKeyAvgInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void singleKeyAvgPreClusterGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 0 };
+
+ PreclusteredGroupOperatorDescriptor grouper = new PreclusteredGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new CountFieldAggregatorFactory(true),
+ new AvgFieldGroupAggregatorFactory(1, true) }),
+ outputRec);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "singleKeyAvgInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void singleKeyAvgExtGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 0 };
+ int frameLimits = 4;
+ int tableSize = 8;
+
+ ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ frameLimits,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new UTF8StringNormalizedKeyComputerFactory(),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, false),
+ new CountFieldAggregatorFactory(false),
+ new AvgFieldGroupAggregatorFactory(1, false) }),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, false),
+ new IntSumFieldAggregatorFactory(2, false),
+ new AvgFieldMergeAggregatorFactory(3, false) }),
+ outputRec,
+ new HashSpillableTableFactory(
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ tableSize), true);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "singleKeyAvgExtGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void singleKeyMinMaxStringInmemGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 0 };
+ int tableSize = 8;
+
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new MinMaxStringFieldAggregatorFactory(15,
+ true, false) }), outputRec, tableSize);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "singleKeyAvgInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void singleKeyMinMaxStringPreClusterGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 0 };
+
+ PreclusteredGroupOperatorDescriptor grouper = new PreclusteredGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new MinMaxStringFieldAggregatorFactory(15,
+ true, false) }), outputRec);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "singleKeyAvgInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void singleKeyMinMaxStringExtGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 0 };
+ int frameLimits = 4;
+ int tableSize = 8;
+
+ ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ frameLimits,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new UTF8StringNormalizedKeyComputerFactory(),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, false),
+ new MinMaxStringFieldAggregatorFactory(15,
+ true, true) }),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, false),
+ new MinMaxStringFieldAggregatorFactory(2, true,
+ true) }),
+ outputRec,
+ new HashSpillableTableFactory(
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ tableSize), true);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "singleKeyAvgExtGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void multiKeySumInmemGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 8, 0 };
+ int tableSize = 8;
+
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
+ spec, keyFields, new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new IntSumFieldAggregatorFactory(3, true) }),
+ outputRec, tableSize);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec, new FieldHashPartitionComputerFactory(keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "multiKeySumInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void multiKeySumPreClusterGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 8, 0 };
+
+ PreclusteredGroupOperatorDescriptor grouper = new PreclusteredGroupOperatorDescriptor(
+ spec, keyFields,
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new IntSumFieldAggregatorFactory(3, true) }),
+ outputRec);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec, new FieldHashPartitionComputerFactory(keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "multiKeySumInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void multiKeySumExtGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 8, 0 };
+ int frameLimits = 4;
+ int tableSize = 8;
+
+ ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ frameLimits,
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new UTF8StringNormalizedKeyComputerFactory(),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, false),
+ new IntSumFieldAggregatorFactory(3, false) }),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(2, false),
+ new IntSumFieldAggregatorFactory(3, false) }),
+ outputRec,
+ new HashSpillableTableFactory(
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ tableSize), true);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec, new FieldHashPartitionComputerFactory(keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "multiKeySumExtGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void multiKeyAvgInmemGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 8, 0 };
+ int tableSize = 8;
+
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
+ spec, keyFields, new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new CountFieldAggregatorFactory(true),
+ new AvgFieldGroupAggregatorFactory(1, true) }),
+ outputRec, tableSize);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec, new FieldHashPartitionComputerFactory(keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "multiKeyAvgInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void multiKeyAvgPreClusterGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 8, 0 };
+
+ PreclusteredGroupOperatorDescriptor grouper = new PreclusteredGroupOperatorDescriptor(
+ spec, keyFields,
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new CountFieldAggregatorFactory(true),
+ new AvgFieldGroupAggregatorFactory(1, true) }),
+ outputRec);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec, new FieldHashPartitionComputerFactory(keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "multiKeyAvgInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void multiKeyAvgExtGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 8, 0 };
+ int frameLimits = 4;
+ int tableSize = 8;
+
+ ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ frameLimits,
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new UTF8StringNormalizedKeyComputerFactory(),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, false),
+ new CountFieldAggregatorFactory(false),
+ new AvgFieldGroupAggregatorFactory(1, false) }),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(2, false),
+ new IntSumFieldAggregatorFactory(3, false),
+ new AvgFieldMergeAggregatorFactory(4, false) }),
+ outputRec,
+ new HashSpillableTableFactory(
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ tableSize), true);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec, new FieldHashPartitionComputerFactory(keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "multiKeyAvgExtGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void multiKeyMinMaxStringInmemGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 8, 0 };
+ int tableSize = 8;
+
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
+ spec, keyFields, new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new MinMaxStringFieldAggregatorFactory(15,
+ true, false) }), outputRec, tableSize);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec, new FieldHashPartitionComputerFactory(keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "multiKeyMinMaxStringInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void multiKeyMinMaxStringPreClusterGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 8, 0 };
+
+ PreclusteredGroupOperatorDescriptor grouper = new PreclusteredGroupOperatorDescriptor(
+ spec, keyFields,
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new MinMaxStringFieldAggregatorFactory(15,
+ true, false) }), outputRec);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec, new FieldHashPartitionComputerFactory(keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "multiKeyMinMaxStringPreClusterGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @Test
+ public void multiKeyMinMaxStringExtGroupTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, NC2_ID);
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 8, 0 };
+ int frameLimits = 4;
+ int tableSize = 8;
+
+ ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ frameLimits,
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new UTF8StringNormalizedKeyComputerFactory(),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, false),
+ new MinMaxStringFieldAggregatorFactory(15,
+ true, true) }),
+ new MultiFieldsAggregatorFactory(new int[] { 0, 1 },
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(2, false),
+ new MinMaxStringFieldAggregatorFactory(3, true,
+ true) }),
+ outputRec,
+ new HashSpillableTableFactory(
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ tableSize), true);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(
+ spec, new FieldHashPartitionComputerFactory(keyFields,
+ new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "multiKeyMinMaxStringExtGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC2_ID, NC1_ID);
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java
index 0a0df60..ce72ec5 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/CountOfCountsTest.java
@@ -20,23 +20,22 @@
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.CountAggregatorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IFieldValueResultingAggregatorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.MultiAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNReplicatingConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
@@ -45,8 +44,11 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
@@ -68,7 +70,8 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC1_ID);
InMemorySortOperatorDescriptor sorter = new InMemorySortOperatorDescriptor(spec, new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, desc);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ desc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
RecordDescriptor desc2 = new RecordDescriptor(new ISerializerDeserializer[] {
@@ -76,28 +79,31 @@
PreclusteredGroupOperatorDescriptor group = new PreclusteredGroupOperatorDescriptor(
spec,
new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- new MultiAggregatorFactory(new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }),
desc2);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, group, NC1_ID);
InMemorySortOperatorDescriptor sorter2 = new InMemorySortOperatorDescriptor(spec, new int[] { 1 },
- new IBinaryComparatorFactory[] { IntegerBinaryComparatorFactory.INSTANCE }, desc2);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, desc2);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter2, NC1_ID);
RecordDescriptor desc3 = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
PreclusteredGroupOperatorDescriptor group2 = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 1 },
- new IBinaryComparatorFactory[] { IntegerBinaryComparatorFactory.INSTANCE }, new MultiAggregatorFactory(
- new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }), desc3);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }), desc3);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, group2, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(conn1, csvScanner, 0, sorter, 0);
IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
@@ -105,7 +111,8 @@
IConnectorDescriptor conn3 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(conn3, group, 0, sorter2, 0);
IConnectorDescriptor conn4 = new OneToOneConnectorDescriptor(spec);
@@ -135,7 +142,8 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC1_ID);
InMemorySortOperatorDescriptor sorter = new InMemorySortOperatorDescriptor(spec, new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, desc);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ desc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID, NC2_ID, NC1_ID, NC2_ID);
RecordDescriptor desc2 = new RecordDescriptor(new ISerializerDeserializer[] {
@@ -143,28 +151,31 @@
PreclusteredGroupOperatorDescriptor group = new PreclusteredGroupOperatorDescriptor(
spec,
new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- new MultiAggregatorFactory(new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }),
desc2);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, group, NC1_ID, NC2_ID, NC1_ID, NC2_ID);
InMemorySortOperatorDescriptor sorter2 = new InMemorySortOperatorDescriptor(spec, new int[] { 1 },
- new IBinaryComparatorFactory[] { IntegerBinaryComparatorFactory.INSTANCE }, desc2);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, desc2);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter2, NC1_ID, NC2_ID);
RecordDescriptor desc3 = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
PreclusteredGroupOperatorDescriptor group2 = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 1 },
- new IBinaryComparatorFactory[] { IntegerBinaryComparatorFactory.INSTANCE }, new MultiAggregatorFactory(
- new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }), desc3);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }), desc3);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, group2, NC1_ID, NC2_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(conn1, csvScanner, 0, sorter, 0);
IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
@@ -172,7 +183,8 @@
IConnectorDescriptor conn3 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(conn3, group, 0, sorter2, 0);
IConnectorDescriptor conn4 = new OneToOneConnectorDescriptor(spec);
@@ -202,7 +214,8 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC1_ID);
ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 3, new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, desc);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ desc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID, NC2_ID, NC1_ID, NC2_ID);
RecordDescriptor desc2 = new RecordDescriptor(new ISerializerDeserializer[] {
@@ -210,28 +223,31 @@
PreclusteredGroupOperatorDescriptor group = new PreclusteredGroupOperatorDescriptor(
spec,
new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- new MultiAggregatorFactory(new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }),
desc2);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, group, NC1_ID, NC2_ID, NC1_ID, NC2_ID);
InMemorySortOperatorDescriptor sorter2 = new InMemorySortOperatorDescriptor(spec, new int[] { 1 },
- new IBinaryComparatorFactory[] { IntegerBinaryComparatorFactory.INSTANCE }, desc2);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, desc2);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter2, NC1_ID, NC2_ID);
RecordDescriptor desc3 = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
PreclusteredGroupOperatorDescriptor group2 = new PreclusteredGroupOperatorDescriptor(spec, new int[] { 1 },
- new IBinaryComparatorFactory[] { IntegerBinaryComparatorFactory.INSTANCE }, new MultiAggregatorFactory(
- new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }), desc3);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }), desc3);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, group2, NC1_ID, NC2_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(conn1, csvScanner, 0, sorter, 0);
IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
@@ -239,7 +255,8 @@
IConnectorDescriptor conn3 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(conn3, group, 0, sorter2, 0);
IConnectorDescriptor conn4 = new OneToOneConnectorDescriptor(spec);
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/LocalityAwareConnectorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/LocalityAwareConnectorTest.java
new file mode 100644
index 0000000..a7a0a73
--- /dev/null
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/LocalityAwareConnectorTest.java
@@ -0,0 +1,254 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.tests.integration;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.BitSet;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.FloatParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.connectors.GlobalHashingLocalityMap;
+import edu.uci.ics.hyracks.dataflow.std.connectors.HashtableLocalityMap;
+import edu.uci.ics.hyracks.dataflow.std.connectors.LocalityAwareMToNPartitioningConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.FloatSumFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.IntSumFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
+
+public class LocalityAwareConnectorTest extends AbstractMultiNCIntegrationTest {
+
+ final IFileSplitProvider splitProvider = new ConstantFileSplitProvider(new FileSplit[] {
+ new FileSplit("asterix-001", new FileReference(new File("data/tpch0.001/lineitem.tbl"))),
+ new FileSplit("asterix-002", new FileReference(new File("data/tpch0.001/lineitem.tbl"))),
+ new FileSplit("asterix-003", new FileReference(new File("data/tpch0.001/lineitem.tbl"))),
+ new FileSplit("asterix-004", new FileReference(new File("data/tpch0.001/lineitem.tbl"))) });
+
+ final RecordDescriptor desc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+
+ final ITupleParserFactory tupleParserFactory = new DelimitedDataTupleParserFactory(new IValueParserFactory[] {
+ UTF8StringParserFactory.INSTANCE, IntegerParserFactory.INSTANCE, IntegerParserFactory.INSTANCE,
+ IntegerParserFactory.INSTANCE, IntegerParserFactory.INSTANCE, FloatParserFactory.INSTANCE,
+ FloatParserFactory.INSTANCE, FloatParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, }, '|');
+
+ /**
+ * Test of aggregations using locality aware connector. The output two files should be the
+ * same, each of which is the aggregation of two copies of the lineitem.tbl.
+ *
+ * Note that if the hashing connector is not working correctly, the two files may be different. This
+ * also means that even the output files are the same, the hashing may have other problems.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void localityAwareAggregationTest() throws Exception {
+
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, "asterix-001", "asterix-002", "asterix-003", "asterix-004");
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 0 };
+ int tableSize = 8;
+
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new IntSumFieldAggregatorFactory(3, true),
+ new FloatSumFieldAggregatorFactory(5, true) }),
+ outputRec, tableSize);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ "asterix-005", "asterix-006");
+
+ BitSet nodemap = new BitSet(8);
+
+ nodemap.set(0);
+ nodemap.set(2);
+ nodemap.set(5);
+ nodemap.set(7);
+
+ IConnectorDescriptor conn1 = new LocalityAwareMToNPartitioningConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }), new HashtableLocalityMap(nodemap));
+
+ new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "localityAwareSumInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ "asterix-005", "asterix-006");
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ /**
+ * Test for locality aware connector, using the global hashing node mapper. This should have
+ * the exactly the same result as using {@link MToNPartitioningConnectorDescriptor}.
+ *
+ * @throws Exception
+ */
+ @Test
+ public void globalPartitionAggregationTest() throws Exception {
+
+ JobSpecification spec = new JobSpecification();
+
+ FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(
+ spec, splitProvider, tupleParserFactory, desc);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ csvScanner, "asterix-001", "asterix-002", "asterix-003", "asterix-004");
+
+ RecordDescriptor outputRec = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE,
+ FloatSerializerDeserializer.INSTANCE });
+
+ int[] keyFields = new int[] { 0 };
+ int tableSize = 8;
+
+ HashGroupOperatorDescriptor grouper = new HashGroupOperatorDescriptor(
+ spec,
+ keyFields,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] {
+ new IntSumFieldAggregatorFactory(1, true),
+ new IntSumFieldAggregatorFactory(3, true),
+ new FloatSumFieldAggregatorFactory(5, true) }),
+ outputRec, tableSize);
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper,
+ "asterix-005", "asterix-006");
+
+ IConnectorDescriptor conn1 = new LocalityAwareMToNPartitioningConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }), new GlobalHashingLocalityMap());
+
+ new MToNPartitioningConnectorDescriptor(
+ spec,
+ new FieldHashPartitionComputerFactory(
+ keyFields,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(conn1, csvScanner, 0, grouper, 0);
+
+ AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec,
+ "localityAwareSumInmemGroupTest");
+
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ "asterix-005", "asterix-006");
+
+ IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(conn2, grouper, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ private AbstractSingleActivityOperatorDescriptor getPrinter(
+ JobSpecification spec, String prefix) throws IOException {
+
+ AbstractSingleActivityOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(
+ spec, new ConstantFileSplitProvider(new FileSplit[] {
+ new FileSplit("asterix-005", createTempFile()
+ .getAbsolutePath()),
+ new FileSplit("asterix-006", createTempFile()
+ .getAbsolutePath()) }), "\t");
+
+ return printer;
+ }
+
+}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/OptimizedSortMergeTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/OptimizedSortMergeTest.java
index 91214c8..c2d3ebf 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/OptimizedSortMergeTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/OptimizedSortMergeTest.java
@@ -15,7 +15,6 @@
package edu.uci.ics.hyracks.tests.integration;
import java.io.File;
-import java.io.PrintStream;
import org.junit.Test;
@@ -26,8 +25,9 @@
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -43,12 +43,9 @@
import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.OptimizedExternalSortOperatorDescriptor;
-
-//By Pouria
-
public class OptimizedSortMergeTest extends AbstractIntegrationTest {
-
- @Test
+
+ @Test
public void optimizedSortMergeTest01() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -71,9 +68,10 @@
UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID, NC2_ID);
- OptimizedExternalSortOperatorDescriptor sorter = new OptimizedExternalSortOperatorDescriptor(spec, 4, new int[] { 1, 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE,
- UTF8StringBinaryComparatorFactory.INSTANCE }, ordersDesc);
+ OptimizedExternalSortOperatorDescriptor sorter = new OptimizedExternalSortOperatorDescriptor(spec, 4,
+ new int[] { 1, 0 }, new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) }, ordersDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID, NC2_ID);
PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
@@ -81,18 +79,20 @@
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- spec.connect(new MToNPartitioningMergingConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
- new int[] { 1, 0 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE,
- UTF8StringBinaryHashFunctionFactory.INSTANCE }), new int[] { 1, 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE,
- UTF8StringBinaryComparatorFactory.INSTANCE }), sorter, 0, printer, 0);
+ spec.connect(
+ new MToNPartitioningMergingConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(new int[] {
+ 1, 0 }, new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) }), new int[] { 1, 0 },
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) }), sorter, 0,
+ printer, 0);
runTest(spec);
}
-
-
-
- @Test
+
+ @Test
public void optimizedSortMergeTest02() throws Exception {
JobSpecification spec = new JobSpecification();
@@ -114,30 +114,35 @@
UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID, NC2_ID);
-
+
int outputLimit = 200;
- OptimizedExternalSortOperatorDescriptor sorter = new OptimizedExternalSortOperatorDescriptor(spec, 4, outputLimit, new int[] { 1, 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE,
- UTF8StringBinaryComparatorFactory.INSTANCE }, ordersDesc);
+ OptimizedExternalSortOperatorDescriptor sorter = new OptimizedExternalSortOperatorDescriptor(spec, 4,
+ outputLimit, new int[] { 1, 0 }, new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) }, ordersDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID, NC2_ID);
-
+
LimitOperatorDescriptor filter = new LimitOperatorDescriptor(spec, ordersDesc, outputLimit);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, filter, NC1_ID);
-
+
PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- spec.connect(new MToNPartitioningMergingConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
- new int[] { 1, 0 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE,
- UTF8StringBinaryHashFunctionFactory.INSTANCE }), new int[] { 1, 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE,
- UTF8StringBinaryComparatorFactory.INSTANCE }), sorter, 0, filter, 0);
-
+ spec.connect(
+ new MToNPartitioningMergingConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(new int[] {
+ 1, 0 }, new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) }), new int[] { 1, 0 },
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) }), sorter, 0, filter,
+ 0);
+
spec.connect(new OneToOneConnectorDescriptor(spec), filter, 0, printer, 0);
runTest(spec);
}
-
+
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/ScanPrintTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/ScanPrintTest.java
index d343232..9355110 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/ScanPrintTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/ScanPrintTest.java
@@ -20,13 +20,15 @@
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.IntegerBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
@@ -40,7 +42,7 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
public class ScanPrintTest extends AbstractIntegrationTest {
@Test
@@ -61,7 +63,10 @@
desc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC2_ID, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] {
+ new FileSplit(NC2_ID, createTempFile().getAbsolutePath()),
+ new FileSplit(NC1_ID, createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC2_ID, NC1_ID);
IConnectorDescriptor conn = new OneToOneConnectorDescriptor(spec);
@@ -93,12 +98,15 @@
UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(conn1, ordScanner, 0, printer, 0);
spec.addRoot(printer);
@@ -127,12 +135,15 @@
UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { IntegerBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(IntegerPointable.FACTORY) }));
spec.connect(conn1, ordScanner, 0, printer, 0);
spec.addRoot(printer);
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/SortMergeTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/SortMergeTest.java
index 1b6ff14..2c3fddf 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/SortMergeTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/SortMergeTest.java
@@ -19,14 +19,16 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -38,7 +40,7 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
@@ -67,18 +69,23 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID, NC2_ID);
InMemorySortOperatorDescriptor sorter = new InMemorySortOperatorDescriptor(spec, new int[] { 1 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, ordersDesc);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ ordersDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID, NC2_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- spec.connect(new MToNPartitioningMergingConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
- new int[] { 1 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }),
- new int[] { 1 }, new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }),
- sorter, 0, printer, 0);
+ spec.connect(
+ new MToNPartitioningMergingConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
+ new int[] { 1 }, new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }), new int[] { 1 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) }), sorter, 0, printer, 0);
runTest(spec);
}
@@ -107,20 +114,26 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID, NC2_ID);
ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 4, new int[] { 1, 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE,
- UTF8StringBinaryComparatorFactory.INSTANCE }, ordersDesc);
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) }, ordersDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID, NC2_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- spec.connect(new MToNPartitioningMergingConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(
- new int[] { 1, 0 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE,
- UTF8StringBinaryHashFunctionFactory.INSTANCE }), new int[] { 1, 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE,
- UTF8StringBinaryComparatorFactory.INSTANCE }), sorter, 0, printer, 0);
+ spec.connect(
+ new MToNPartitioningMergingConnectorDescriptor(spec, new FieldHashPartitionComputerFactory(new int[] {
+ 1, 0 }, new IBinaryHashFunctionFactory[] {
+ PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) }), new int[] { 1, 0 },
+ new IBinaryComparatorFactory[] {
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) }), sorter, 0,
+ printer, 0);
runTest(spec);
}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOptimizedHybridHashJoinTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOptimizedHybridHashJoinTest.java
index 3820119..0999ee8 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOptimizedHybridHashJoinTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOptimizedHybridHashJoinTest.java
@@ -1,9 +1,6 @@
package edu.uci.ics.hyracks.tests.integration;
-import java.io.DataOutput;
import java.io.File;
-import java.io.IOException;
-
import org.junit.Test;
@@ -11,16 +8,14 @@
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionGeneratorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriter;
-import edu.uci.ics.hyracks.api.dataflow.value.INullWriterFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFamily;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionGeneratorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.UTF8StringBinaryHashFunctionFamily;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -35,37 +30,12 @@
import edu.uci.ics.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
-
public class TPCHCustomerOptimizedHybridHashJoinTest extends AbstractIntegrationTest {
private static final boolean DEBUG = false;
- static private class NoopNullWriterFactory implements INullWriterFactory {
-
- private static final long serialVersionUID = 1L;
- public static final NoopNullWriterFactory INSTANCE = new NoopNullWriterFactory();
-
- private NoopNullWriterFactory() {
- }
-
- @Override
- public INullWriter createNullWriter() {
- return new INullWriter() {
- @Override
- public void writeNull(DataOutput out) throws HyracksDataException {
- try {
- out.writeShort(0);
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
- };
- }
- }
-
-
@Test
public void customerOrderCIDHybridHashJoin_Case1() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
"data/tpch0.001/customer4.tbl"))) };
@@ -76,11 +46,9 @@
UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
-
FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
"data/tpch0.001/orders4.tbl"))) };
-
-
+
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
@@ -115,28 +83,24 @@
UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
-
-
-
- OptimizedHybridHashJoinOperatorDescriptor join = new OptimizedHybridHashJoinOperatorDescriptor(spec, 15, 243, 1.2,
- new int[] { 0 }, new int[] { 1 },
- new IBinaryHashFunctionGeneratorFactory[] { UTF8StringBinaryHashFunctionGeneratorFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- custOrderJoinDesc,
- new JoinComparatorFactory(UTF8StringBinaryComparatorFactory.INSTANCE, 0, 1),
- new JoinComparatorFactory(UTF8StringBinaryComparatorFactory.INSTANCE, 1, 0));
-
-
-
+
+ OptimizedHybridHashJoinOperatorDescriptor join = new OptimizedHybridHashJoinOperatorDescriptor(spec, 15, 243,
+ 1.2, new int[] { 0 }, new int[] { 1 },
+ new IBinaryHashFunctionFamily[] { UTF8StringBinaryHashFunctionFamily.INSTANCE },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, new JoinComparatorFactory(
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 0, 1),
+ new JoinComparatorFactory(PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 1, 0));
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
: new NullSinkOperatorDescriptor(spec);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
-
+
IConnectorDescriptor custJoinConn = new OneToOneConnectorDescriptor(spec);
spec.connect(custJoinConn, custScanner, 0, join, 0);
-
+
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
spec.connect(ordJoinConn, ordScanner, 0, join, 1);
@@ -146,11 +110,10 @@
spec.addRoot(printer);
runTest(spec);
}
-
@Test
public void customerOrderCIDHybridHashJoin_Case2() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
"data/tpch0.001/customer3.tbl"))) };
@@ -161,11 +124,9 @@
UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
-
FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
"data/tpch0.001/orders4.tbl"))) };
-
-
+
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
@@ -200,28 +161,24 @@
UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
-
-
-
- OptimizedHybridHashJoinOperatorDescriptor join = new OptimizedHybridHashJoinOperatorDescriptor(spec, 15, 122, 1.2,
- new int[] { 0 }, new int[] { 1 },
- new IBinaryHashFunctionGeneratorFactory[] { UTF8StringBinaryHashFunctionGeneratorFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- custOrderJoinDesc,
- new JoinComparatorFactory(UTF8StringBinaryComparatorFactory.INSTANCE, 0, 1),
- new JoinComparatorFactory(UTF8StringBinaryComparatorFactory.INSTANCE, 1, 0));
-
-
-
+
+ OptimizedHybridHashJoinOperatorDescriptor join = new OptimizedHybridHashJoinOperatorDescriptor(spec, 15, 122,
+ 1.2, new int[] { 0 }, new int[] { 1 },
+ new IBinaryHashFunctionFamily[] { UTF8StringBinaryHashFunctionFamily.INSTANCE },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, new JoinComparatorFactory(
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 0, 1),
+ new JoinComparatorFactory(PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 1, 0));
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
: new NullSinkOperatorDescriptor(spec);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
-
+
IConnectorDescriptor custJoinConn = new OneToOneConnectorDescriptor(spec);
spec.connect(custJoinConn, custScanner, 0, join, 0);
-
+
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
spec.connect(ordJoinConn, ordScanner, 0, join, 1);
@@ -231,11 +188,11 @@
spec.addRoot(printer);
runTest(spec);
}
-
+
@Test
public void customerOrderCIDHybridHashJoin_Case3() throws Exception {
-
- JobSpecification spec = new JobSpecification();
+
+ JobSpecification spec = new JobSpecification();
FileSplit[] custSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
"data/tpch0.001/customer3.tbl"))) };
@@ -246,12 +203,9 @@
UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
-
FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC2_ID, new FileReference(new File(
"data/tpch0.001/orders1.tbl"))) };
-
-
-
+
IFileSplitProvider ordersSplitsProvider = new ConstantFileSplitProvider(ordersSplits);
RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
@@ -286,28 +240,24 @@
UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
-
-
-
- OptimizedHybridHashJoinOperatorDescriptor join = new OptimizedHybridHashJoinOperatorDescriptor(spec, 6, 122, 1.2,
- new int[] { 0 }, new int[] { 1 },
- new IBinaryHashFunctionGeneratorFactory[] { UTF8StringBinaryHashFunctionGeneratorFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- custOrderJoinDesc,
- new JoinComparatorFactory(UTF8StringBinaryComparatorFactory.INSTANCE, 0, 1),
- new JoinComparatorFactory(UTF8StringBinaryComparatorFactory.INSTANCE, 1, 0));
-
-
-
+
+ OptimizedHybridHashJoinOperatorDescriptor join = new OptimizedHybridHashJoinOperatorDescriptor(spec, 6, 122,
+ 1.2, new int[] { 0 }, new int[] { 1 },
+ new IBinaryHashFunctionFamily[] { UTF8StringBinaryHashFunctionFamily.INSTANCE },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, new JoinComparatorFactory(
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 0, 1),
+ new JoinComparatorFactory(PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 1, 0));
+
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
: new NullSinkOperatorDescriptor(spec);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
-
+
IConnectorDescriptor custJoinConn = new OneToOneConnectorDescriptor(spec);
spec.connect(custJoinConn, custScanner, 0, join, 0);
-
+
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
spec.connect(ordJoinConn, ordScanner, 0, join, 1);
@@ -317,6 +267,5 @@
spec.addRoot(printer);
runTest(spec);
}
-
}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
index d417fda..61d4696 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderHashJoinTest.java
@@ -32,8 +32,9 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -46,17 +47,14 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.join.GraceHashJoinOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.join.HybridHashJoinOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.join.InMemoryHashJoinOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.MaterializingOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
public class TPCHCustomerOrderHashJoinTest extends AbstractIntegrationTest {
- private static final boolean DEBUG = true;
-
- static private class NoopNullWriterFactory implements INullWriterFactory {
+ private static class NoopNullWriterFactory implements INullWriterFactory {
private static final long serialVersionUID = 1L;
public static final NoopNullWriterFactory INSTANCE = new NoopNullWriterFactory();
@@ -141,13 +139,18 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
- InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(spec, new int[] { 1 },
- new int[] { 0 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, 128);
+ InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(
+ spec,
+ new int[] { 1 },
+ new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, 128);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
@@ -213,14 +216,22 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
- GraceHashJoinOperatorDescriptor join = new GraceHashJoinOperatorDescriptor(spec, 4, 10, 200, 1.2,
- new int[] { 1 }, new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc);
+ GraceHashJoinOperatorDescriptor join = new GraceHashJoinOperatorDescriptor(
+ spec,
+ 4,
+ 10,
+ 200,
+ 1.2,
+ new int[] { 1 },
+ new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
@@ -286,14 +297,22 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
- HybridHashJoinOperatorDescriptor join = new HybridHashJoinOperatorDescriptor(spec, 5, 20, 200, 1.2,
- new int[] { 1 }, new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc);
+ HybridHashJoinOperatorDescriptor join = new HybridHashJoinOperatorDescriptor(
+ spec,
+ 5,
+ 20,
+ 200,
+ 1.2,
+ new int[] { 1 },
+ new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
@@ -359,24 +378,23 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
- INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFields().length];
+ INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFieldCount()];
for (int j = 0; j < nullWriterFactories.length; j++) {
nullWriterFactories[j] = NoopNullWriterFactory.INSTANCE;
}
- InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(spec, new int[] { 0 },
- new int[] { 1 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, true,
- nullWriterFactories, 128);
+ InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(
+ spec,
+ new int[] { 0 },
+ new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, true, nullWriterFactories, 128);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
- // FileSplit[] custOrdersJoinSplits = new FileSplit[] { new
- // FileSplit(NC1_ID, new FileReference(new File(
- // "data/tpch0.001/custOrdersLeftOuterJoin.csv"))) };
- // LineFileWriteOperatorDescriptor printer = new
- // LineFileWriteOperatorDescriptor(spec, custOrdersJoinSplits);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
@@ -442,25 +460,27 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
- INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFields().length];
+ INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFieldCount()];
for (int j = 0; j < nullWriterFactories.length; j++) {
nullWriterFactories[j] = NoopNullWriterFactory.INSTANCE;
}
- GraceHashJoinOperatorDescriptor join = new GraceHashJoinOperatorDescriptor(spec, 5, 20, 200, 1.2,
- new int[] { 0 }, new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, true,
- nullWriterFactories);
+ GraceHashJoinOperatorDescriptor join = new GraceHashJoinOperatorDescriptor(
+ spec,
+ 5,
+ 20,
+ 200,
+ 1.2,
+ new int[] { 0 },
+ new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, true, nullWriterFactories);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
- // FileSplit[] custOrdersJoinSplits = new FileSplit[] { new
- // FileSplit(NC1_ID, new FileReference(new File(
- // "data/tpch0.001/custOrdersLeftOuterJoin.csv"))) };
- // LineFileWriteOperatorDescriptor printer = new
- // LineFileWriteOperatorDescriptor(spec, custOrdersJoinSplits);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
@@ -526,25 +546,27 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
- INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFields().length];
+ INullWriterFactory[] nullWriterFactories = new INullWriterFactory[ordersDesc.getFieldCount()];
for (int j = 0; j < nullWriterFactories.length; j++) {
nullWriterFactories[j] = NoopNullWriterFactory.INSTANCE;
}
- HybridHashJoinOperatorDescriptor join = new HybridHashJoinOperatorDescriptor(spec, 5, 20, 200, 1.2,
- new int[] { 0 }, new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, true,
- nullWriterFactories);
+ HybridHashJoinOperatorDescriptor join = new HybridHashJoinOperatorDescriptor(
+ spec,
+ 5,
+ 20,
+ 200,
+ 1.2,
+ new int[] { 0 },
+ new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, true, nullWriterFactories);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
- // FileSplit[] custOrdersJoinSplits = new FileSplit[] { new
- // FileSplit(NC1_ID, new FileReference(new File(
- // "data/tpch0.001/custOrdersLeftOuterJoin.csv"))) };
- // LineFileWriteOperatorDescriptor printer = new
- // LineFileWriteOperatorDescriptor(spec, custOrdersJoinSplits);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
@@ -612,23 +634,30 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID, NC2_ID);
- InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(spec, new int[] { 1 },
- new int[] { 0 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, 128);
+ InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(
+ spec,
+ new int[] { 1 },
+ new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, 128);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID, NC2_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(ordJoinConn, ordScanner, 0, join, 0);
IConnectorDescriptor custJoinConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(custJoinConn, custScanner, 0, join, 1);
IConnectorDescriptor joinPrinterConn = new MToNReplicatingConnectorDescriptor(spec);
@@ -690,24 +719,34 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID, NC2_ID);
- GraceHashJoinOperatorDescriptor join = new GraceHashJoinOperatorDescriptor(spec, 3, 20, 100, 1.2,
- new int[] { 1 }, new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc);
+ GraceHashJoinOperatorDescriptor join = new GraceHashJoinOperatorDescriptor(
+ spec,
+ 3,
+ 20,
+ 100,
+ 1.2,
+ new int[] { 1 },
+ new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID, NC2_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(ordJoinConn, ordScanner, 0, join, 0);
IConnectorDescriptor custJoinConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(custJoinConn, custScanner, 0, join, 1);
IConnectorDescriptor joinPrinterConn = new MToNReplicatingConnectorDescriptor(spec);
@@ -769,24 +808,34 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID, NC2_ID);
- HybridHashJoinOperatorDescriptor join = new HybridHashJoinOperatorDescriptor(spec, 3, 20, 100, 1.2,
- new int[] { 1 }, new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc);
+ HybridHashJoinOperatorDescriptor join = new HybridHashJoinOperatorDescriptor(
+ spec,
+ 3,
+ 20,
+ 100,
+ 1.2,
+ new int[] { 1 },
+ new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID, NC2_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(ordJoinConn, ordScanner, 0, join, 0);
IConnectorDescriptor custJoinConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(custJoinConn, custScanner, 0, join, 1);
IConnectorDescriptor joinPrinterConn = new MToNReplicatingConnectorDescriptor(spec);
@@ -848,23 +897,30 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID, NC2_ID);
- InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(spec, new int[] { 1 },
- new int[] { 0 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, 128);
+ InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(
+ spec,
+ new int[] { 1 },
+ new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, 128);
PartitionConstraintHelper.addPartitionCountConstraint(spec, join, 2);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(ordJoinConn, ordScanner, 0, join, 0);
IConnectorDescriptor custJoinConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(custJoinConn, custScanner, 0, join, 1);
IConnectorDescriptor joinPrinterConn = new MToNReplicatingConnectorDescriptor(spec);
@@ -932,23 +988,30 @@
MaterializingOperatorDescriptor custMat = new MaterializingOperatorDescriptor(spec, custDesc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custMat, NC1_ID, NC2_ID);
- InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(spec, new int[] { 1 },
- new int[] { 0 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc, 128);
+ InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(
+ spec,
+ new int[] { 1 },
+ new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory.of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, 128);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID, NC2_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordPartConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(ordPartConn, ordScanner, 0, ordMat, 0);
IConnectorDescriptor custPartConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(custPartConn, custScanner, 0, custMat, 0);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
index cab54f4..6411390 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/TPCHCustomerOrderNestedLoopJoinTest.java
@@ -30,7 +30,8 @@
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
@@ -41,14 +42,11 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.join.NestedLoopJoinOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.NullSinkOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
public class TPCHCustomerOrderNestedLoopJoinTest extends AbstractIntegrationTest {
- private static final boolean DEBUG = true;
-
- static class JoinComparatorFactory implements ITuplePairComparatorFactory {
+ private static class JoinComparatorFactory implements ITuplePairComparatorFactory {
private static final long serialVersionUID = 1L;
private final IBinaryComparatorFactory bFactory;
@@ -67,7 +65,7 @@
}
}
- static class JoinComparator implements ITuplePairComparator {
+ private static class JoinComparator implements ITuplePairComparator {
private final IBinaryComparator bComparator;
private final int field0;
@@ -166,11 +164,12 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID);
NestedLoopJoinOperatorDescriptor join = new NestedLoopJoinOperatorDescriptor(spec, new JoinComparatorFactory(
- UTF8StringBinaryComparatorFactory.INSTANCE, 1, 0), custOrderJoinDesc, 4);
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 1, 0), custOrderJoinDesc, 4);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
@@ -239,11 +238,12 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID, NC2_ID);
NestedLoopJoinOperatorDescriptor join = new NestedLoopJoinOperatorDescriptor(spec, new JoinComparatorFactory(
- UTF8StringBinaryComparatorFactory.INSTANCE, 1, 0), custOrderJoinDesc, 5);
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 1, 0), custOrderJoinDesc, 5);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, join, NC1_ID, NC2_ID);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
@@ -312,11 +312,12 @@
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, custScanner, NC1_ID, NC2_ID);
NestedLoopJoinOperatorDescriptor join = new NestedLoopJoinOperatorDescriptor(spec, new JoinComparatorFactory(
- UTF8StringBinaryComparatorFactory.INSTANCE, 1, 0), custOrderJoinDesc, 6);
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 1, 0), custOrderJoinDesc, 6);
PartitionConstraintHelper.addPartitionCountConstraint(spec, join, 2);
- IOperatorDescriptor printer = DEBUG ? new PrinterOperatorDescriptor(spec)
- : new NullSinkOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
IConnectorDescriptor ordJoinConn = new OneToOneConnectorDescriptor(spec);
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/UnionTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/UnionTest.java
index b1089fe..37b55b8 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/UnionTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/integration/UnionTest.java
@@ -19,6 +19,7 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
@@ -32,7 +33,7 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.union.UnionAllOperatorDescriptor;
public class UnionTest extends AbstractIntegrationTest {
@@ -64,7 +65,10 @@
UnionAllOperatorDescriptor unionAll = new UnionAllOperatorDescriptor(spec, 2, desc);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, unionAll, NC2_ID, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] {
+ new FileSplit(NC2_ID, createTempFile().getAbsolutePath()),
+ new FileSplit(NC1_ID, createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC2_ID, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), csvScanner01, 0, unionAll, 0);
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/BinaryTokenizerOperatorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/BinaryTokenizerOperatorTest.java
new file mode 100644
index 0000000..76a60ae
--- /dev/null
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/BinaryTokenizerOperatorTest.java
@@ -0,0 +1,74 @@
+package edu.uci.ics.hyracks.tests.invertedindex;
+
+import java.io.File;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizerFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.ITokenFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.UTF8WordTokenFactory;
+import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
+
+public class BinaryTokenizerOperatorTest extends AbstractIntegrationTest {
+
+ @Test
+ public void tokenizerTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
+
+ FileSplit[] dblpTitleFileSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/cleanednumbereddblptitles.txt"))) };
+ IFileSplitProvider dblpTitleSplitProvider = new ConstantFileSplitProvider(dblpTitleFileSplits);
+ RecordDescriptor dblpTitleRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+
+ FileScanOperatorDescriptor dblpTitleScanner = new FileScanOperatorDescriptor(spec, dblpTitleSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE }, '|'), dblpTitleRecDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, dblpTitleScanner, NC1_ID);
+
+ RecordDescriptor tokenizerRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+
+ ITokenFactory tokenFactory = new UTF8WordTokenFactory();
+ IBinaryTokenizerFactory tokenizerFactory = new DelimitedUTF8StringBinaryTokenizerFactory(true, false,
+ tokenFactory);
+ int[] tokenFields = { 1 };
+ int[] keyFields = { 0 };
+ BinaryTokenizerOperatorDescriptor binaryTokenizer = new BinaryTokenizerOperatorDescriptor(spec,
+ tokenizerRecDesc, tokenizerFactory, tokenFields, keyFields);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, binaryTokenizer, NC1_ID);
+
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), dblpTitleScanner, 0, binaryTokenizer, 0);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), binaryTokenizer, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/InvertedIndexOperatorsTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/InvertedIndexOperatorsTest.java
index c84ef74..2206a26 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/InvertedIndexOperatorsTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/InvertedIndexOperatorsTest.java
@@ -5,6 +5,7 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
@@ -20,7 +21,7 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizerFactory;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
@@ -57,7 +58,9 @@
tokenizerRecDesc, tokenizerFactory, tokenFields, projFields);
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, binaryTokenizer, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
spec.connect(new OneToOneConnectorDescriptor(spec), dblpTitleScanner, 0, binaryTokenizer, 0);
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/WordInvertedIndexTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/WordInvertedIndexTest.java
new file mode 100644
index 0000000..298c146
--- /dev/null
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/invertedindex/WordInvertedIndexTest.java
@@ -0,0 +1,316 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.tests.invertedindex;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
+import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifierFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.InvertedIndexBulkLoadOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.dataflow.InvertedIndexSearchOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.ConjunctiveSearchModifierFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.DelimitedUTF8StringBinaryTokenizerFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.ITokenFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.UTF8WordTokenFactory;
+import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
+import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
+
+public class WordInvertedIndexTest extends AbstractIntegrationTest {
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
+
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
+ private IIndexDataflowHelperFactory btreeDataflowHelperFactory = new BTreeDataflowHelperFactory();
+
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
+ private final String dateString = simpleDateFormat.format(new Date());
+ private final String primaryFileName = System.getProperty("java.io.tmpdir") + sep + "primaryBtree" + dateString;
+ private final String btreeFileName = System.getProperty("java.io.tmpdir") + sep + "invIndexBtree" + dateString;
+ private final String invListsFileName = System.getProperty("java.io.tmpdir") + sep + "invIndexLists" + dateString;
+
+ private IFileSplitProvider primaryFileSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
+ private IFileSplitProvider btreeFileSplitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(
+ NC1_ID, new FileReference(new File(btreeFileName))) });
+ private IFileSplitProvider invListsFileSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(invListsFileName))) });
+
+ // Primary BTree index.
+ private int primaryFieldCount = 2;
+ private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ private int primaryKeyFieldCount = 1;
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
+ private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ primaryTupleWriterFactory);
+ private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+
+ // Inverted index BTree dictionary.
+ private ITypeTraits[] tokenTypeTraits = new ITypeTraits[1];
+ private IBinaryComparatorFactory[] tokenComparatorFactories = new IBinaryComparatorFactory[1];
+
+ // Inverted index stuff.
+ private int invListElementFieldCount = 1;
+ private ITypeTraits[] invListsTypeTraits = new ITypeTraits[invListElementFieldCount];
+ private IBinaryComparatorFactory[] invListsComparatorFactories = new IBinaryComparatorFactory[invListElementFieldCount];
+ private RecordDescriptor tokenizerRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+ private RecordDescriptor invListsRecDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { IntegerSerializerDeserializer.INSTANCE });
+
+ // Tokenizer stuff.
+ private ITokenFactory tokenFactory = new UTF8WordTokenFactory();
+ private IBinaryTokenizerFactory tokenizerFactory = new DelimitedUTF8StringBinaryTokenizerFactory(true, false,
+ tokenFactory);
+
+ @Before
+ public void setup() throws Exception {
+ // Field declarations and comparators for primary BTree index.
+ primaryTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ // Field declarations and comparators for tokens.
+ tokenTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ tokenComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
+
+ // Field declarations and comparators for inverted lists.
+ invListsTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ invListsComparatorFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ loadPrimaryIndex();
+ printPrimaryIndex();
+ loadInvertedIndex();
+ }
+
+ @Test
+ public void testConjunctiveSearcher() throws Exception {
+ IInvertedIndexSearchModifierFactory conjunctiveSearchModifierFactory = new ConjunctiveSearchModifierFactory();
+ searchInvertedIndex("of", conjunctiveSearchModifierFactory);
+ searchInvertedIndex("3d", conjunctiveSearchModifierFactory);
+ searchInvertedIndex("of the human", conjunctiveSearchModifierFactory);
+ }
+
+ private IOperatorDescriptor createFileScanOp(JobSpecification spec) {
+ FileSplit[] dblpTitleFileSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/cleanednumbereddblptitles.txt"))) };
+ IFileSplitProvider dblpTitleSplitProvider = new ConstantFileSplitProvider(dblpTitleFileSplits);
+ RecordDescriptor dblpTitleRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ IntegerSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+ FileScanOperatorDescriptor dblpTitleScanner = new FileScanOperatorDescriptor(spec, dblpTitleSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { IntegerParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE }, '|'), dblpTitleRecDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, dblpTitleScanner, NC1_ID);
+ return dblpTitleScanner;
+ }
+
+ private IOperatorDescriptor createPrimaryBulkLoadOp(JobSpecification spec) {
+ int[] fieldPermutation = { 0, 1 };
+ TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, primaryFileSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ btreeDataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
+ return primaryBtreeBulkLoad;
+ }
+
+ private IOperatorDescriptor createScanKeyProviderOp(JobSpecification spec) throws HyracksDataException {
+ // build dummy tuple containing nothing
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
+ tb.addFieldEndOffset();
+ ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
+ return keyProviderOp;
+ }
+
+ private IOperatorDescriptor createPrimaryScanOp(JobSpecification spec) throws HyracksDataException {
+ int[] lowKeyFields = null; // - infinity
+ int[] highKeyFields = null; // + infinity
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
+ storageManager, indexRegistryProvider, primaryFileSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
+ highKeyFields, true, true, btreeDataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
+ return primaryBtreeSearchOp;
+ }
+
+ private void loadPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ // Assuming that the data is pre-sorted on the key. No need to sort
+ // before bulk load.
+ IOperatorDescriptor fileScanOp = createFileScanOp(spec);
+ IOperatorDescriptor primaryBulkLoad = createPrimaryBulkLoadOp(spec);
+ spec.connect(new OneToOneConnectorDescriptor(spec), fileScanOp, 0, primaryBulkLoad, 0);
+ spec.addRoot(primaryBulkLoad);
+ runTest(spec);
+ }
+
+ private void printPrimaryIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ IOperatorDescriptor keyProviderOp = createScanKeyProviderOp(spec);
+ IOperatorDescriptor primaryScanOp = createPrimaryScanOp(spec);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, printer, 0);
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ private IOperatorDescriptor createExternalSortOp(JobSpecification spec, int[] sortFields,
+ RecordDescriptor outputRecDesc) {
+ ExternalSortOperatorDescriptor externalSortOp = new ExternalSortOperatorDescriptor(spec, 1000, sortFields,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, outputRecDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, externalSortOp, NC1_ID);
+ return externalSortOp;
+ }
+
+ private IOperatorDescriptor createBinaryTokenizerOp(JobSpecification spec, int[] tokenFields, int[] keyFields) {
+ BinaryTokenizerOperatorDescriptor binaryTokenizer = new BinaryTokenizerOperatorDescriptor(spec,
+ tokenizerRecDesc, tokenizerFactory, tokenFields, keyFields);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, binaryTokenizer, NC1_ID);
+ return binaryTokenizer;
+ }
+
+ private IOperatorDescriptor createInvertedIndexBulkLoadOp(JobSpecification spec, int[] fieldPermutation) {
+ InvertedIndexBulkLoadOperatorDescriptor invIndexBulkLoadOp = new InvertedIndexBulkLoadOperatorDescriptor(spec,
+ fieldPermutation, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
+ indexRegistryProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
+ invListsComparatorFactories, btreeDataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, invIndexBulkLoadOp, NC1_ID);
+ return invIndexBulkLoadOp;
+ }
+
+ public void loadInvertedIndex() throws Exception {
+ JobSpecification spec = new JobSpecification();
+ IOperatorDescriptor keyProviderOp = createScanKeyProviderOp(spec);
+ IOperatorDescriptor primaryScanOp = createPrimaryScanOp(spec);
+ int[] tokenFields = { 1 };
+ int[] keyFields = { 0 };
+ IOperatorDescriptor binaryTokenizerOp = createBinaryTokenizerOp(spec, tokenFields, keyFields);
+ int[] sortFields = { 0, 1 };
+ IOperatorDescriptor externalSortOp = createExternalSortOp(spec, sortFields, tokenizerRecDesc);
+ int[] fieldPermutation = { 0, 1 };
+ IOperatorDescriptor invIndexBulkLoadOp = createInvertedIndexBulkLoadOp(spec, fieldPermutation);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, binaryTokenizerOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), binaryTokenizerOp, 0, externalSortOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), externalSortOp, 0, invIndexBulkLoadOp, 0);
+ spec.addRoot(invIndexBulkLoadOp);
+ runTest(spec);
+ }
+
+ private IOperatorDescriptor createQueryProviderOp(JobSpecification spec, String queryString)
+ throws HyracksDataException {
+ // Build tuple with exactly one field, which is the query,
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(1);
+ DataOutput dos = tb.getDataOutput();
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(queryString, dos);
+ tb.addFieldEndOffset();
+ ISerializerDeserializer[] querySerde = { UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor queryRecDesc = new RecordDescriptor(querySerde);
+ ConstantTupleSourceOperatorDescriptor queryProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ queryRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, queryProviderOp, NC1_ID);
+ return queryProviderOp;
+ }
+
+ private IOperatorDescriptor createInvertedIndexSearchOp(JobSpecification spec,
+ IInvertedIndexSearchModifierFactory searchModifierFactory) {
+ InvertedIndexSearchOperatorDescriptor invIndexSearchOp = new InvertedIndexSearchOperatorDescriptor(spec, 0,
+ storageManager, btreeFileSplitProvider, invListsFileSplitProvider, indexRegistryProvider,
+ tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits, invListsComparatorFactories,
+ btreeDataflowHelperFactory, tokenizerFactory, searchModifierFactory, invListsRecDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, invIndexSearchOp, NC1_ID);
+ return invIndexSearchOp;
+ }
+
+ public void searchInvertedIndex(String queryString, IInvertedIndexSearchModifierFactory searchModifierFactory)
+ throws Exception {
+ JobSpecification spec = new JobSpecification();
+ IOperatorDescriptor queryProviderOp = createQueryProviderOp(spec, queryString);
+ IOperatorDescriptor invIndexSearchOp = createInvertedIndexSearchOp(spec, searchModifierFactory);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ spec.connect(new OneToOneConnectorDescriptor(spec), queryProviderOp, 0, invIndexSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), invIndexSearchOp, 0, printer, 0);
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+}
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexSearchOperatorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexSearchOperatorTest.java
index 1923c10..0c0df42 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexSearchOperatorTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexSearchOperatorTest.java
@@ -25,15 +25,17 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.DoubleBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.DoubleParserFactory;
@@ -45,193 +47,164 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.DoublePrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
-import edu.uci.ics.hyracks.test.support.TestTreeIndexRegistryProvider;
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
-public class RTreePrimaryIndexSearchOperatorTest extends
- AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20, 20);
- }
+public class RTreePrimaryIndexSearchOperatorTest extends AbstractIntegrationTest {
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
- private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
- private ITreeIndexOpHelperFactory opHelperFactory = new RTreeOpHelperFactory();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
+ private IIndexDataflowHelperFactory dataflowHelperFactory = new RTreeDataflowHelperFactory();
- private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
- "ddMMyy-hhmmssSS");
- private final static String sep = System.getProperty("file.separator");
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
- // field, type and key declarations for primary R-tree index
- private int primaryFieldCount = 5;
- private int primaryKeyFieldCount = 4;
- private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private IPrimitiveValueProviderFactory[] primaryValueProviderFactories = new IPrimitiveValueProviderFactory[primaryKeyFieldCount];
+ // field, type and key declarations for primary R-tree index
+ private int primaryFieldCount = 5;
+ private int primaryKeyFieldCount = 4;
+ private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private RTreeTypeAwareTupleWriterFactory primaryTupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- primaryTypeTraits);
+ private RTreeTypeAwareTupleWriterFactory primaryTupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
+ primaryTypeTraits);
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory, primaryKeyFieldCount);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(
- primaryTupleWriterFactory, primaryKeyFieldCount);
+ private ITreeIndexFrameFactory primaryInteriorFrameFactory;
+ private ITreeIndexFrameFactory primaryLeafFrameFactory;
- private static String primaryRTreeName = "primary"
- + simpleDateFormat.format(new Date());
- private static String primaryFileName = System
- .getProperty("java.io.tmpdir") + sep + primaryRTreeName;
+ private static String primaryRTreeName = "primary" + simpleDateFormat.format(new Date());
+ private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryRTreeName;
- private IFileSplitProvider primaryRTreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- primaryFileName))) });
+ private IFileSplitProvider primaryRTreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
- @Before
- public void setup() throws Exception {
- // field, type and key declarations for primary R-tree index
- primaryTypeTraits[0] = new TypeTrait(8);
- primaryTypeTraits[1] = new TypeTrait(8);
- primaryTypeTraits[2] = new TypeTrait(8);
- primaryTypeTraits[3] = new TypeTrait(8);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryComparatorFactories[0] = DoubleBinaryComparatorFactory.INSTANCE;
- primaryComparatorFactories[1] = primaryComparatorFactories[0];
- primaryComparatorFactories[2] = primaryComparatorFactories[0];
- primaryComparatorFactories[3] = primaryComparatorFactories[0];
- primaryValueProviderFactories[0] = DoublePrimitiveValueProviderFactory.INSTANCE;
- primaryValueProviderFactories[1] = primaryValueProviderFactories[0];
- primaryValueProviderFactories[2] = primaryValueProviderFactories[0];
- primaryValueProviderFactories[3] = primaryValueProviderFactories[0];
+ @Before
+ public void setup() throws Exception {
+ // field, type and key declarations for primary R-tree index
+ primaryTypeTraits[0] = DoublePointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = DoublePointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = DoublePointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = DoublePointable.TYPE_TRAITS;
+ primaryTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
+ primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ primaryComparatorFactories[1] = primaryComparatorFactories[0];
+ primaryComparatorFactories[2] = primaryComparatorFactories[0];
+ primaryComparatorFactories[3] = primaryComparatorFactories[0];
- loadPrimaryIndexTest();
- }
+ IPrimitiveValueProviderFactory[] primaryValueProviderFactories = RTreeUtils
+ .createPrimitiveValueProviderFactories(primaryComparatorFactories.length, DoublePointable.FACTORY);
- public void loadPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ primaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(primaryTupleWriterFactory,
+ primaryValueProviderFactories);
+ primaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(primaryTupleWriterFactory, primaryValueProviderFactories);
- FileSplit[] objectsSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/spatial.txt"))) };
- IFileSplitProvider objectsSplitProvider = new ConstantFileSplitProvider(
- objectsSplits);
- RecordDescriptor objectsDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ loadPrimaryIndexTest();
+ }
- FileScanOperatorDescriptor objScanner = new FileScanOperatorDescriptor(
- spec, objectsSplitProvider,
- new DelimitedDataTupleParserFactory(new IValueParserFactory[] {
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE }, '|'), objectsDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- objScanner, NC1_ID);
+ public void loadPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- int[] fieldPermutation = { 0, 1, 2, 3, 4 };
- TreeIndexBulkLoadOperatorDescriptor primaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryRTreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, primaryValueProviderFactories,
- fieldPermutation, 0.7f, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryRTreeBulkLoad, NC1_ID);
+ FileSplit[] objectsSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/spatial.txt"))) };
+ IFileSplitProvider objectsSplitProvider = new ConstantFileSplitProvider(objectsSplits);
+ RecordDescriptor objectsDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- spec.connect(new OneToOneConnectorDescriptor(spec), objScanner, 0,
- primaryRTreeBulkLoad, 0);
+ FileScanOperatorDescriptor objScanner = new FileScanOperatorDescriptor(spec, objectsSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { DoubleParserFactory.INSTANCE,
+ DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE }, '|'), objectsDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, objScanner, NC1_ID);
- spec.addRoot(primaryRTreeBulkLoad);
- runTest(spec);
- }
+ int[] fieldPermutation = { 0, 1, 2, 3, 4 };
+ TreeIndexBulkLoadOperatorDescriptor primaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, primaryRTreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryRTreeBulkLoad, NC1_ID);
- @Test
- public void searchPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ spec.connect(new OneToOneConnectorDescriptor(spec), objScanner, 0, primaryRTreeBulkLoad, 0);
- // build tuple
- ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount);
- DataOutput dos = tb.getDataOutput();
+ spec.addRoot(primaryRTreeBulkLoad);
+ runTest(spec);
+ }
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(61.2894, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(-149.624, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(61.8894, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(-149.024, dos);
- tb.addFieldEndOffset();
+ @Test
+ public void searchPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- ISerializerDeserializer[] keyRecDescSers = {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ // build tuple
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount);
+ DataOutput dos = tb.getDataOutput();
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
- spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
- tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- keyProviderOp, NC1_ID);
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(61.2894, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(-149.624, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(61.8894, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(-149.024, dos);
+ tb.addFieldEndOffset();
- int[] keyFields = { 0, 1, 2, 3 };
+ ISerializerDeserializer[] keyRecDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- RTreeSearchOperatorDescriptor primaryRTreeSearchOp = new RTreeSearchOperatorDescriptor(
- spec, primaryRecDesc, storageManager,
- treeIndexRegistryProvider, primaryRTreeSplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory,
- primaryTypeTraits, primaryComparatorFactories,
- primaryValueProviderFactories, keyFields, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryRTreeSearchOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
- NC1_ID);
+ int[] keyFields = { 0, 1, 2, 3 };
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
- primaryRTreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- primaryRTreeSearchOp, 0, printer, 0);
+ RTreeSearchOperatorDescriptor primaryRTreeSearchOp = new RTreeSearchOperatorDescriptor(spec, primaryRecDesc,
+ storageManager, indexRegistryProvider, primaryRTreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, keyFields,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryRTreeSearchOp, NC1_ID);
- spec.addRoot(printer);
- runTest(spec);
- }
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
- @AfterClass
- public static void cleanup() throws Exception {
- File primary = new File(primaryFileName);
- primary.deleteOnExit();
- }
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryRTreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryRTreeSearchOp, 0, printer, 0);
+
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primary = new File(primaryFileName);
+ primary.deleteOnExit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexStatsOperatorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexStatsOperatorTest.java
index b6b4e20..a4912e8 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexStatsOperatorTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreePrimaryIndexStatsOperatorTest.java
@@ -24,14 +24,15 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.DoubleBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.DoubleParserFactory;
@@ -43,151 +44,135 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexStatsOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.DoublePrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
-import edu.uci.ics.hyracks.test.support.TestTreeIndexRegistryProvider;
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
public class RTreePrimaryIndexStatsOperatorTest extends AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20, 20);
- }
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
- private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
- private ITreeIndexOpHelperFactory opHelperFactory = new RTreeOpHelperFactory();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
+ private IIndexDataflowHelperFactory dataflowHelperFactory = new RTreeDataflowHelperFactory();
- private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
- "ddMMyy-hhmmssSS");
- private final static String sep = System.getProperty("file.separator");
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
- // field, type and key declarations for primary R-tree index
- private int primaryFieldCount = 5;
- private int primaryKeyFieldCount = 4;
- private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private IPrimitiveValueProviderFactory[] primaryValueProviderFactories = new IPrimitiveValueProviderFactory[primaryKeyFieldCount];
+ // field, type and key declarations for primary R-tree index
+ private int primaryFieldCount = 5;
+ private int primaryKeyFieldCount = 4;
+ private ITypeTraits[] primaryTypeTraits = new ITypeTraits[primaryFieldCount];
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private RTreeTypeAwareTupleWriterFactory primaryTupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- primaryTypeTraits);
+ private RTreeTypeAwareTupleWriterFactory primaryTupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
+ primaryTypeTraits);
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory, primaryKeyFieldCount);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(
- primaryTupleWriterFactory, primaryKeyFieldCount);
+ private ITreeIndexFrameFactory primaryInteriorFrameFactory;
+ private ITreeIndexFrameFactory primaryLeafFrameFactory;
- private static String primaryRTreeName = "primary"
- + simpleDateFormat.format(new Date());
- private static String primaryFileName = System
- .getProperty("java.io.tmpdir") + sep + primaryRTreeName;
+ private static String primaryRTreeName = "primary" + simpleDateFormat.format(new Date());
+ private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryRTreeName;
- private IFileSplitProvider primaryRTreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- primaryFileName))) });
+ private IFileSplitProvider primaryRTreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
- @Before
- public void setup() throws Exception {
- // field, type and key declarations for primary R-tree index
- primaryTypeTraits[0] = new TypeTrait(8);
- primaryTypeTraits[1] = new TypeTrait(8);
- primaryTypeTraits[2] = new TypeTrait(8);
- primaryTypeTraits[3] = new TypeTrait(8);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryComparatorFactories[0] = DoubleBinaryComparatorFactory.INSTANCE;
- primaryComparatorFactories[1] = primaryComparatorFactories[0];
- primaryComparatorFactories[2] = primaryComparatorFactories[0];
- primaryComparatorFactories[3] = primaryComparatorFactories[0];
- primaryValueProviderFactories[0] = DoublePrimitiveValueProviderFactory.INSTANCE;
- primaryValueProviderFactories[1] = primaryValueProviderFactories[0];
- primaryValueProviderFactories[2] = primaryValueProviderFactories[0];
- primaryValueProviderFactories[3] = primaryValueProviderFactories[0];
+ @Before
+ public void setup() throws Exception {
+ // field, type and key declarations for primary R-tree index
+ primaryTypeTraits[0] = DoublePointable.TYPE_TRAITS;
+ primaryTypeTraits[1] = DoublePointable.TYPE_TRAITS;
+ primaryTypeTraits[2] = DoublePointable.TYPE_TRAITS;
+ primaryTypeTraits[3] = DoublePointable.TYPE_TRAITS;
+ primaryTypeTraits[4] = DoublePointable.TYPE_TRAITS;
+ primaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ primaryComparatorFactories[1] = primaryComparatorFactories[0];
+ primaryComparatorFactories[2] = primaryComparatorFactories[0];
+ primaryComparatorFactories[3] = primaryComparatorFactories[0];
- loadPrimaryIndexTest();
- }
+ IPrimitiveValueProviderFactory[] primaryValueProviderFactories = RTreeUtils
+ .createPrimitiveValueProviderFactories(primaryComparatorFactories.length, DoublePointable.FACTORY);
- public void loadPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ primaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(primaryTupleWriterFactory,
+ primaryValueProviderFactories);
+ primaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(primaryTupleWriterFactory, primaryValueProviderFactories);
- FileSplit[] objectsSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/spatial.txt"))) };
- IFileSplitProvider objectsSplitProvider = new ConstantFileSplitProvider(
- objectsSplits);
- RecordDescriptor objectsDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ loadPrimaryIndexTest();
+ }
- FileScanOperatorDescriptor objScanner = new FileScanOperatorDescriptor(
- spec, objectsSplitProvider,
- new DelimitedDataTupleParserFactory(new IValueParserFactory[] {
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE }, '|'), objectsDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- objScanner, NC1_ID);
+ public void loadPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- int[] fieldPermutation = { 0, 1, 2, 3, 4 };
- TreeIndexBulkLoadOperatorDescriptor primaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryRTreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, primaryValueProviderFactories,
- fieldPermutation, 0.7f, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryRTreeBulkLoad, NC1_ID);
+ FileSplit[] objectsSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/spatial.txt"))) };
+ IFileSplitProvider objectsSplitProvider = new ConstantFileSplitProvider(objectsSplits);
+ RecordDescriptor objectsDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- spec.connect(new OneToOneConnectorDescriptor(spec), objScanner, 0,
- primaryRTreeBulkLoad, 0);
+ FileScanOperatorDescriptor objScanner = new FileScanOperatorDescriptor(spec, objectsSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { DoubleParserFactory.INSTANCE,
+ DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE }, '|'), objectsDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, objScanner, NC1_ID);
- spec.addRoot(primaryRTreeBulkLoad);
- runTest(spec);
- }
+ int[] fieldPermutation = { 0, 1, 2, 3, 4 };
+ TreeIndexBulkLoadOperatorDescriptor primaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, primaryRTreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryRTreeBulkLoad, NC1_ID);
- @Test
- public void showPrimaryIndexStats() throws Exception {
- JobSpecification spec = new JobSpecification();
+ spec.connect(new OneToOneConnectorDescriptor(spec), objScanner, 0, primaryRTreeBulkLoad, 0);
- TreeIndexStatsOperatorDescriptor primaryStatsOp = new TreeIndexStatsOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryRTreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryStatsOp, NC1_ID);
+ spec.addRoot(primaryRTreeBulkLoad);
+ runTest(spec);
+ }
- spec.addRoot(primaryStatsOp);
- runTest(spec);
- }
+ @Test
+ public void showPrimaryIndexStats() throws Exception {
+ JobSpecification spec = new JobSpecification();
- @AfterClass
- public static void cleanup() throws Exception {
- File primary = new File(primaryFileName);
- primary.deleteOnExit();
- }
+ TreeIndexStatsOperatorDescriptor primaryStatsOp = new TreeIndexStatsOperatorDescriptor(spec, storageManager,
+ indexRegistryProvider, primaryRTreeSplitProvider, primaryInteriorFrameFactory, primaryLeafFrameFactory,
+ primaryTypeTraits, primaryComparatorFactories, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryStatsOp, NC1_ID);
+
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
+
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryStatsOp, 0, printer, 0);
+ spec.addRoot(printer);
+ runTest(spec);
+ }
+
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primary = new File(primaryFileName);
+ primary.deleteOnExit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeSecondaryIndexSearchOperatorTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeSecondaryIndexSearchOperatorTest.java
index cc92e97..025f675 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeSecondaryIndexSearchOperatorTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeSecondaryIndexSearchOperatorTest.java
@@ -25,16 +25,17 @@
import org.junit.Test;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.DoubleBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.DoubleParserFactory;
@@ -46,429 +47,280 @@
import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.misc.ConstantTupleSourceOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexBulkLoadOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.rtree.dataflow.RTreeSearchOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.rtree.impls.DoublePrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+import edu.uci.ics.hyracks.test.support.TestIndexRegistryProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestStorageManagerInterface;
-import edu.uci.ics.hyracks.test.support.TestTreeIndexRegistryProvider;
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
-public class RTreeSecondaryIndexSearchOperatorTest extends
- AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20, 20);
- }
+public class RTreeSecondaryIndexSearchOperatorTest extends AbstractIntegrationTest {
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
- private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
- private ITreeIndexOpHelperFactory opHelperFactory = new RTreeOpHelperFactory();
- private ITreeIndexOpHelperFactory bTreeopHelperFactory = new BTreeOpHelperFactory();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<IIndex> indexRegistryProvider = new TestIndexRegistryProvider();
+ private IIndexDataflowHelperFactory dataflowHelperFactory = new RTreeDataflowHelperFactory();
+ private IIndexDataflowHelperFactory btreeDataflowHelperFactory = new BTreeDataflowHelperFactory();
- private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
- "ddMMyy-hhmmssSS");
- private final static String sep = System.getProperty("file.separator");
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
- // field, type and key declarations for primary R-tree index
- private int primaryFieldCount = 5;
- private int primaryKeyFieldCount = 4;
- private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private IPrimitiveValueProviderFactory[] primaryValueProviderFactories = new IPrimitiveValueProviderFactory[primaryKeyFieldCount];
+ // field, type and key declarations for primary B-tree index
+ private int primaryBTreeFieldCount = 10;
+ private ITypeTraits[] primaryBTreeTypeTraits = new ITypeTraits[primaryBTreeFieldCount];
+ private int primaryBTreeKeyFieldCount = 1;
+ private IBinaryComparatorFactory[] primaryBTreeComparatorFactories = new IBinaryComparatorFactory[primaryBTreeKeyFieldCount];
+ private TypeAwareTupleWriterFactory primaryBTreeTupleWriterFactory = new TypeAwareTupleWriterFactory(
+ primaryBTreeTypeTraits);
+ private ITreeIndexFrameFactory primaryBTreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ primaryBTreeTupleWriterFactory);
+ private ITreeIndexFrameFactory primaryBTreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(
+ primaryBTreeTupleWriterFactory);
- private RTreeTypeAwareTupleWriterFactory primaryTupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- primaryTypeTraits);
+ private static String primaryBTreeName = "primaryBTree" + simpleDateFormat.format(new Date());
+ private static String primaryBTreeFileName = System.getProperty("java.io.tmpdir") + sep + primaryBTreeName;
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(
- primaryTupleWriterFactory, primaryKeyFieldCount);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(
- primaryTupleWriterFactory, primaryKeyFieldCount);
+ private IFileSplitProvider primaryBTreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryBTreeFileName))) });
- private static String primaryRTreeName = "primary"
- + simpleDateFormat.format(new Date());
- private static String primaryFileName = System
- .getProperty("java.io.tmpdir") + sep + primaryRTreeName;
+ private RecordDescriptor primaryBTreeRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE, });
- private IFileSplitProvider primaryRTreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- primaryFileName))) });
+ // field, type and key declarations for secondary indexes
+ private int secondaryFieldCount = 5;
+ private ITypeTraits[] secondaryTypeTraits = new ITypeTraits[secondaryFieldCount];
+ private int secondaryKeyFieldCount = 4;
+ private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
+ private IPrimitiveValueProviderFactory[] secondaryValueProviderFactories = new IPrimitiveValueProviderFactory[secondaryKeyFieldCount];
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ private RTreeTypeAwareTupleWriterFactory secondaryTupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
+ secondaryTypeTraits);
- // field, type and key declarations for primary B-tree index
- private int primaryBTreeFieldCount = 10;
- private ITypeTrait[] primaryBTreeTypeTraits = new ITypeTrait[primaryBTreeFieldCount];
- private int primaryBTreeKeyFieldCount = 1;
- private IBinaryComparatorFactory[] primaryBTreeComparatorFactories = new IBinaryComparatorFactory[primaryBTreeKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryBTreeTupleWriterFactory = new TypeAwareTupleWriterFactory(
- primaryBTreeTypeTraits);
- private ITreeIndexFrameFactory primaryBTreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
- primaryBTreeTupleWriterFactory);
- private ITreeIndexFrameFactory primaryBTreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(
- primaryBTreeTupleWriterFactory);
+ private ITreeIndexFrameFactory secondaryInteriorFrameFactory;
+ private ITreeIndexFrameFactory secondaryLeafFrameFactory;
- private static String primaryBTreeName = "primaryBTree"
- + simpleDateFormat.format(new Date());
- private static String primaryBTreeFileName = System
- .getProperty("java.io.tmpdir") + sep + primaryBTreeName;
+ private static String secondaryRTreeName = "secondary" + simpleDateFormat.format(new Date());
+ private static String secondaryFileName = System.getProperty("java.io.tmpdir") + sep + secondaryRTreeName;
- private IFileSplitProvider primaryBTreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- primaryBTreeFileName))) });
+ private IFileSplitProvider secondaryRTreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(secondaryFileName))) });
- private RecordDescriptor primaryBTreeRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE, });
+ private RecordDescriptor secondaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- // field, type and key declarations for secondary indexes
- private int secondaryFieldCount = 5;
- private ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
- private int secondaryKeyFieldCount = 4;
- private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
- private IPrimitiveValueProviderFactory[] secondaryValueProviderFactories = new IPrimitiveValueProviderFactory[secondaryKeyFieldCount];
+ @Before
+ public void setup() throws Exception {
+ // field, type and key declarations for primary B-tree index
+ primaryBTreeTypeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ primaryBTreeTypeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ primaryBTreeTypeTraits[2] = UTF8StringPointable.TYPE_TRAITS;
+ primaryBTreeTypeTraits[3] = UTF8StringPointable.TYPE_TRAITS;
+ primaryBTreeTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
+ primaryBTreeTypeTraits[5] = UTF8StringPointable.TYPE_TRAITS;
+ primaryBTreeTypeTraits[6] = DoublePointable.TYPE_TRAITS;
+ primaryBTreeTypeTraits[7] = DoublePointable.TYPE_TRAITS;
+ primaryBTreeTypeTraits[8] = DoublePointable.TYPE_TRAITS;
+ primaryBTreeTypeTraits[9] = DoublePointable.TYPE_TRAITS;
+ primaryBTreeComparatorFactories[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY);
- private RTreeTypeAwareTupleWriterFactory secondaryTupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- secondaryTypeTraits);
+ // field, type and key declarations for secondary indexes
+ secondaryTypeTraits[0] = DoublePointable.TYPE_TRAITS;
+ secondaryTypeTraits[1] = DoublePointable.TYPE_TRAITS;
+ secondaryTypeTraits[2] = DoublePointable.TYPE_TRAITS;
+ secondaryTypeTraits[3] = DoublePointable.TYPE_TRAITS;
+ secondaryTypeTraits[4] = UTF8StringPointable.TYPE_TRAITS;
+ secondaryComparatorFactories[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY);
+ secondaryComparatorFactories[1] = secondaryComparatorFactories[0];
+ secondaryComparatorFactories[2] = secondaryComparatorFactories[0];
+ secondaryComparatorFactories[3] = secondaryComparatorFactories[0];
+ secondaryValueProviderFactories[0] = DoublePrimitiveValueProviderFactory.INSTANCE;
+ secondaryValueProviderFactories[1] = secondaryValueProviderFactories[0];
+ secondaryValueProviderFactories[2] = secondaryValueProviderFactories[0];
+ secondaryValueProviderFactories[3] = secondaryValueProviderFactories[0];
- private ITreeIndexFrameFactory secondaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(
- secondaryTupleWriterFactory, secondaryKeyFieldCount);
- private ITreeIndexFrameFactory secondaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(
- secondaryTupleWriterFactory, secondaryKeyFieldCount);
+ IPrimitiveValueProviderFactory[] secondaryValueProviderFactories = RTreeUtils
+ .createPrimitiveValueProviderFactories(secondaryComparatorFactories.length, DoublePointable.FACTORY);
- private static String secondaryRTreeName = "secondary"
- + simpleDateFormat.format(new Date());
- private static String secondaryFileName = System
- .getProperty("java.io.tmpdir") + sep + secondaryRTreeName;
+ secondaryInteriorFrameFactory = new RTreeNSMInteriorFrameFactory(secondaryTupleWriterFactory,
+ secondaryValueProviderFactories);
+ secondaryLeafFrameFactory = new RTreeNSMLeafFrameFactory(secondaryTupleWriterFactory,
+ secondaryValueProviderFactories);
- private IFileSplitProvider secondaryRTreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- secondaryFileName))) });
+ loadPrimaryBTreeIndexTest();
+ loadSecondaryIndexTest();
+ }
- private RecordDescriptor secondaryRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ public void loadPrimaryBTreeIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- @Before
- public void setup() throws Exception {
- // field, type and key declarations for primary R-tree index
- primaryTypeTraits[0] = new TypeTrait(8);
- primaryTypeTraits[1] = new TypeTrait(8);
- primaryTypeTraits[2] = new TypeTrait(8);
- primaryTypeTraits[3] = new TypeTrait(8);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryComparatorFactories[0] = DoubleBinaryComparatorFactory.INSTANCE;
- primaryComparatorFactories[1] = primaryComparatorFactories[0];
- primaryComparatorFactories[2] = primaryComparatorFactories[0];
- primaryComparatorFactories[3] = primaryComparatorFactories[0];
- primaryValueProviderFactories[0] = DoublePrimitiveValueProviderFactory.INSTANCE;
- primaryValueProviderFactories[1] = primaryValueProviderFactories[0];
- primaryValueProviderFactories[2] = primaryValueProviderFactories[0];
- primaryValueProviderFactories[3] = primaryValueProviderFactories[0];
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ "data/orders-with-locations.txt"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE });
- // field, type and key declarations for primary B-tree index
- primaryBTreeTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryBTreeTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryBTreeTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryBTreeTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryBTreeTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryBTreeTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryBTreeTypeTraits[6] = new TypeTrait(8);
- primaryBTreeTypeTraits[7] = new TypeTrait(8);
- primaryBTreeTypeTraits[8] = new TypeTrait(8);
- primaryBTreeTypeTraits[9] = new TypeTrait(8);
- primaryBTreeComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
+ new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
+ DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE, DoubleParserFactory.INSTANCE,
+ DoubleParserFactory.INSTANCE }, '|'), ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
- // field, type and key declarations for secondary indexes
- secondaryTypeTraits[0] = new TypeTrait(8);
- secondaryTypeTraits[1] = new TypeTrait(8);
- secondaryTypeTraits[2] = new TypeTrait(8);
- secondaryTypeTraits[3] = new TypeTrait(8);
- secondaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryComparatorFactories[0] = DoubleBinaryComparatorFactory.INSTANCE;
- secondaryComparatorFactories[1] = secondaryComparatorFactories[0];
- secondaryComparatorFactories[2] = secondaryComparatorFactories[0];
- secondaryComparatorFactories[3] = secondaryComparatorFactories[0];
- secondaryValueProviderFactories[0] = DoublePrimitiveValueProviderFactory.INSTANCE;
- secondaryValueProviderFactories[1] = secondaryValueProviderFactories[0];
- secondaryValueProviderFactories[2] = secondaryValueProviderFactories[0];
- secondaryValueProviderFactories[3] = secondaryValueProviderFactories[0];
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 0 },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
- loadPrimaryIndexTest();
- loadPrimaryBTreeIndexTest();
- loadSecondaryIndexTest();
- }
+ int[] fieldPermutation = { 0, 1, 2, 4, 5, 7, 9, 10, 11, 12 };
+ TreeIndexBulkLoadOperatorDescriptor primaryBTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, primaryBTreeSplitProvider, primaryBTreeInteriorFrameFactory,
+ primaryBTreeLeafFrameFactory, primaryBTreeTypeTraits, primaryBTreeComparatorFactories,
+ fieldPermutation, 0.7f, btreeDataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBTreeBulkLoad, NC1_ID);
- public void loadPrimaryBTreeIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/orders-with-locations.txt"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
- ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE });
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, primaryBTreeBulkLoad, 0);
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
- spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
- new IValueParserFactory[] {
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE }, '|'),
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- ordScanner, NC1_ID);
+ spec.addRoot(primaryBTreeBulkLoad);
+ runTest(spec);
+ }
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
- spec,
- 1000,
- new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
- NC1_ID);
+ public void loadSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- int[] fieldPermutation = { 0, 1, 2, 4, 5, 7, 9, 10, 11, 12 };
- TreeIndexBulkLoadOperatorDescriptor primaryBTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryBTreeSplitProvider, primaryBTreeInteriorFrameFactory,
- primaryBTreeLeafFrameFactory, primaryBTreeTypeTraits,
- primaryBTreeComparatorFactories, null, fieldPermutation, 0.7f,
- bTreeopHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBTreeBulkLoad, NC1_ID);
+ // build dummy tuple containing nothing
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryBTreeKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
- sorter, 0);
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
+ tb.addFieldEndOffset();
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
- primaryBTreeBulkLoad, 0);
+ ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- spec.addRoot(primaryBTreeBulkLoad);
- runTest(spec);
- }
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- public void loadPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ int[] lowKeyFields = null; // - infinity
+ int[] highKeyFields = null; // + infinity
- FileSplit[] objectsSplits = new FileSplit[] { new FileSplit(NC1_ID,
- new FileReference(new File("data/spatial.txt"))) };
- IFileSplitProvider objectsSplitProvider = new ConstantFileSplitProvider(
- objectsSplits);
- RecordDescriptor objectsDesc = new RecordDescriptor(
- new ISerializerDeserializer[] {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ // scan primary index
+ BTreeSearchOperatorDescriptor primaryBTreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
+ primaryBTreeRecDesc, storageManager, indexRegistryProvider, primaryBTreeSplitProvider,
+ primaryBTreeInteriorFrameFactory, primaryBTreeLeafFrameFactory, primaryBTreeTypeTraits,
+ primaryBTreeComparatorFactories, true, lowKeyFields, highKeyFields, true, true,
+ btreeDataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBTreeSearchOp, NC1_ID);
- FileScanOperatorDescriptor objScanner = new FileScanOperatorDescriptor(
- spec, objectsSplitProvider,
- new DelimitedDataTupleParserFactory(new IValueParserFactory[] {
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- DoubleParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE }, '|'), objectsDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- objScanner, NC1_ID);
+ // load secondary index
+ int[] fieldPermutation = { 6, 7, 8, 9, 0 };
+ TreeIndexBulkLoadOperatorDescriptor secondaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
+ storageManager, indexRegistryProvider, secondaryRTreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
+ dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryRTreeBulkLoad, NC1_ID);
- int[] fieldPermutation = { 0, 1, 2, 3, 4 };
- TreeIndexBulkLoadOperatorDescriptor primaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- primaryRTreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits,
- primaryComparatorFactories, secondaryValueProviderFactories,
- fieldPermutation, 0.7f, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryRTreeBulkLoad, NC1_ID);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBTreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), primaryBTreeSearchOp, 0, secondaryRTreeBulkLoad, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), objScanner, 0,
- primaryRTreeBulkLoad, 0);
+ spec.addRoot(secondaryRTreeBulkLoad);
+ runTest(spec);
+ }
- spec.addRoot(primaryRTreeBulkLoad);
- runTest(spec);
- }
+ @Test
+ public void searchSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- public void loadSecondaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ // build tuple
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
+ DataOutput dos = tb.getDataOutput();
- // build dummy tuple containing nothing
- ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
- DataOutput dos = tb.getDataOutput();
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(61.2894, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(-149.624, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(61.8894, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(-149.024, dos);
+ tb.addFieldEndOffset();
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
- tb.addFieldEndOffset();
+ ISerializerDeserializer[] keyRecDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ISerializerDeserializer[] keyRecDescSers = {
- UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
+ keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
- spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
- tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- keyProviderOp, NC1_ID);
+ int[] keyFields = { 0, 1, 2, 3 };
- int[] lowKeyFields = null; // - infinity
- int[] highKeyFields = null; // + infinity
+ RTreeSearchOperatorDescriptor secondaryRTreeSearchOp = new RTreeSearchOperatorDescriptor(spec,
+ secondaryRecDesc, storageManager, indexRegistryProvider, secondaryRTreeSplitProvider,
+ secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, keyFields, dataflowHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryRTreeSearchOp, NC1_ID);
- // scan primary index
- BTreeSearchOperatorDescriptor primaryBTreeSearchOp = new BTreeSearchOperatorDescriptor(
- spec, primaryBTreeRecDesc, storageManager,
- treeIndexRegistryProvider, primaryBTreeSplitProvider,
- primaryBTreeInteriorFrameFactory, primaryBTreeLeafFrameFactory,
- primaryBTreeTypeTraits, primaryBTreeComparatorFactories, true,
- lowKeyFields, highKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- primaryBTreeSearchOp, NC1_ID);
+ IFileSplitProvider outSplits = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC1_ID,
+ createTempFile().getAbsolutePath()) });
+ IOperatorDescriptor printer = new PlainFileWriterOperatorDescriptor(spec, outSplits, ",");
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
- // load secondary index
- int[] fieldPermutation = { 6, 7, 8, 9, 0 };
- TreeIndexBulkLoadOperatorDescriptor secondaryRTreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
- spec, storageManager, treeIndexRegistryProvider,
- secondaryRTreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, secondaryValueProviderFactories,
- fieldPermutation, 0.7f, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- secondaryRTreeBulkLoad, NC1_ID);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, secondaryRTreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), secondaryRTreeSearchOp, 0, printer, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
- primaryBTreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- primaryBTreeSearchOp, 0, secondaryRTreeBulkLoad, 0);
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- spec.addRoot(secondaryRTreeBulkLoad);
- runTest(spec);
- }
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primaryBTree = new File(primaryBTreeFileName);
+ primaryBTree.deleteOnExit();
- @Test
- public void searchSecondaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
-
- // build tuple
- ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
- DataOutput dos = tb.getDataOutput();
-
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(61.2894, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(-149.624, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(61.8894, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(-149.024, dos);
- tb.addFieldEndOffset();
-
- ISerializerDeserializer[] keyRecDescSers = {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
-
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
- spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
- tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- keyProviderOp, NC1_ID);
-
- int[] keyFields = { 0, 1, 2, 3 };
-
- RTreeSearchOperatorDescriptor secondaryRTreeSearchOp = new RTreeSearchOperatorDescriptor(
- spec, secondaryRecDesc, storageManager,
- treeIndexRegistryProvider, secondaryRTreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory,
- secondaryTypeTraits, secondaryComparatorFactories,
- secondaryValueProviderFactories, keyFields, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
- secondaryRTreeSearchOp, NC1_ID);
-
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
- NC1_ID);
-
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
- secondaryRTreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec),
- secondaryRTreeSearchOp, 0, printer, 0);
-
- spec.addRoot(printer);
- runTest(spec);
- }
-
- @AfterClass
- public static void cleanup() throws Exception {
- File primary = new File(primaryFileName);
- primary.deleteOnExit();
-
- File primaryBTree = new File(primaryBTreeFileName);
- primaryBTree.deleteOnExit();
-
- File secondary = new File(secondaryFileName);
- secondary.deleteOnExit();
- }
+ File secondary = new File(secondaryFileName);
+ secondary.deleteOnExit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/spillable/ExternalAggregateTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/spillable/ExternalAggregateTest.java
deleted file mode 100644
index 2f66bfd..0000000
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/spillable/ExternalAggregateTest.java
+++ /dev/null
@@ -1,386 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.tests.spillable;
-
-import java.io.File;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
-import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.normalizers.UTF8StringNormalizedKeyComputerFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.parsers.FloatParserFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.AvgAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.ConcatAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.CountAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IntSumAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.MultiAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
-import edu.uci.ics.hyracks.dataflow.std.file.DelimitedDataTupleParserFactory;
-import edu.uci.ics.hyracks.dataflow.std.file.FileScanOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.file.FileSplit;
-import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.dataflow.std.file.ITupleParserFactory;
-import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableGroupingTableFactory;
-import edu.uci.ics.hyracks.dataflow.std.misc.PrinterOperatorDescriptor;
-import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
-
-/**
- * @author jarodwen
- */
-public class ExternalAggregateTest extends AbstractIntegrationTest {
-
- final IFileSplitProvider splitProvider = new ConstantFileSplitProvider(new FileSplit[] { new FileSplit(NC2_ID,
- new FileReference(new File("data/tpch0.001/lineitem.tbl"))) });
-
- static final String outSplitsPrefix = System.getProperty("java.io.tmpdir");
-
- static final String outSplits1 = "nc1:" + outSplitsPrefix + "/aggregation_";
- static final String outSplits2 = "nc2:" + outSplitsPrefix + "/aggregation_";
-
- static final boolean isOutputFile = true;
-
- final RecordDescriptor desc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE,
- FloatSerializerDeserializer.INSTANCE, FloatSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
-
- final ITupleParserFactory tupleParserFactory = new DelimitedDataTupleParserFactory(new IValueParserFactory[] {
- UTF8StringParserFactory.INSTANCE, IntegerParserFactory.INSTANCE, IntegerParserFactory.INSTANCE,
- IntegerParserFactory.INSTANCE, IntegerParserFactory.INSTANCE, FloatParserFactory.INSTANCE,
- FloatParserFactory.INSTANCE, FloatParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, }, '|');
-
- private static FileSplit[] parseFileSplits(String fileSplits) {
- String[] splits = fileSplits.split(",");
- FileSplit[] fSplits = new FileSplit[splits.length];
- for (int i = 0; i < splits.length; ++i) {
- String s = splits[i].trim();
- int idx = s.indexOf(':');
- if (idx < 0) {
- throw new IllegalArgumentException("File split " + s + " not well formed");
- }
- fSplits[i] = new FileSplit(s.substring(0, idx), new FileReference(new File(s.substring(idx + 1))));
- }
- return fSplits;
- }
-
- private static AbstractSingleActivityOperatorDescriptor getPrinter(JobSpecification spec, boolean isFile,
- String prefix) {
- AbstractSingleActivityOperatorDescriptor printer;
-
- if (!isOutputFile)
- printer = new PrinterOperatorDescriptor(spec);
- else
- printer = new PlainFileWriterOperatorDescriptor(spec, new ConstantFileSplitProvider(
- parseFileSplits(outSplits1 + prefix + ".nc1, " + outSplits2 + prefix + ".nc2")), "\t");
-
- return printer;
- }
-
- @Test
- public void hashSingleKeyScalarGroupTest() throws Exception {
- JobSpecification spec = new JobSpecification();
-
- FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(spec, splitProvider, tupleParserFactory,
- desc);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC2_ID);
-
- RecordDescriptor outputRec = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
-
- int[] keyFields = new int[] { 0 };
- int frameLimits = 3;
- int tableSize = 8;
-
- ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(spec, keyFields, frameLimits,
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- new UTF8StringNormalizedKeyComputerFactory(), new CountAggregatorDescriptorFactory(),
- new IntSumAggregatorDescriptorFactory(keyFields.length), outputRec,
- new HashSpillableGroupingTableFactory(new FieldHashPartitionComputerFactory(keyFields,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }), tableSize),
- true);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(keyFields,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
- spec.connect(conn1, csvScanner, 0, grouper, 0);
-
- AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec, isOutputFile,
- "hashSingleKeyScalarGroupTest");
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
- spec.connect(conn2, grouper, 0, printer, 0);
-
- spec.addRoot(printer);
- runTest(spec);
- }
-
- @Test
- public void hashMultipleKeyScalarGroupTest() throws Exception {
- JobSpecification spec = new JobSpecification();
-
- FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(spec, splitProvider, tupleParserFactory,
- desc);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC2_ID);
-
- RecordDescriptor outputRec = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, });
-
- int[] keyFields = new int[] { 0, 9 };
- int frameLimits = 3;
- int tableSize = 8;
-
- ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(spec, keyFields, frameLimits,
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE,
- UTF8StringBinaryComparatorFactory.INSTANCE }, new UTF8StringNormalizedKeyComputerFactory(),
- new IntSumAggregatorDescriptorFactory(1), new IntSumAggregatorDescriptorFactory(keyFields.length),
- outputRec, new HashSpillableGroupingTableFactory(new FieldHashPartitionComputerFactory(keyFields,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE,
- UTF8StringBinaryHashFunctionFactory.INSTANCE }), tableSize), true);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(keyFields, new IBinaryHashFunctionFactory[] {
- UTF8StringBinaryHashFunctionFactory.INSTANCE, UTF8StringBinaryHashFunctionFactory.INSTANCE, }));
- spec.connect(conn1, csvScanner, 0, grouper, 0);
-
- AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec, isOutputFile,
- "hashMultipleKeyScalarGroupTest");
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
- spec.connect(conn2, grouper, 0, printer, 0);
-
- spec.addRoot(printer);
- runTest(spec);
- }
-
- @Test
- public void hashMultipleKeyMultipleScalarGroupTest() throws Exception {
- JobSpecification spec = new JobSpecification();
-
- FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(spec, splitProvider, tupleParserFactory,
- desc);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC2_ID);
-
- RecordDescriptor outputRec = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, });
-
- int[] keyFields = new int[] { 0, 9 };
- int frameLimits = 3;
- int tableSize = 8;
-
- ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(spec, keyFields, frameLimits,
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE,
- UTF8StringBinaryComparatorFactory.INSTANCE }, new UTF8StringNormalizedKeyComputerFactory(),
- new MultiAggregatorDescriptorFactory(new IAggregatorDescriptorFactory[] {
- new IntSumAggregatorDescriptorFactory(1, 2), new IntSumAggregatorDescriptorFactory(2, 3) }),
- new MultiAggregatorDescriptorFactory(new IAggregatorDescriptorFactory[] {
- new IntSumAggregatorDescriptorFactory(2, 2), new IntSumAggregatorDescriptorFactory(3, 3) }),
- outputRec, new HashSpillableGroupingTableFactory(new FieldHashPartitionComputerFactory(keyFields,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE,
- UTF8StringBinaryHashFunctionFactory.INSTANCE }), tableSize), true);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(keyFields, new IBinaryHashFunctionFactory[] {
- UTF8StringBinaryHashFunctionFactory.INSTANCE, UTF8StringBinaryHashFunctionFactory.INSTANCE, }));
- spec.connect(conn1, csvScanner, 0, grouper, 0);
-
- AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec, isOutputFile,
- "hashMultipleKeyMultipleScalarGroupTest");
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
- spec.connect(conn2, grouper, 0, printer, 0);
-
- spec.addRoot(printer);
- runTest(spec);
- }
-
- @Test
- public void hashMultipleKeyNonScalarGroupTest() throws Exception {
- JobSpecification spec = new JobSpecification();
-
- FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(spec, splitProvider, tupleParserFactory,
- desc);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC2_ID);
-
- RecordDescriptor outputRec = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
-
- int[] keyFields = new int[] { 0 };
- int frameLimits = 3;
- int tableSize = 8;
-
- ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(spec, keyFields, frameLimits,
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- new UTF8StringNormalizedKeyComputerFactory(), new ConcatAggregatorDescriptorFactory(9),
- new ConcatAggregatorDescriptorFactory(keyFields.length), outputRec,
- new HashSpillableGroupingTableFactory(new FieldHashPartitionComputerFactory(keyFields,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }), tableSize),
- true);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(keyFields,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
- spec.connect(conn1, csvScanner, 0, grouper, 0);
-
- AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec, isOutputFile,
- "hashMultipleKeyNonScalarGroupTest");
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
- spec.connect(conn2, grouper, 0, printer, 0);
-
- spec.addRoot(printer);
- runTest(spec);
- }
-
- @Test
- public void hashMultipleKeyMultipleFieldsGroupTest() throws Exception {
- JobSpecification spec = new JobSpecification();
-
- FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(spec, splitProvider, tupleParserFactory,
- desc);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC2_ID);
-
- RecordDescriptor outputRec = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
-
- int[] keyFields = new int[] { 0, 9 };
- int frameLimits = 3;
- int tableSize = 8;
-
- ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(spec, keyFields, frameLimits,
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE,
- UTF8StringBinaryComparatorFactory.INSTANCE }, new UTF8StringNormalizedKeyComputerFactory(),
- new MultiAggregatorDescriptorFactory(new IAggregatorDescriptorFactory[] {
- new IntSumAggregatorDescriptorFactory(1, 2), new IntSumAggregatorDescriptorFactory(2, 3),
- new ConcatAggregatorDescriptorFactory(9, 4) }), new MultiAggregatorDescriptorFactory(
- new IAggregatorDescriptorFactory[] { new IntSumAggregatorDescriptorFactory(2, 2),
- new IntSumAggregatorDescriptorFactory(3, 3),
- new ConcatAggregatorDescriptorFactory(4, 4) }), outputRec,
- new HashSpillableGroupingTableFactory(new FieldHashPartitionComputerFactory(keyFields,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE,
- UTF8StringBinaryHashFunctionFactory.INSTANCE }), tableSize), true);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(keyFields, new IBinaryHashFunctionFactory[] {
- UTF8StringBinaryHashFunctionFactory.INSTANCE, UTF8StringBinaryHashFunctionFactory.INSTANCE, }));
- spec.connect(conn1, csvScanner, 0, grouper, 0);
-
- AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec, isOutputFile,
- "hashMultipleKeyMultipleFieldsGroupTest");
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
- spec.connect(conn2, grouper, 0, printer, 0);
-
- spec.addRoot(printer);
- runTest(spec);
- }
-
- @Test
- public void hashSingleKeyScalarAvgGroupTest() throws Exception {
- JobSpecification spec = new JobSpecification();
-
- FileScanOperatorDescriptor csvScanner = new FileScanOperatorDescriptor(spec, splitProvider, tupleParserFactory,
- desc);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, csvScanner, NC2_ID);
-
- RecordDescriptor outputRec = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
-
- int[] keyFields = new int[] { 0 };
- int frameLimits = 3;
- int tableSize = 8;
-
- ExternalGroupOperatorDescriptor grouper = new ExternalGroupOperatorDescriptor(spec, keyFields, frameLimits,
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- new UTF8StringNormalizedKeyComputerFactory(), new AvgAggregatorDescriptorFactory(1),
- new AvgAggregatorDescriptorFactory(keyFields.length), outputRec, new HashSpillableGroupingTableFactory(
- new FieldHashPartitionComputerFactory(keyFields,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }),
- tableSize), true);
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, grouper, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn1 = new MToNPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(keyFields,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
- spec.connect(conn1, csvScanner, 0, grouper, 0);
-
- AbstractSingleActivityOperatorDescriptor printer = getPrinter(spec, isOutputFile,
- "hashSingleKeyScalarGroupTest");
-
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC2_ID, NC1_ID);
-
- IConnectorDescriptor conn2 = new OneToOneConnectorDescriptor(spec);
- spec.connect(conn2, grouper, 0, printer, 0);
-
- spec.addRoot(printer);
- runTest(spec);
- }
-}
diff --git a/hyracks-examples/pom.xml b/hyracks-examples/pom.xml
index 7e66a6e..a7021a9 100644
--- a/hyracks-examples/pom.xml
+++ b/hyracks-examples/pom.xml
@@ -2,13 +2,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<packaging>pom</packaging>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<modules>
diff --git a/hyracks-examples/text-example/.project b/hyracks-examples/text-example/.project
deleted file mode 100644
index 4e057cd..0000000
--- a/hyracks-examples/text-example/.project
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>text-example</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/text-example/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/text-example/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 4562b1a..0000000
--- a/hyracks-examples/text-example/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/text-example/pom.xml b/hyracks-examples/text-example/pom.xml
index d18f0cb..d334644 100644
--- a/hyracks-examples/text-example/pom.xml
+++ b/hyracks-examples/text-example/pom.xml
@@ -2,13 +2,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>text-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<packaging>pom</packaging>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<modules>
diff --git a/hyracks-examples/text-example/textapp/.classpath b/hyracks-examples/text-example/textapp/.classpath
deleted file mode 100644
index f2cc5f7..0000000
--- a/hyracks-examples/text-example/textapp/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/text-example/textapp/.project b/hyracks-examples/text-example/textapp/.project
deleted file mode 100644
index 4f3af14..0000000
--- a/hyracks-examples/text-example/textapp/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>textapp</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/text-example/textapp/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/text-example/textapp/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 692202d..0000000
--- a/hyracks-examples/text-example/textapp/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,13 +0,0 @@
-#Thu May 19 22:55:12 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.debug.lineNumber=generate
-org.eclipse.jdt.core.compiler.debug.localVariable=generate
-org.eclipse.jdt.core.compiler.debug.sourceFile=generate
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/text-example/textapp/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/text-example/textapp/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 4562b1a..0000000
--- a/hyracks-examples/text-example/textapp/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/text-example/textapp/pom.xml b/hyracks-examples/text-example/textapp/pom.xml
index 269ef2b..913a95a 100644
--- a/hyracks-examples/text-example/textapp/pom.xml
+++ b/hyracks-examples/text-example/textapp/pom.xml
@@ -2,15 +2,44 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.text</groupId>
<artifactId>textapp</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>text-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <versionRange>[1.0.0,)</versionRange>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -48,7 +77,7 @@
<plugin>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-maven-plugin</artifactId>
- <version>0.0.1</version>
+ <version>0.0.2</version>
<configuration>
<hyracksServerHome>${basedir}/../../../hyracks-server/target/hyracks-server-${project.version}-binary-assembly</hyracksServerHome>
<hyracksCLIHome>${basedir}/../../../hyracks-cli/target/hyracks-cli-${project.version}-binary-assembly</hyracksCLIHome>
@@ -135,13 +164,13 @@
<dependency>
<groupId>edu.uci.ics.hyracks.examples.text</groupId>
<artifactId>texthelper</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks.examples.text</groupId>
<artifactId>textclient</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>test</scope>
</dependency>
diff --git a/hyracks-examples/text-example/textclient/.classpath b/hyracks-examples/text-example/textclient/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-examples/text-example/textclient/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/text-example/textclient/.project b/hyracks-examples/text-example/textclient/.project
deleted file mode 100644
index 04307d3..0000000
--- a/hyracks-examples/text-example/textclient/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>textclient</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/text-example/textclient/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/text-example/textclient/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index dfac000..0000000
--- a/hyracks-examples/text-example/textclient/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:05 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/text-example/textclient/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/text-example/textclient/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 4562b1a..0000000
--- a/hyracks-examples/text-example/textclient/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/text-example/textclient/pom.xml b/hyracks-examples/text-example/textclient/pom.xml
index 42e9655..c6f3832 100644
--- a/hyracks-examples/text-example/textclient/pom.xml
+++ b/hyracks-examples/text-example/textclient/pom.xml
@@ -2,25 +2,25 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.text</groupId>
<artifactId>textclient</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>text-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks.examples.text</groupId>
<artifactId>texthelper</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
@@ -41,6 +41,7 @@
<artifactId>appassembler-maven-plugin</artifactId>
<executions>
<execution>
+ <id>textclient</id>
<configuration>
<programs>
<program>
@@ -56,6 +57,23 @@
<goal>assemble</goal>
</goals>
</execution>
+ <execution>
+ <id>groupclient</id>
+ <configuration>
+ <programs>
+ <program>
+ <mainClass>edu.uci.ics.hyracks.examples.text.client.ExternalGroupClient</mainClass>
+ <name>groupclient</name>
+ </program>
+ </programs>
+ <repositoryLayout>flat</repositoryLayout>
+ <repositoryName>lib</repositoryName>
+ </configuration>
+ <phase>package</phase>
+ <goals>
+ <goal>assemble</goal>
+ </goals>
+ </execution>
</executions>
</plugin>
<plugin>
diff --git a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
index 68e33cb..66f7efc 100644
--- a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
+++ b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/ExternalGroupClient.java
@@ -19,7 +19,7 @@
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
@@ -31,8 +31,9 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.IntegerBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
@@ -42,13 +43,6 @@
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IntegerParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.CountAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.CountAggregatorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IFieldValueResultingAggregatorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IntSumAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.MultiAggregatorDescriptorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.MultiAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
@@ -60,10 +54,12 @@
import edu.uci.ics.hyracks.dataflow.std.file.FrameFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableGroupingTableFactory;
-import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.HashSpillableTableFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.IntSumFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
/**
@@ -101,6 +97,9 @@
@Option(name = "-out-plain", usage = "Whether to output plain text (default: true)", required = false)
public boolean outPlain = true;
+
+ @Option(name = "-algo", usage = "The algorithm to be used", required = true)
+ public int algo;
}
/**
@@ -111,14 +110,15 @@
CmdLineParser parser = new CmdLineParser(options);
parser.parseArgument(args);
- IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+ IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
JobSpecification job;
- for (int i = 0; i < 3; i++) {
+ for (int i = 0; i < 6; i++) {
long start = System.currentTimeMillis();
- job = createJob(parseFileSplits(options.inFileSplits), parseFileSplits(options.outFileSplits, i % 2),
- options.htSize, options.sbSize, options.framesLimit, options.sortOutput, i % 3, options.outPlain);
+ job = createJob(parseFileSplits(options.inFileSplits), parseFileSplits(options.outFileSplits, i),
+ options.htSize, options.sbSize, options.framesLimit, options.sortOutput, options.algo,
+ options.outPlain);
System.out.print(i + "\t" + (System.currentTimeMillis() - start));
start = System.currentTimeMillis();
@@ -200,102 +200,100 @@
AbstractOperatorDescriptor grouper;
switch (alg) {
- case 0: // External hash group
- grouper = new ExternalGroupOperatorDescriptor(
- spec,
- keys,
- framesLimit,
- new IBinaryComparatorFactory[] {
- // IntegerBinaryComparatorFactory.INSTANCE,
- IntegerBinaryComparatorFactory.INSTANCE },
- new IntegerNormalizedKeyComputerFactory(),
- new MultiAggregatorDescriptorFactory(
- new IAggregatorDescriptorFactory[] { new CountAggregatorDescriptorFactory() }),
- new MultiAggregatorDescriptorFactory(
- new IAggregatorDescriptorFactory[] { new IntSumAggregatorDescriptorFactory(keys.length) }),
- outDesc, new HashSpillableGroupingTableFactory(new FieldHashPartitionComputerFactory(keys,
- new IBinaryHashFunctionFactory[] {
- // IntegerBinaryHashFunctionFactory.INSTANCE,
- IntegerBinaryHashFunctionFactory.INSTANCE }), htSize), false);
+ case 0: // new external hash graph
+ grouper = new edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor(spec, keys,
+ framesLimit, new IBinaryComparatorFactory[] {
+ // PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
+ new IntegerNormalizedKeyComputerFactory(), new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(false) }),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new IntSumFieldAggregatorFactory(keys.length,
+ false) }), outDesc, new HashSpillableTableFactory(
+ new FieldHashPartitionComputerFactory(keys, new IBinaryHashFunctionFactory[] {
+ // PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) }), htSize), false);
createPartitionConstraint(spec, grouper, outSplits);
// Connect scanner with the grouper
- IConnectorDescriptor scanGroupConn = new MToNPartitioningConnectorDescriptor(spec,
+ IConnectorDescriptor scanGroupConnDef2 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(keys, new IBinaryHashFunctionFactory[] {
- // IntegerBinaryHashFunctionFactory.INSTANCE,
- IntegerBinaryHashFunctionFactory.INSTANCE }));
- spec.connect(scanGroupConn, fileScanner, 0, grouper, 0);
+ // PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) }));
+ spec.connect(scanGroupConnDef2, fileScanner, 0, grouper, 0);
+
break;
- case 1: // External sort + pre-cluster
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, framesLimit, keys,
+ case 1: // External-sort + new-precluster
+ ExternalSortOperatorDescriptor sorter2 = new ExternalSortOperatorDescriptor(spec, framesLimit, keys,
new IBinaryComparatorFactory[] {
- // IntegerBinaryComparatorFactory.INSTANCE,
- IntegerBinaryComparatorFactory.INSTANCE }, inDesc);
- createPartitionConstraint(spec, sorter, inSplits);
+ // PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) }, inDesc);
+ createPartitionConstraint(spec, sorter2, inSplits);
// Connect scan operator with the sorter
- IConnectorDescriptor scanSortConn = new MToNPartitioningConnectorDescriptor(spec,
+ IConnectorDescriptor scanSortConn2 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(keys, new IBinaryHashFunctionFactory[] {
- // IntegerBinaryHashFunctionFactory.INSTANCE,
- IntegerBinaryHashFunctionFactory.INSTANCE }));
- spec.connect(scanSortConn, fileScanner, 0, sorter, 0);
+ // PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) }));
+ spec.connect(scanSortConn2, fileScanner, 0, sorter2, 0);
- grouper = new PreclusteredGroupOperatorDescriptor(spec, keys, new IBinaryComparatorFactory[] {
- // IntegerBinaryComparatorFactory.INSTANCE,
- IntegerBinaryComparatorFactory.INSTANCE }, new MultiAggregatorFactory(
- new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }), outDesc);
+ grouper = new edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor(spec, keys,
+ new IBinaryComparatorFactory[] {
+ // PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }),
+ outDesc);
createPartitionConstraint(spec, grouper, outSplits);
// Connect sorter with the pre-cluster
- OneToOneConnectorDescriptor sortGroupConn = new OneToOneConnectorDescriptor(spec);
- spec.connect(sortGroupConn, sorter, 0, grouper, 0);
+ OneToOneConnectorDescriptor sortGroupConn2 = new OneToOneConnectorDescriptor(spec);
+ spec.connect(sortGroupConn2, sorter2, 0, grouper, 0);
break;
- case 2: // In-memory hash group
+ case 2: // Inmem
grouper = new HashGroupOperatorDescriptor(spec, keys, new FieldHashPartitionComputerFactory(keys,
new IBinaryHashFunctionFactory[] {
- // IntegerBinaryHashFunctionFactory.INSTANCE,
- IntegerBinaryHashFunctionFactory.INSTANCE }), new IBinaryComparatorFactory[] {
- // IntegerBinaryComparatorFactory.INSTANCE,
- IntegerBinaryComparatorFactory.INSTANCE }, new MultiAggregatorFactory(
- new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }), outDesc,
- htSize);
+ // PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] {
+ // PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }),
+ outDesc, htSize);
createPartitionConstraint(spec, grouper, outSplits);
// Connect scanner with the grouper
- IConnectorDescriptor scanConn = new MToNPartitioningConnectorDescriptor(spec,
+ IConnectorDescriptor scanConn2 = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(keys, new IBinaryHashFunctionFactory[] {
- // IntegerBinaryHashFunctionFactory.INSTANCE,
- IntegerBinaryHashFunctionFactory.INSTANCE }));
- spec.connect(scanConn, fileScanner, 0, grouper, 0);
+ // PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) }));
+ spec.connect(scanConn2, fileScanner, 0, grouper, 0);
break;
default:
- grouper = new ExternalGroupOperatorDescriptor(
- spec,
- keys,
- framesLimit,
- new IBinaryComparatorFactory[] {
- // IntegerBinaryComparatorFactory.INSTANCE,
- IntegerBinaryComparatorFactory.INSTANCE },
- new IntegerNormalizedKeyComputerFactory(),
- new MultiAggregatorDescriptorFactory(
- new IAggregatorDescriptorFactory[] { new CountAggregatorDescriptorFactory() }),
- new MultiAggregatorDescriptorFactory(
- new IAggregatorDescriptorFactory[] { new IntSumAggregatorDescriptorFactory(keys.length) }),
- outDesc, new HashSpillableGroupingTableFactory(new FieldHashPartitionComputerFactory(keys,
- new IBinaryHashFunctionFactory[] {
- // IntegerBinaryHashFunctionFactory.INSTANCE,
- IntegerBinaryHashFunctionFactory.INSTANCE }), htSize), false);
+ grouper = new edu.uci.ics.hyracks.dataflow.std.group.ExternalGroupOperatorDescriptor(spec, keys,
+ framesLimit, new IBinaryComparatorFactory[] {
+ // PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY) },
+ new IntegerNormalizedKeyComputerFactory(), new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(false) }),
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new IntSumFieldAggregatorFactory(keys.length,
+ false) }), outDesc, new HashSpillableTableFactory(
+ new FieldHashPartitionComputerFactory(keys, new IBinaryHashFunctionFactory[] {
+ // PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) }), htSize), false);
createPartitionConstraint(spec, grouper, outSplits);
// Connect scanner with the grouper
IConnectorDescriptor scanGroupConnDef = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(keys, new IBinaryHashFunctionFactory[] {
- // IntegerBinaryHashFunctionFactory.INSTANCE,
- IntegerBinaryHashFunctionFactory.INSTANCE }));
+ // PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY),
+ PointableBinaryHashFunctionFactory.of(IntegerPointable.FACTORY) }));
spec.connect(scanGroupConnDef, fileScanner, 0, grouper, 0);
}
diff --git a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
index 51d3661..1708259 100644
--- a/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
+++ b/hyracks-examples/text-example/textclient/src/main/java/edu/uci/ics/hyracks/examples/text/client/WordCountMain.java
@@ -15,11 +15,12 @@
package edu.uci.ics.hyracks.examples.text.client;
import java.io.File;
+import java.util.EnumSet;
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
@@ -29,17 +30,16 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.normalizers.UTF8StringNormalizedKeyComputerFactory;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.CountAggregatorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IFieldValueResultingAggregatorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.MultiAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
@@ -49,7 +49,10 @@
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.file.PlainFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
import edu.uci.ics.hyracks.dataflow.std.group.PreclusteredGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.sort.InMemorySortOperatorDescriptor;
import edu.uci.ics.hyracks.examples.text.WordTupleParserFactory;
@@ -60,7 +63,7 @@
public String host;
@Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)")
- public int port = 1099;
+ public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
public String app;
@@ -82,6 +85,9 @@
@Option(name = "-sortbuffer-size", usage = "Sort buffer size in frames (default: 32768)", required = false)
public int sbSize = 32768;
+
+ @Option(name = "-runtime-profiling", usage = "Indicates if runtime profiling should be enabled. (default: false)")
+ public boolean runtimeProfiling = false;
}
public static void main(String[] args) throws Exception {
@@ -89,13 +95,14 @@
CmdLineParser parser = new CmdLineParser(options);
parser.parseArgument(args);
- IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+ IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
JobSpecification job = createJob(parseFileSplits(options.inFileSplits), parseFileSplits(options.outFileSplits),
options.algo, options.htSize, options.sbSize, options.format);
long start = System.currentTimeMillis();
- JobId jobId = hcc.createJob(options.app, job);
+ JobId jobId = hcc.createJob(options.app, job, options.runtimeProfiling ? EnumSet.of(JobFlag.PROFILE_RUNTIME)
+ : EnumSet.noneOf(JobFlag.class));
hcc.start(jobId);
hcc.waitForCompletion(jobId);
long end = System.currentTimeMillis();
@@ -134,19 +141,25 @@
IOperatorDescriptor gBy;
int[] keys = new int[] { 0 };
if ("hash".equalsIgnoreCase(algo)) {
- gBy = new HashGroupOperatorDescriptor(spec, keys, new FieldHashPartitionComputerFactory(keys,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }),
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- new MultiAggregatorFactory(
- new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }),
+ gBy = new HashGroupOperatorDescriptor(
+ spec,
+ keys,
+ new FieldHashPartitionComputerFactory(keys,
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }),
groupResultDesc, htSize);
createPartitionConstraint(spec, gBy, outSplits);
IConnectorDescriptor scanGroupConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(keys,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(scanGroupConn, wordScanner, 0, gBy, 0);
} else {
- IBinaryComparatorFactory[] cfs = new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE };
+ IBinaryComparatorFactory[] cfs = new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory
+ .of(UTF8StringPointable.FACTORY) };
IOperatorDescriptor sorter = "memsort".equalsIgnoreCase(algo) ? new InMemorySortOperatorDescriptor(spec,
keys, new UTF8StringNormalizedKeyComputerFactory(), cfs, wordDesc)
: new ExternalSortOperatorDescriptor(spec, sbSize, keys,
@@ -155,13 +168,16 @@
IConnectorDescriptor scanSortConn = new MToNPartitioningConnectorDescriptor(spec,
new FieldHashPartitionComputerFactory(keys,
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
spec.connect(scanSortConn, wordScanner, 0, sorter, 0);
- gBy = new PreclusteredGroupOperatorDescriptor(spec, keys,
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- new MultiAggregatorFactory(
- new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }),
+ gBy = new PreclusteredGroupOperatorDescriptor(
+ spec,
+ keys,
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }),
groupResultDesc);
createPartitionConstraint(spec, gBy, outSplits);
OneToOneConnectorDescriptor sortGroupConn = new OneToOneConnectorDescriptor(spec);
diff --git a/hyracks-examples/text-example/texthelper/.classpath b/hyracks-examples/text-example/texthelper/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-examples/text-example/texthelper/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/text-example/texthelper/.project b/hyracks-examples/text-example/texthelper/.project
deleted file mode 100644
index 19ce234..0000000
--- a/hyracks-examples/text-example/texthelper/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>texthelper</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/text-example/texthelper/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/text-example/texthelper/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index dfac000..0000000
--- a/hyracks-examples/text-example/texthelper/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:05 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/text-example/texthelper/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/text-example/texthelper/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 4562b1a..0000000
--- a/hyracks-examples/text-example/texthelper/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Sep 28 14:37:42 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/text-example/texthelper/pom.xml b/hyracks-examples/text-example/texthelper/pom.xml
index c21b5a3..b0bfcef 100644
--- a/hyracks-examples/text-example/texthelper/pom.xml
+++ b/hyracks-examples/text-example/texthelper/pom.xml
@@ -2,27 +2,31 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.text</groupId>
<artifactId>texthelper</artifactId>
- <version>0.1.8-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>text-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/hyracks-examples/tpch-example/.project b/hyracks-examples/tpch-example/.project
deleted file mode 100644
index dfa44a1..0000000
--- a/hyracks-examples/tpch-example/.project
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>tpch-example</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/tpch-example/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/tpch-example/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index e6f9a9e..0000000
--- a/hyracks-examples/tpch-example/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Sun Aug 29 19:38:10 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/tpch-example/pom.xml b/hyracks-examples/tpch-example/pom.xml
index 60b00f1..f2c8786 100644
--- a/hyracks-examples/tpch-example/pom.xml
+++ b/hyracks-examples/tpch-example/pom.xml
@@ -2,13 +2,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>tpch-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<packaging>pom</packaging>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-examples</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<modules>
diff --git a/hyracks-examples/tpch-example/tpchapp/.classpath b/hyracks-examples/tpch-example/tpchapp/.classpath
deleted file mode 100644
index d0bec0f..0000000
--- a/hyracks-examples/tpch-example/tpchapp/.classpath
+++ /dev/null
@@ -1,6 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/J2SE-1.5"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/tpch-example/tpchapp/.project b/hyracks-examples/tpch-example/tpchapp/.project
deleted file mode 100644
index 46037da..0000000
--- a/hyracks-examples/tpch-example/tpchapp/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>tpchapp</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/tpch-example/tpchapp/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/tpch-example/tpchapp/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 0526f68..0000000
--- a/hyracks-examples/tpch-example/tpchapp/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Thu Dec 15 06:58:55 PST 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.5
-org.eclipse.jdt.core.compiler.compliance=1.5
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.5
diff --git a/hyracks-examples/tpch-example/tpchapp/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/tpch-example/tpchapp/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index e6f9a9e..0000000
--- a/hyracks-examples/tpch-example/tpchapp/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Sun Aug 29 19:38:10 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/tpch-example/tpchapp/pom.xml b/hyracks-examples/tpch-example/tpchapp/pom.xml
index 5e11a20..05256a3 100644
--- a/hyracks-examples/tpch-example/tpchapp/pom.xml
+++ b/hyracks-examples/tpch-example/tpchapp/pom.xml
@@ -2,15 +2,42 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.tpch</groupId>
<artifactId>tpchapp</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>tpch-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.eclipse.m2e</groupId>
+ <artifactId>lifecycle-mapping</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <lifecycleMappingMetadata>
+ <pluginExecutions>
+ <pluginExecution>
+ <pluginExecutionFilter>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <versionRange>[1.0.0,)</versionRange>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ </pluginExecutionFilter>
+ <action>
+ <ignore />
+ </action>
+ </pluginExecution>
+ </pluginExecutions>
+ </lifecycleMappingMetadata>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -51,8 +78,13 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
</dependencies>
</project>
diff --git a/hyracks-examples/tpch-example/tpchclient/.classpath b/hyracks-examples/tpch-example/tpchclient/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-examples/tpch-example/tpchclient/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-examples/tpch-example/tpchclient/.project b/hyracks-examples/tpch-example/tpchclient/.project
deleted file mode 100644
index b0effc6..0000000
--- a/hyracks-examples/tpch-example/tpchclient/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>tpchclient</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-examples/tpch-example/tpchclient/.settings/org.eclipse.jdt.core.prefs b/hyracks-examples/tpch-example/tpchclient/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 375e12e..0000000
--- a/hyracks-examples/tpch-example/tpchclient/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:07 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-examples/tpch-example/tpchclient/.settings/org.maven.ide.eclipse.prefs b/hyracks-examples/tpch-example/tpchclient/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 1b13d8b..0000000
--- a/hyracks-examples/tpch-example/tpchclient/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Wed Aug 11 19:09:15 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-examples/tpch-example/tpchclient/pom.xml b/hyracks-examples/tpch-example/tpchclient/pom.xml
index bc93289..d9b5fce 100644
--- a/hyracks-examples/tpch-example/tpchclient/pom.xml
+++ b/hyracks-examples/tpch-example/tpchclient/pom.xml
@@ -2,21 +2,24 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks.examples.tpch</groupId>
<artifactId>tpchclient</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks.examples</groupId>
<artifactId>tpch-example</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<dependencies>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-data-std</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java b/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
index 09d0e5a..72533e7 100644
--- a/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
+++ b/hyracks-examples/tpch-example/tpchclient/src/main/java/edu/uci/ics/hyracks/examples/tpch/client/Main.java
@@ -20,29 +20,32 @@
import org.kohsuke.args4j.CmdLineParser;
import org.kohsuke.args4j.Option;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.client.IHyracksClientConnection;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.constraints.PartitionConstraintHelper;
import edu.uci.ics.hyracks.api.dataflow.IConnectorDescriptor;
import edu.uci.ics.hyracks.api.dataflow.IOperatorDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ITuplePairComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.hash.UTF8StringBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.parsers.UTF8StringParserFactory;
import edu.uci.ics.hyracks.dataflow.common.data.partition.FieldHashPartitionComputerFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.CountAggregatorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.IFieldValueResultingAggregatorFactory;
-import edu.uci.ics.hyracks.dataflow.std.aggregators.MultiAggregatorFactory;
import edu.uci.ics.hyracks.dataflow.std.connectors.MToNPartitioningConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.connectors.OneToOneConnectorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.ConstantFileSplitProvider;
@@ -52,7 +55,13 @@
import edu.uci.ics.hyracks.dataflow.std.file.FrameFileWriterOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
import edu.uci.ics.hyracks.dataflow.std.group.HashGroupOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.group.IFieldAggregateDescriptorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.CountFieldAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.group.aggregators.MultiFieldsAggregatorFactory;
+import edu.uci.ics.hyracks.dataflow.std.join.GraceHashJoinOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.join.HybridHashJoinOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.join.InMemoryHashJoinOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.join.NestedLoopJoinOperatorDescriptor;
public class Main {
private static class Options {
@@ -60,7 +69,7 @@
public String host;
@Option(name = "-port", usage = "Hyracks Cluster Controller Port (default: 1099)", required = false)
- public int port = 1099;
+ public int port = 1098;
@Option(name = "-app", usage = "Hyracks Application name", required = true)
public String app;
@@ -79,6 +88,29 @@
@Option(name = "-profile", usage = "Enable/Disable profiling. (default: enabled)")
public boolean profile = true;
+
+ @Option(name = "-table-size", usage = "Table size for in-memory hash join", required = false)
+ public int tableSize = 8191;
+
+ @Option(name = "-algo", usage = "Join types", required = true)
+ public String algo;
+
+ // For grace/hybrid hash join only
+ @Option(name = "-mem-size", usage = "Memory size for hash join", required = true)
+ public int memSize;
+
+ @Option(name = "-input-size", usage = "Input size of the grace/hybrid hash join", required = false)
+ public int graceInputSize = 10;
+
+ @Option(name = "-records-per-frame", usage = "Records per frame for grace/hybrid hash join", required = false)
+ public int graceRecordsPerFrame = 200;
+
+ @Option(name = "-grace-factor", usage = "Factor of the grace/hybrid hash join", required = false)
+ public double graceFactor = 1.2;
+
+ // Whether group-by is processed after the join
+ @Option(name = "-has-groupby", usage = "Whether to have group-by operation after join (default: disabled)", required = false)
+ public boolean hasGroupBy = false;
}
public static void main(String[] args) throws Exception {
@@ -86,11 +118,12 @@
CmdLineParser parser = new CmdLineParser(options);
parser.parseArgument(args);
- IHyracksClientConnection hcc = new HyracksRMIConnection(options.host, options.port);
+ IHyracksClientConnection hcc = new HyracksConnection(options.host, options.port);
JobSpecification job = createJob(parseFileSplits(options.inFileCustomerSplits),
parseFileSplits(options.inFileOrderSplits), parseFileSplits(options.outFileSplits),
- options.numJoinPartitions);
+ options.numJoinPartitions, options.algo, options.graceInputSize, options.graceRecordsPerFrame,
+ options.graceFactor, options.memSize, options.tableSize, options.hasGroupBy);
long start = System.currentTimeMillis();
JobId jobId = hcc.createJob(options.app, job,
@@ -116,7 +149,8 @@
}
private static JobSpecification createJob(FileSplit[] customerSplits, FileSplit[] orderSplits,
- FileSplit[] resultSplits, int numJoinPartitions) {
+ FileSplit[] resultSplits, int numJoinPartitions, String algo, int graceInputSize, int graceRecordsPerFrame,
+ double graceFactor, int memSize, int tableSize, boolean hasGroupBy) throws HyracksDataException {
JobSpecification spec = new JobSpecification();
IFileSplitProvider custSplitsProvider = new ConstantFileSplitProvider(customerSplits);
@@ -161,46 +195,99 @@
UTF8StringParserFactory.INSTANCE }, '|'), custDesc);
createPartitionConstraint(spec, custScanner, customerSplits);
- InMemoryHashJoinOperatorDescriptor join = new InMemoryHashJoinOperatorDescriptor(spec, new int[] { 0 },
- new int[] { 1 }, new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, custOrderJoinDesc,
- 6000000);
+ IOperatorDescriptor join;
+
+ if ("nestedloop".equalsIgnoreCase(algo)) {
+ join = new NestedLoopJoinOperatorDescriptor(spec, new JoinComparatorFactory(
+ PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY), 0, 1), custOrderJoinDesc, memSize);
+
+ } else if ("gracehash".equalsIgnoreCase(algo)) {
+ join = new GraceHashJoinOperatorDescriptor(
+ spec,
+ memSize,
+ graceInputSize,
+ graceRecordsPerFrame,
+ graceFactor,
+ new int[] { 0 },
+ new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc);
+
+ } else if ("hybridhash".equalsIgnoreCase(algo)) {
+ join = new HybridHashJoinOperatorDescriptor(
+ spec,
+ memSize,
+ graceInputSize,
+ graceRecordsPerFrame,
+ graceFactor,
+ new int[] { 0 },
+ new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc);
+
+ } else {
+ join = new InMemoryHashJoinOperatorDescriptor(
+ spec,
+ new int[] { 0 },
+ new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) },
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ custOrderJoinDesc, 6000000);
+ }
+
PartitionConstraintHelper.addPartitionCountConstraint(spec, join, numJoinPartitions);
- RecordDescriptor groupResultDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+ IConnectorDescriptor ordJoinConn = new MToNPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 1 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(ordJoinConn, ordScanner, 0, join, 1);
- HashGroupOperatorDescriptor gby = new HashGroupOperatorDescriptor(
- spec,
- new int[] { 6 },
- new FieldHashPartitionComputerFactory(new int[] { 6 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }),
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
- new MultiAggregatorFactory(new IFieldValueResultingAggregatorFactory[] { new CountAggregatorFactory() }),
- groupResultDesc, 16);
- createPartitionConstraint(spec, gby, resultSplits);
+ IConnectorDescriptor custJoinConn = new MToNPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 0 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(custJoinConn, custScanner, 0, join, 0);
+
+ IOperatorDescriptor endingOp = join;
+
+ if (hasGroupBy) {
+
+ RecordDescriptor groupResultDesc = new RecordDescriptor(new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+
+ HashGroupOperatorDescriptor gby = new HashGroupOperatorDescriptor(
+ spec,
+ new int[] { 6 },
+ new FieldHashPartitionComputerFactory(new int[] { 6 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }),
+ new IBinaryComparatorFactory[] { PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY) },
+ new MultiFieldsAggregatorFactory(
+ new IFieldAggregateDescriptorFactory[] { new CountFieldAggregatorFactory(true) }),
+ groupResultDesc, 16);
+ createPartitionConstraint(spec, gby, resultSplits);
+
+ IConnectorDescriptor joinGroupConn = new MToNPartitioningConnectorDescriptor(spec,
+ new FieldHashPartitionComputerFactory(new int[] { 6 },
+ new IBinaryHashFunctionFactory[] { PointableBinaryHashFunctionFactory
+ .of(UTF8StringPointable.FACTORY) }));
+ spec.connect(joinGroupConn, join, 0, gby, 0);
+
+ endingOp = gby;
+ }
IFileSplitProvider outSplitProvider = new ConstantFileSplitProvider(resultSplits);
FrameFileWriterOperatorDescriptor writer = new FrameFileWriterOperatorDescriptor(spec, outSplitProvider);
createPartitionConstraint(spec, writer, resultSplits);
- IConnectorDescriptor ordJoinConn = new MToNPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(new int[] { 1 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
- spec.connect(ordJoinConn, ordScanner, 0, join, 1);
-
- IConnectorDescriptor custJoinConn = new MToNPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(new int[] { 0 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
- spec.connect(custJoinConn, custScanner, 0, join, 0);
-
- IConnectorDescriptor joinGroupConn = new MToNPartitioningConnectorDescriptor(spec,
- new FieldHashPartitionComputerFactory(new int[] { 6 },
- new IBinaryHashFunctionFactory[] { UTF8StringBinaryHashFunctionFactory.INSTANCE }));
- spec.connect(joinGroupConn, join, 0, gby, 0);
-
- IConnectorDescriptor gbyPrinterConn = new OneToOneConnectorDescriptor(spec);
- spec.connect(gbyPrinterConn, gby, 0, writer, 0);
+ IConnectorDescriptor endingPrinterConn = new OneToOneConnectorDescriptor(spec);
+ spec.connect(endingPrinterConn, endingOp, 0, writer, 0);
spec.addRoot(writer);
return spec;
@@ -213,4 +300,60 @@
}
PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, op, parts);
}
+
+ static class JoinComparatorFactory implements ITuplePairComparatorFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final IBinaryComparatorFactory bFactory;
+ private final int pos0;
+ private final int pos1;
+
+ public JoinComparatorFactory(IBinaryComparatorFactory bFactory, int pos0, int pos1) {
+ this.bFactory = bFactory;
+ this.pos0 = pos0;
+ this.pos1 = pos1;
+ }
+
+ @Override
+ public ITuplePairComparator createTuplePairComparator() {
+ return new JoinComparator(bFactory.createBinaryComparator(), pos0, pos1);
+ }
+ }
+
+ static class JoinComparator implements ITuplePairComparator {
+
+ private final IBinaryComparator bComparator;
+ private final int field0;
+ private final int field1;
+
+ public JoinComparator(IBinaryComparator bComparator, int field0, int field1) {
+ this.bComparator = bComparator;
+ this.field0 = field0;
+ this.field1 = field1;
+ }
+
+ @Override
+ public int compare(IFrameTupleAccessor accessor0, int tIndex0, IFrameTupleAccessor accessor1, int tIndex1) {
+ int tStart0 = accessor0.getTupleStartOffset(tIndex0);
+ int fStartOffset0 = accessor0.getFieldSlotsLength() + tStart0;
+
+ int tStart1 = accessor1.getTupleStartOffset(tIndex1);
+ int fStartOffset1 = accessor1.getFieldSlotsLength() + tStart1;
+
+ int fStart0 = accessor0.getFieldStartOffset(tIndex0, field0);
+ int fEnd0 = accessor0.getFieldEndOffset(tIndex0, field0);
+ int fLen0 = fEnd0 - fStart0;
+
+ int fStart1 = accessor1.getFieldStartOffset(tIndex1, field1);
+ int fEnd1 = accessor1.getFieldEndOffset(tIndex1, field1);
+ int fLen1 = fEnd1 - fStart1;
+
+ int c = bComparator.compare(accessor0.getBuffer().array(), fStart0 + fStartOffset0, fLen0, accessor1
+ .getBuffer().array(), fStart1 + fStartOffset1, fLen1);
+ if (c != 0) {
+ return c;
+ }
+ return 0;
+ }
+ }
}
\ No newline at end of file
diff --git a/hyracks-hadoop-compat/.classpath b/hyracks-hadoop-compat/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-hadoop-compat/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-hadoop-compat/.project b/hyracks-hadoop-compat/.project
deleted file mode 100644
index 7d50383..0000000
--- a/hyracks-hadoop-compat/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-hadoop-compat</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-hadoop-compat/.settings/org.eclipse.jdt.core.prefs b/hyracks-hadoop-compat/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index dfac000..0000000
--- a/hyracks-hadoop-compat/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:05 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-hadoop-compat/.settings/org.maven.ide.eclipse.prefs b/hyracks-hadoop-compat/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index e03a9fc..0000000
--- a/hyracks-hadoop-compat/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Oct 19 11:05:30 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-hadoop-compat/pom.xml b/hyracks-hadoop-compat/pom.xml
index bb807a6..2de7e04 100644
--- a/hyracks-hadoop-compat/pom.xml
+++ b/hyracks-hadoop-compat/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-hadoop-compat</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -79,7 +79,7 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-hadoop</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java b/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java
index 9e76a3e..eadf679 100644
--- a/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java
+++ b/hyracks-hadoop-compat/src/main/java/edu/uci/ics/hyracks/hadoop/compat/client/HyracksClient.java
@@ -5,7 +5,7 @@
import java.util.Properties;
import java.util.Set;
-import edu.uci.ics.hyracks.api.client.HyracksRMIConnection;
+import edu.uci.ics.hyracks.api.client.HyracksConnection;
import edu.uci.ics.hyracks.api.job.JobFlag;
import edu.uci.ics.hyracks.api.job.JobId;
import edu.uci.ics.hyracks.api.job.JobSpecification;
@@ -15,7 +15,7 @@
public class HyracksClient {
- private static HyracksRMIConnection connection;
+ private static HyracksConnection connection;
private static final String jobProfilingKey = "jobProfilingKey";
Set<String> systemLibs;
@@ -25,7 +25,7 @@
private void initialize(Properties properties) throws Exception {
String clusterController = (String) properties.get(ConfigurationConstants.clusterControllerHost);
- connection = new HyracksRMIConnection(clusterController, 1099);
+ connection = new HyracksConnection(clusterController, 1098);
systemLibs = new HashSet<String>();
for (String systemLib : ConfigurationConstants.systemLibs) {
String systemLibPath = properties.getProperty(systemLib);
diff --git a/hyracks-ipc/pom.xml b/hyracks-ipc/pom.xml
new file mode 100644
index 0000000..49c4323
--- /dev/null
+++ b/hyracks-ipc/pom.xml
@@ -0,0 +1,31 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-ipc</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCHandle.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCHandle.java
new file mode 100644
index 0000000..8a3630f
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCHandle.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.api;
+
+import java.net.InetSocketAddress;
+
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+
+public interface IIPCHandle {
+ public InetSocketAddress getRemoteAddress();
+
+ public long send(long requestId, Object payload, Exception exception) throws IPCException;
+
+ public void setAttachment(Object attachment);
+
+ public Object getAttachment();
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCI.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCI.java
new file mode 100644
index 0000000..24ab943
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IIPCI.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.api;
+
+public interface IIPCI {
+ public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception);
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IPCPerformanceCounters.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IPCPerformanceCounters.java
new file mode 100644
index 0000000..1873378
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IPCPerformanceCounters.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.api;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class IPCPerformanceCounters {
+ private final AtomicLong nMessagesSent;
+
+ private final AtomicLong nMessageBytesSent;
+
+ private final AtomicLong nMessagesReceived;
+
+ private final AtomicLong nMessageBytesReceived;
+
+ public IPCPerformanceCounters() {
+ nMessagesSent = new AtomicLong();
+ nMessageBytesSent = new AtomicLong();
+ nMessagesReceived = new AtomicLong();
+ nMessageBytesReceived = new AtomicLong();
+ }
+
+ public long getMessageSentCount() {
+ return nMessagesSent.get();
+ }
+
+ public void addMessageSentCount(long delta) {
+ nMessagesSent.addAndGet(delta);
+ }
+
+ public long getMessageBytesSent() {
+ return nMessageBytesSent.get();
+ }
+
+ public void addMessageBytesSent(long delta) {
+ nMessageBytesSent.addAndGet(delta);
+ }
+
+ public long getMessageReceivedCount() {
+ return nMessagesReceived.get();
+ }
+
+ public void addMessageReceivedCount(long delta) {
+ nMessagesReceived.addAndGet(delta);
+ }
+
+ public long getMessageBytesReceived() {
+ return nMessageBytesReceived.get();
+ }
+
+ public void addMessageBytesReceived(long delta) {
+ nMessageBytesReceived.addAndGet(delta);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IPayloadSerializerDeserializer.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IPayloadSerializerDeserializer.java
new file mode 100644
index 0000000..62648ff
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IPayloadSerializerDeserializer.java
@@ -0,0 +1,27 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.api;
+
+import java.nio.ByteBuffer;
+
+public interface IPayloadSerializerDeserializer {
+ public Object deserializeObject(ByteBuffer buffer, int length) throws Exception;
+
+ public Exception deserializeException(ByteBuffer buffer, int length) throws Exception;
+
+ public byte[] serializeObject(Object object) throws Exception;
+
+ public byte[] serializeException(Exception object) throws Exception;
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IResponseCallback.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IResponseCallback.java
new file mode 100644
index 0000000..7d25f88
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/IResponseCallback.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.api;
+
+public interface IResponseCallback {
+ public void callback(IIPCHandle handle, Object response, Exception exception);
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/RPCInterface.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/RPCInterface.java
new file mode 100644
index 0000000..3340516
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/api/RPCInterface.java
@@ -0,0 +1,72 @@
+package edu.uci.ics.hyracks.ipc.api;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class RPCInterface implements IIPCI {
+ private final Map<Long, Request> reqMap;
+
+ public RPCInterface() {
+ reqMap = new HashMap<Long, RPCInterface.Request>();
+ }
+
+ public Object call(IIPCHandle handle, Object request) throws Exception {
+ Request req;
+ synchronized (this) {
+ req = new Request();
+ long mid = handle.send(-1, request, null);
+ reqMap.put(mid, req);
+ }
+ return req.getResponse();
+ }
+
+ @Override
+ public void deliverIncomingMessage(IIPCHandle handle, long mid, long rmid, Object payload, Exception exception) {
+ Request req;
+ synchronized (this) {
+ req = reqMap.remove(rmid);
+ }
+ assert req != null;
+ if (exception != null) {
+ req.setException(exception);
+ } else {
+ req.setResult(payload);
+ }
+ }
+
+ private static class Request {
+ private boolean pending;
+
+ private Object result;
+
+ private Exception exception;
+
+ Request() {
+ pending = true;
+ result = null;
+ exception = null;
+ }
+
+ synchronized void setResult(Object result) {
+ this.pending = false;
+ this.result = result;
+ notifyAll();
+ }
+
+ synchronized void setException(Exception exception) {
+ this.pending = false;
+ this.exception = exception;
+ notifyAll();
+ }
+
+ synchronized Object getResponse() throws Exception {
+ while (pending) {
+ wait();
+ }
+ if (exception != null) {
+ throw exception;
+ }
+ return result;
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/exceptions/IPCException.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/exceptions/IPCException.java
new file mode 100644
index 0000000..9ecf015
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/exceptions/IPCException.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.exceptions;
+
+public class IPCException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public IPCException() {
+ super();
+ }
+
+ public IPCException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public IPCException(String message) {
+ super(message);
+ }
+
+ public IPCException(Throwable cause) {
+ super(cause);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/HandleState.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/HandleState.java
new file mode 100644
index 0000000..47c3d4a
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/HandleState.java
@@ -0,0 +1,23 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.impl;
+
+enum HandleState {
+ INITIAL,
+ CONNECT_SENT,
+ CONNECT_RECEIVED,
+ CONNECTED,
+ CLOSED,
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCConnectionManager.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCConnectionManager.java
new file mode 100644
index 0000000..8e42d53
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCConnectionManager.java
@@ -0,0 +1,311 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+public class IPCConnectionManager {
+ private static final Logger LOGGER = Logger.getLogger(IPCConnectionManager.class.getName());
+
+ private final IPCSystem system;
+
+ private final NetworkThread networkThread;
+
+ private final ServerSocketChannel serverSocketChannel;
+
+ private final Map<InetSocketAddress, IPCHandle> ipcHandleMap;
+
+ private final List<IPCHandle> pendingConnections;
+
+ private final List<IPCHandle> workingPendingConnections;
+
+ private final List<Message> sendList;
+
+ private final List<Message> workingSendList;
+
+ private final InetSocketAddress address;
+
+ private volatile boolean stopped;
+
+ IPCConnectionManager(IPCSystem system, InetSocketAddress socketAddress) throws IOException {
+ this.system = system;
+ this.networkThread = new NetworkThread();
+ this.serverSocketChannel = ServerSocketChannel.open();
+ serverSocketChannel.socket().setReuseAddress(true);
+ serverSocketChannel.configureBlocking(false);
+ ServerSocket socket = serverSocketChannel.socket();
+ socket.bind(socketAddress);
+ address = new InetSocketAddress(socket.getInetAddress(), socket.getLocalPort());
+ ipcHandleMap = new HashMap<InetSocketAddress, IPCHandle>();
+ pendingConnections = new ArrayList<IPCHandle>();
+ workingPendingConnections = new ArrayList<IPCHandle>();
+ sendList = new ArrayList<Message>();
+ workingSendList = new ArrayList<Message>();
+ }
+
+ InetSocketAddress getAddress() {
+ return address;
+ }
+
+ void start() {
+ stopped = false;
+ networkThread.start();
+ }
+
+ void stop() throws IOException {
+ stopped = true;
+ serverSocketChannel.close();
+ }
+
+ IPCHandle getIPCHandle(InetSocketAddress remoteAddress) throws IOException, InterruptedException {
+ IPCHandle handle;
+ synchronized (this) {
+ handle = ipcHandleMap.get(remoteAddress);
+ if (handle == null) {
+ handle = new IPCHandle(system, remoteAddress);
+ pendingConnections.add(handle);
+ networkThread.selector.wakeup();
+ }
+ }
+ handle.waitTillConnected();
+ return handle;
+ }
+
+ synchronized void registerHandle(IPCHandle handle) {
+ ipcHandleMap.put(handle.getRemoteAddress(), handle);
+ }
+
+ synchronized void write(Message msg) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Enqueued message: " + msg);
+ }
+ sendList.add(msg);
+ networkThread.selector.wakeup();
+ }
+
+ private synchronized void collectOutstandingWork() {
+ if (!pendingConnections.isEmpty()) {
+ moveAll(pendingConnections, workingPendingConnections);
+ }
+ if (!sendList.isEmpty()) {
+ moveAll(sendList, workingSendList);
+ }
+ }
+
+ private Message createInitialReqMessage(IPCHandle handle) {
+ Message msg = new Message(handle);
+ msg.setMessageId(system.createMessageId());
+ msg.setRequestMessageId(-1);
+ msg.setFlag(Message.INITIAL_REQ);
+ msg.setPayload(address);
+ return msg;
+ }
+
+ private Message createInitialAckMessage(IPCHandle handle, Message req) {
+ Message msg = new Message(handle);
+ msg.setMessageId(system.createMessageId());
+ msg.setRequestMessageId(req.getMessageId());
+ msg.setFlag(Message.INITIAL_ACK);
+ msg.setPayload(null);
+ return msg;
+ }
+
+ void ack(IPCHandle handle, Message req) {
+ write(createInitialAckMessage(handle, req));
+ }
+
+ private class NetworkThread extends Thread {
+ private final Selector selector;
+
+ public NetworkThread() {
+ super("IPC Network Listener Thread");
+ setDaemon(true);
+ try {
+ selector = Selector.open();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ serverSocketChannel.register(selector, SelectionKey.OP_ACCEPT);
+ } catch (ClosedChannelException e) {
+ throw new RuntimeException(e);
+ }
+ BitSet unsentMessagesBitmap = new BitSet();
+ List<Message> tempUnsentMessages = new ArrayList<Message>();
+ while (!stopped) {
+ try {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Starting Select");
+ }
+ int n = selector.select();
+ collectOutstandingWork();
+ if (!workingPendingConnections.isEmpty()) {
+ for (IPCHandle handle : workingPendingConnections) {
+ SocketChannel channel = SocketChannel.open();
+ channel.configureBlocking(false);
+ SelectionKey cKey = null;
+ if (channel.connect(handle.getRemoteAddress())) {
+ cKey = channel.register(selector, SelectionKey.OP_READ);
+ handle.setState(HandleState.CONNECT_SENT);
+ write(createInitialReqMessage(handle));
+ } else {
+ cKey = channel.register(selector, SelectionKey.OP_CONNECT);
+ }
+ handle.setKey(cKey);
+ cKey.attach(handle);
+ }
+ workingPendingConnections.clear();
+ }
+ if (!workingSendList.isEmpty()) {
+ unsentMessagesBitmap.clear();
+ int len = workingSendList.size();
+ for (int i = 0; i < len; ++i) {
+ Message msg = workingSendList.get(i);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Processing send of message: " + msg);
+ }
+ IPCHandle handle = msg.getIPCHandle();
+ if (handle.getState() != HandleState.CLOSED) {
+ if (!handle.full()) {
+ while (true) {
+ ByteBuffer buffer = handle.getOutBuffer();
+ buffer.compact();
+ boolean success = msg.write(buffer);
+ buffer.flip();
+ if (success) {
+ system.getPerformanceCounters().addMessageSentCount(1);
+ SelectionKey key = handle.getKey();
+ key.interestOps(key.interestOps() | SelectionKey.OP_WRITE);
+ } else {
+ if (!buffer.hasRemaining()) {
+ handle.resizeOutBuffer();
+ continue;
+ }
+ handle.markFull();
+ unsentMessagesBitmap.set(i);
+ }
+ break;
+ }
+ } else {
+ unsentMessagesBitmap.set(i);
+ }
+ }
+ }
+ copyUnsentMessages(unsentMessagesBitmap, tempUnsentMessages);
+ }
+ if (n > 0) {
+ for (Iterator<SelectionKey> i = selector.selectedKeys().iterator(); i.hasNext();) {
+ SelectionKey key = i.next();
+ i.remove();
+ SelectableChannel sc = key.channel();
+ if (key.isReadable()) {
+ SocketChannel channel = (SocketChannel) sc;
+ IPCHandle handle = (IPCHandle) key.attachment();
+ ByteBuffer readBuffer = handle.getInBuffer();
+ int len = channel.read(readBuffer);
+ system.getPerformanceCounters().addMessageBytesReceived(len);
+ if (len < 0) {
+ key.cancel();
+ channel.close();
+ handle.close();
+ } else {
+ handle.processIncomingMessages();
+ if (!readBuffer.hasRemaining()) {
+ handle.resizeInBuffer();
+ }
+ }
+ } else if (key.isWritable()) {
+ SocketChannel channel = (SocketChannel) sc;
+ IPCHandle handle = (IPCHandle) key.attachment();
+ ByteBuffer writeBuffer = handle.getOutBuffer();
+ int len = channel.write(writeBuffer);
+ system.getPerformanceCounters().addMessageBytesSent(len);
+ if (len < 0) {
+ key.cancel();
+ channel.close();
+ handle.close();
+ } else if (!writeBuffer.hasRemaining()) {
+ key.interestOps(key.interestOps() & ~SelectionKey.OP_WRITE);
+ }
+ if (handle.full()) {
+ handle.clearFull();
+ selector.wakeup();
+ }
+ } else if (key.isAcceptable()) {
+ assert sc == serverSocketChannel;
+ SocketChannel channel = serverSocketChannel.accept();
+ channel.configureBlocking(false);
+ IPCHandle handle = new IPCHandle(system, null);
+ SelectionKey cKey = channel.register(selector, SelectionKey.OP_READ);
+ handle.setKey(cKey);
+ cKey.attach(handle);
+ handle.setState(HandleState.CONNECT_RECEIVED);
+ } else if (key.isConnectable()) {
+ SocketChannel channel = (SocketChannel) sc;
+ if (channel.finishConnect()) {
+ IPCHandle handle = (IPCHandle) key.attachment();
+ handle.setState(HandleState.CONNECT_SENT);
+ registerHandle(handle);
+ key.interestOps(SelectionKey.OP_READ);
+ write(createInitialReqMessage(handle));
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private void copyUnsentMessages(BitSet unsentMessagesBitmap, List<Message> tempUnsentMessages) {
+ assert tempUnsentMessages.isEmpty();
+ for (int i = unsentMessagesBitmap.nextSetBit(0); i >= 0; i = unsentMessagesBitmap.nextSetBit(i + 1)) {
+ tempUnsentMessages.add(workingSendList.get(i));
+ }
+ workingSendList.clear();
+ moveAll(tempUnsentMessages, workingSendList);
+ }
+ }
+
+ private <T> void moveAll(List<T> source, List<T> target) {
+ int len = source.size();
+ for (int i = 0; i < len; ++i) {
+ target.add(source.get(i));
+ }
+ source.clear();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java
new file mode 100644
index 0000000..749afe2
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCHandle.java
@@ -0,0 +1,190 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.impl;
+
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+
+final class IPCHandle implements IIPCHandle {
+ private static final int DEFAULT_BUFFER_SIZE = 1024 * 1024;
+
+ private final IPCSystem system;
+
+ private InetSocketAddress remoteAddress;
+
+ private HandleState state;
+
+ private SelectionKey key;
+
+ private Object attachment;
+
+ private ByteBuffer inBuffer;
+
+ private ByteBuffer outBuffer;
+
+ private boolean full;
+
+ IPCHandle(IPCSystem system, InetSocketAddress remoteAddress) {
+ this.system = system;
+ this.remoteAddress = remoteAddress;
+ inBuffer = ByteBuffer.allocate(DEFAULT_BUFFER_SIZE);
+ outBuffer = ByteBuffer.allocate(DEFAULT_BUFFER_SIZE);
+ outBuffer.flip();
+ state = HandleState.INITIAL;
+ }
+
+ @Override
+ public InetSocketAddress getRemoteAddress() {
+ return remoteAddress;
+ }
+
+ IPCSystem getIPCSystem() {
+ return system;
+ }
+
+ void setRemoteAddress(InetSocketAddress remoteAddress) {
+ this.remoteAddress = remoteAddress;
+ }
+
+ @Override
+ public synchronized long send(long requestId, Object req, Exception exception) throws IPCException {
+ if (state != HandleState.CONNECTED) {
+ throw new IPCException("Handle is not in Connected state");
+ }
+ Message msg = new Message(this);
+ long mid = system.createMessageId();
+ msg.setMessageId(mid);
+ msg.setRequestMessageId(requestId);
+ if (exception != null) {
+ msg.setFlag(Message.ERROR);
+ msg.setPayload(exception);
+ } else {
+ msg.setFlag(Message.NORMAL);
+ msg.setPayload(req);
+ }
+ system.getConnectionManager().write(msg);
+ return mid;
+ }
+
+ @Override
+ public void setAttachment(Object attachment) {
+ this.attachment = attachment;
+ }
+
+ @Override
+ public Object getAttachment() {
+ return attachment;
+ }
+
+ SelectionKey getKey() {
+ return key;
+ }
+
+ void setKey(SelectionKey key) {
+ this.key = key;
+ }
+
+ public synchronized boolean isConnected() {
+ return state == HandleState.CONNECTED;
+ }
+
+ synchronized HandleState getState() {
+ return state;
+ }
+
+ synchronized void setState(HandleState state) {
+ this.state = state;
+ notifyAll();
+ }
+
+ synchronized void waitTillConnected() throws InterruptedException {
+ while (!isConnected()) {
+ wait();
+ }
+ }
+
+ ByteBuffer getInBuffer() {
+ return inBuffer;
+ }
+
+ ByteBuffer getOutBuffer() {
+ return outBuffer;
+ }
+
+ synchronized void close() {
+ setState(HandleState.CLOSED);
+ }
+
+ synchronized void processIncomingMessages() {
+ inBuffer.flip();
+ while (Message.hasMessage(inBuffer)) {
+ Message message = new Message(this);
+ try {
+ message.read(inBuffer);
+ } catch (Exception e) {
+ message.setFlag(Message.ERROR);
+ message.setPayload(e);
+ }
+ system.getPerformanceCounters().addMessageReceivedCount(1);
+
+ if (state == HandleState.CONNECT_RECEIVED) {
+ remoteAddress = (InetSocketAddress) message.getPayload();
+ system.getConnectionManager().registerHandle(this);
+ setState(HandleState.CONNECTED);
+ system.getConnectionManager().ack(this, message);
+ continue;
+ } else if (state == HandleState.CONNECT_SENT) {
+ if (message.getFlag() == Message.INITIAL_ACK) {
+ setState(HandleState.CONNECTED);
+ } else {
+ throw new IllegalStateException();
+ }
+ continue;
+ }
+ system.deliverIncomingMessage(message);
+ }
+ inBuffer.compact();
+ }
+
+ void resizeInBuffer() {
+ inBuffer.flip();
+ ByteBuffer readBuffer = ByteBuffer.allocate(inBuffer.capacity() * 2);
+ readBuffer.put(inBuffer);
+ inBuffer = readBuffer;
+ }
+
+ void resizeOutBuffer() {
+ ByteBuffer writeBuffer = ByteBuffer.allocate(outBuffer.capacity() * 2);
+ writeBuffer.put(outBuffer);
+ writeBuffer.flip();
+ outBuffer = writeBuffer;
+ }
+
+ void markFull() {
+ full = true;
+ }
+
+ void clearFull() {
+ full = false;
+ }
+
+ boolean full() {
+ return full;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCSystem.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCSystem.java
new file mode 100644
index 0000000..d7e383d
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/IPCSystem.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.impl;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicLong;
+
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.IIPCI;
+import edu.uci.ics.hyracks.ipc.api.IPCPerformanceCounters;
+import edu.uci.ics.hyracks.ipc.api.IPayloadSerializerDeserializer;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+
+public class IPCSystem {
+ private final IPCConnectionManager cMgr;
+
+ private final IIPCI ipci;
+
+ private final IPayloadSerializerDeserializer serde;
+
+ private final AtomicLong midFactory;
+
+ private final IPCPerformanceCounters perfCounters;
+
+ public IPCSystem(InetSocketAddress socketAddress, IIPCI ipci, IPayloadSerializerDeserializer serde)
+ throws IOException {
+ cMgr = new IPCConnectionManager(this, socketAddress);
+ this.ipci = ipci;
+ this.serde = serde;
+ midFactory = new AtomicLong();
+ perfCounters = new IPCPerformanceCounters();
+ }
+
+ public InetSocketAddress getSocketAddress() {
+ return cMgr.getAddress();
+ }
+
+ public void start() {
+ cMgr.start();
+ }
+
+ public IIPCHandle getHandle(InetSocketAddress remoteAddress) throws IPCException {
+ try {
+ return cMgr.getIPCHandle(remoteAddress);
+ } catch (IOException e) {
+ throw new IPCException(e);
+ } catch (InterruptedException e) {
+ throw new IPCException(e);
+ }
+ }
+
+ IPayloadSerializerDeserializer getSerializerDeserializer() {
+ return serde;
+ }
+
+ long createMessageId() {
+ return midFactory.incrementAndGet();
+ }
+
+ void deliverIncomingMessage(final Message message) {
+ long mid = message.getMessageId();
+ long rmid = message.getRequestMessageId();
+ Object payload = null;
+ Exception exception = null;
+ if (message.getFlag() == Message.ERROR) {
+ exception = (Exception) message.getPayload();
+ } else {
+ payload = message.getPayload();
+ }
+ ipci.deliverIncomingMessage(message.getIPCHandle(), mid, rmid, payload, exception);
+ }
+
+ IPCConnectionManager getConnectionManager() {
+ return cMgr;
+ }
+
+ public IPCPerformanceCounters getPerformanceCounters() {
+ return perfCounters;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/JavaSerializationBasedPayloadSerializerDeserializer.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/JavaSerializationBasedPayloadSerializerDeserializer.java
new file mode 100644
index 0000000..fdf8e92
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/JavaSerializationBasedPayloadSerializerDeserializer.java
@@ -0,0 +1,53 @@
+package edu.uci.ics.hyracks.ipc.impl;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.ipc.api.IPayloadSerializerDeserializer;
+
+public class JavaSerializationBasedPayloadSerializerDeserializer implements IPayloadSerializerDeserializer {
+ @Override
+ public Object deserializeObject(ByteBuffer buffer, int length) throws Exception {
+ return deserialize(buffer, length);
+ }
+
+ @Override
+ public Exception deserializeException(ByteBuffer buffer, int length) throws Exception {
+ return (Exception) deserialize(buffer, length);
+ }
+
+ @Override
+ public byte[] serializeObject(Object object) throws Exception {
+ return serialize(object);
+ }
+
+ @Override
+ public byte[] serializeException(Exception exception) throws Exception {
+ return serialize(exception);
+ }
+
+ public static void serialize(OutputStream out, Object object) throws Exception {
+ ObjectOutputStream oos = new ObjectOutputStream(out);
+ oos.writeObject(object);
+ oos.flush();
+ }
+
+ private Object deserialize(ByteBuffer buffer, int length) throws Exception {
+ ObjectInputStream ois = new ObjectInputStream(new ByteArrayInputStream(buffer.array(), buffer.position(),
+ length));
+ Object object = ois.readObject();
+ ois.close();
+ return object;
+ }
+
+ private byte[] serialize(Object object) throws Exception {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ serialize(baos, object);
+ baos.close();
+ return baos.toByteArray();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/Message.java b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/Message.java
new file mode 100644
index 0000000..6bb3156
--- /dev/null
+++ b/hyracks-ipc/src/main/java/edu/uci/ics/hyracks/ipc/impl/Message.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.impl;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.ipc.api.IPayloadSerializerDeserializer;
+
+class Message {
+ private static final int MSG_SIZE_SIZE = 4;
+
+ private static final int HEADER_SIZE = 17;
+
+ static final byte INITIAL_REQ = 1;
+
+ static final byte INITIAL_ACK = 2;
+
+ static final byte ERROR = 3;
+
+ static final byte NORMAL = 0;
+
+ private IPCHandle ipcHandle;
+
+ private long messageId;
+
+ private long requestMessageId;
+
+ private byte flag;
+
+ private Object payload;
+
+ Message(IPCHandle ipcHandle) {
+ this.ipcHandle = ipcHandle;
+ }
+
+ IPCHandle getIPCHandle() {
+ return ipcHandle;
+ }
+
+ void setMessageId(long messageId) {
+ this.messageId = messageId;
+ }
+
+ long getMessageId() {
+ return messageId;
+ }
+
+ void setRequestMessageId(long requestMessageId) {
+ this.requestMessageId = requestMessageId;
+ }
+
+ long getRequestMessageId() {
+ return requestMessageId;
+ }
+
+ void setFlag(byte flag) {
+ this.flag = flag;
+ }
+
+ byte getFlag() {
+ return flag;
+ }
+
+ void setPayload(Object payload) {
+ this.payload = payload;
+ }
+
+ Object getPayload() {
+ return payload;
+ }
+
+ static boolean hasMessage(ByteBuffer buffer) {
+ if (buffer.remaining() < MSG_SIZE_SIZE) {
+ return false;
+ }
+ int msgSize = buffer.getInt(buffer.position());
+ return buffer.remaining() >= msgSize + MSG_SIZE_SIZE;
+ }
+
+ void read(ByteBuffer buffer) throws Exception {
+ assert hasMessage(buffer);
+ int msgSize = buffer.getInt();
+ messageId = buffer.getLong();
+ requestMessageId = buffer.getLong();
+ flag = buffer.get();
+ int finalPosition = buffer.position() + msgSize - HEADER_SIZE;
+ int length = msgSize - HEADER_SIZE;
+ try {
+ IPayloadSerializerDeserializer serde = ipcHandle.getIPCSystem().getSerializerDeserializer();
+ payload = flag == ERROR ? serde.deserializeException(buffer, length) : serde.deserializeObject(buffer,
+ length);
+ } finally {
+ buffer.position(finalPosition);
+ }
+ }
+
+ boolean write(ByteBuffer buffer) throws Exception {
+ IPayloadSerializerDeserializer serde = ipcHandle.getIPCSystem().getSerializerDeserializer();
+ byte[] bytes = flag == ERROR ? serde.serializeException((Exception) payload) : serde.serializeObject(payload);
+ if (buffer.remaining() >= MSG_SIZE_SIZE + HEADER_SIZE + bytes.length) {
+ buffer.putInt(HEADER_SIZE + bytes.length);
+ buffer.putLong(messageId);
+ buffer.putLong(requestMessageId);
+ buffer.put(flag);
+ buffer.put(bytes);
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return "MSG[" + messageId + ":" + requestMessageId + ":" + flag + ":" + payload + "]";
+ }
+}
\ No newline at end of file
diff --git a/hyracks-ipc/src/test/java/edu/uci/ics/hyracks/ipc/tests/IPCTest.java b/hyracks-ipc/src/test/java/edu/uci/ics/hyracks/ipc/tests/IPCTest.java
new file mode 100644
index 0000000..5b2f660
--- /dev/null
+++ b/hyracks-ipc/src/test/java/edu/uci/ics/hyracks/ipc/tests/IPCTest.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.ipc.tests;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.ipc.api.IIPCHandle;
+import edu.uci.ics.hyracks.ipc.api.IIPCI;
+import edu.uci.ics.hyracks.ipc.api.RPCInterface;
+import edu.uci.ics.hyracks.ipc.exceptions.IPCException;
+import edu.uci.ics.hyracks.ipc.impl.IPCSystem;
+import edu.uci.ics.hyracks.ipc.impl.JavaSerializationBasedPayloadSerializerDeserializer;
+
+public class IPCTest {
+ @Test
+ public void test() throws Exception {
+ IPCSystem server = createServerIPCSystem();
+ server.start();
+ InetSocketAddress serverAddr = server.getSocketAddress();
+
+ RPCInterface rpci = new RPCInterface();
+ IPCSystem client = createClientIPCSystem(rpci);
+ client.start();
+
+ IIPCHandle handle = client.getHandle(serverAddr);
+
+ for (int i = 0; i < 100; ++i) {
+ Assert.assertEquals(rpci.call(handle, Integer.valueOf(i)), Integer.valueOf(2 * i));
+ }
+
+ try {
+ rpci.call(handle, "Foo");
+ Assert.assertTrue(false);
+ } catch (Exception e) {
+ Assert.assertTrue(true);
+ }
+ }
+
+ private IPCSystem createServerIPCSystem() throws IOException {
+ final Executor executor = Executors.newCachedThreadPool();
+ IIPCI ipci = new IIPCI() {
+ @Override
+ public void deliverIncomingMessage(final IIPCHandle handle, final long mid, long rmid,
+ final Object payload, Exception exception) {
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ Object result = null;
+ Exception exception = null;
+ try {
+ Integer i = (Integer) payload;
+ result = i.intValue() * 2;
+ } catch (Exception e) {
+ exception = e;
+ }
+ try {
+ handle.send(mid, result, exception);
+ } catch (IPCException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ }
+ };
+ return new IPCSystem(new InetSocketAddress("127.0.0.1", 0), ipci,
+ new JavaSerializationBasedPayloadSerializerDeserializer());
+ }
+
+ private IPCSystem createClientIPCSystem(RPCInterface rpci) throws IOException {
+ return new IPCSystem(new InetSocketAddress("127.0.0.1", 0), rpci,
+ new JavaSerializationBasedPayloadSerializerDeserializer());
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/pom.xml b/hyracks-net/pom.xml
new file mode 100644
index 0000000..12004f7
--- /dev/null
+++ b/hyracks-net/pom.xml
@@ -0,0 +1,31 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>hyracks-net</artifactId>
+ <parent>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
+ </parent>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>2.0.2</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.8.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java
new file mode 100644
index 0000000..2f27bf0
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/IBufferAcceptor.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.buffers;
+
+import java.nio.ByteBuffer;
+
+public interface IBufferAcceptor {
+ public void accept(ByteBuffer buffer);
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java
new file mode 100644
index 0000000..c395ac9
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/buffers/ICloseableBufferAcceptor.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.buffers;
+
+public interface ICloseableBufferAcceptor extends IBufferAcceptor {
+ public void close();
+
+ public void error(int ecode);
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/exceptions/NetException.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/exceptions/NetException.java
new file mode 100644
index 0000000..ecd0373
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/exceptions/NetException.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.exceptions;
+
+public class NetException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public NetException() {
+ }
+
+ public NetException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+ public NetException(String message) {
+ super(message);
+ }
+
+ public NetException(Throwable cause) {
+ super(cause);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
new file mode 100644
index 0000000..4b55d4b
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelControlBlock.java
@@ -0,0 +1,347 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SocketChannel;
+import java.util.ArrayDeque;
+import java.util.Deque;
+import java.util.Queue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+
+public class ChannelControlBlock {
+ private static final Logger LOGGER = Logger.getLogger(ChannelControlBlock.class.getName());
+
+ private final ChannelSet cSet;
+
+ private final int channelId;
+
+ private final ReadInterface ri;
+
+ private final WriteInterface wi;
+
+ private final AtomicBoolean localClose;
+
+ private final AtomicBoolean localCloseAck;
+
+ private final AtomicBoolean remoteClose;
+
+ ChannelControlBlock(ChannelSet cSet, int channelId) {
+ this.cSet = cSet;
+ this.channelId = channelId;
+ this.ri = new ReadInterface();
+ this.wi = new WriteInterface();
+ localClose = new AtomicBoolean();
+ localCloseAck = new AtomicBoolean();
+ remoteClose = new AtomicBoolean();
+ }
+
+ int getChannelId() {
+ return channelId;
+ }
+
+ public IChannelReadInterface getReadInterface() {
+ return ri;
+ }
+
+ public IChannelWriteInterface getWriteInterface() {
+ return wi;
+ }
+
+ private final class ReadInterface implements IChannelReadInterface {
+ private final Deque<ByteBuffer> riEmptyStack;
+
+ private final IBufferAcceptor eba = new IBufferAcceptor() {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ int delta = buffer.remaining();
+ synchronized (ChannelControlBlock.this) {
+ if (remoteClose.get()) {
+ return;
+ }
+ riEmptyStack.push(buffer);
+ }
+ cSet.addPendingCredits(channelId, delta);
+ }
+ };
+
+ private ICloseableBufferAcceptor fba;
+
+ private volatile int credits;
+
+ private ByteBuffer currentReadBuffer;
+
+ ReadInterface() {
+ riEmptyStack = new ArrayDeque<ByteBuffer>();
+ credits = 0;
+ }
+
+ @Override
+ public void setFullBufferAcceptor(ICloseableBufferAcceptor fullBufferAcceptor) {
+ fba = fullBufferAcceptor;
+ }
+
+ @Override
+ public IBufferAcceptor getEmptyBufferAcceptor() {
+ return eba;
+ }
+
+ int read(SocketChannel sc, int size) throws IOException, NetException {
+ while (true) {
+ if (size <= 0) {
+ return size;
+ }
+ if (ri.currentReadBuffer == null) {
+ ri.currentReadBuffer = ri.riEmptyStack.poll();
+ assert ri.currentReadBuffer != null;
+ }
+ int rSize = Math.min(size, ri.currentReadBuffer.remaining());
+ if (rSize > 0) {
+ ri.currentReadBuffer.limit(ri.currentReadBuffer.position() + rSize);
+ int len;
+ try {
+ len = sc.read(ri.currentReadBuffer);
+ if (len < 0) {
+ throw new NetException("Socket Closed");
+ }
+ } finally {
+ ri.currentReadBuffer.limit(ri.currentReadBuffer.capacity());
+ }
+ size -= len;
+ if (len < rSize) {
+ return size;
+ }
+ } else {
+ return size;
+ }
+ if (ri.currentReadBuffer.remaining() <= 0) {
+ flush();
+ }
+ }
+ }
+
+ void flush() {
+ if (currentReadBuffer != null) {
+ currentReadBuffer.flip();
+ fba.accept(ri.currentReadBuffer);
+ currentReadBuffer = null;
+ }
+ }
+ }
+
+ private final class WriteInterface implements IChannelWriteInterface {
+ private final Queue<ByteBuffer> wiFullQueue;
+
+ private boolean channelWritabilityState;
+
+ private final ICloseableBufferAcceptor fba = new ICloseableBufferAcceptor() {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ synchronized (ChannelControlBlock.this) {
+ wiFullQueue.add(buffer);
+ adjustChannelWritability();
+ }
+ }
+
+ @Override
+ public void close() {
+ synchronized (ChannelControlBlock.this) {
+ if (eos) {
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Received duplicate close() on channel: " + channelId);
+ }
+ return;
+ }
+ eos = true;
+ adjustChannelWritability();
+ }
+ }
+
+ @Override
+ public void error(int ecode) {
+ synchronized (ChannelControlBlock.this) {
+ WriteInterface.this.ecode = ecode;
+ adjustChannelWritability();
+ }
+ }
+ };
+
+ private IBufferAcceptor eba;
+
+ private int credits;
+
+ private boolean eos;
+
+ private boolean eosSent;
+
+ private int ecode;
+
+ private boolean ecodeSent;
+
+ private ByteBuffer currentWriteBuffer;
+
+ WriteInterface() {
+ wiFullQueue = new ArrayDeque<ByteBuffer>();
+ credits = 0;
+ eos = false;
+ eosSent = false;
+ ecode = -1;
+ ecodeSent = false;
+ }
+
+ @Override
+ public void setEmptyBufferAcceptor(IBufferAcceptor emptyBufferAcceptor) {
+ eba = emptyBufferAcceptor;
+ }
+
+ @Override
+ public ICloseableBufferAcceptor getFullBufferAcceptor() {
+ return fba;
+ }
+
+ void write(MultiplexedConnection.WriterState writerState) throws NetException {
+ if (currentWriteBuffer == null) {
+ currentWriteBuffer = wiFullQueue.poll();
+ }
+ if (currentWriteBuffer != null) {
+ int size = Math.min(currentWriteBuffer.remaining(), credits);
+ if (size > 0) {
+ credits -= size;
+ writerState.command.setChannelId(channelId);
+ writerState.command.setCommandType(MuxDemuxCommand.CommandType.DATA);
+ writerState.command.setData(size);
+ writerState.reset(currentWriteBuffer, size, ChannelControlBlock.this);
+ } else {
+ adjustChannelWritability();
+ }
+ } else if (ecode >= 0 && !ecodeSent) {
+ writerState.command.setChannelId(channelId);
+ writerState.command.setCommandType(MuxDemuxCommand.CommandType.ERROR);
+ writerState.command.setData(ecode);
+ writerState.reset(null, 0, null);
+ ecodeSent = true;
+ localClose.set(true);
+ adjustChannelWritability();
+ } else if (wi.eos && !wi.eosSent) {
+ writerState.command.setChannelId(channelId);
+ writerState.command.setCommandType(MuxDemuxCommand.CommandType.CLOSE_CHANNEL);
+ writerState.command.setData(0);
+ writerState.reset(null, 0, null);
+ eosSent = true;
+ localClose.set(true);
+ adjustChannelWritability();
+ }
+ }
+
+ void writeComplete() {
+ if (currentWriteBuffer.remaining() <= 0) {
+ currentWriteBuffer.clear();
+ eba.accept(currentWriteBuffer);
+ currentWriteBuffer = null;
+ adjustChannelWritability();
+ }
+ }
+
+ private boolean computeWritability() {
+ boolean writableDataPresent = currentWriteBuffer != null || !wiFullQueue.isEmpty();
+ if (writableDataPresent) {
+ return credits > 0;
+ }
+ if (eos && !eosSent) {
+ return true;
+ }
+ if (ecode >= 0 && !ecodeSent) {
+ return true;
+ }
+ return false;
+ }
+
+ void adjustChannelWritability() {
+ boolean writable = computeWritability();
+ if (writable) {
+ if (!channelWritabilityState) {
+ cSet.markPendingWrite(channelId);
+ }
+ } else {
+ if (channelWritabilityState) {
+ cSet.unmarkPendingWrite(channelId);
+ }
+ }
+ channelWritabilityState = writable;
+ }
+ }
+
+ synchronized void write(MultiplexedConnection.WriterState writerState) throws NetException {
+ wi.write(writerState);
+ }
+
+ synchronized void writeComplete() {
+ wi.writeComplete();
+ }
+
+ synchronized int read(SocketChannel sc, int size) throws IOException, NetException {
+ return ri.read(sc, size);
+ }
+
+ int getReadCredits() {
+ return ri.credits;
+ }
+
+ void setReadCredits(int credits) {
+ this.ri.credits = credits;
+ }
+
+ synchronized void addWriteCredits(int delta) {
+ wi.credits += delta;
+ wi.adjustChannelWritability();
+ }
+
+ synchronized void reportRemoteEOS() {
+ ri.flush();
+ ri.fba.close();
+ remoteClose.set(true);
+ }
+
+ boolean getRemoteEOS() {
+ return remoteClose.get();
+ }
+
+ synchronized void reportLocalEOSAck() {
+ localCloseAck.set(true);
+ }
+
+ synchronized void reportRemoteError(int ecode) {
+ ri.flush();
+ ri.fba.error(ecode);
+ remoteClose.set(true);
+ }
+
+ boolean completelyClosed() {
+ return localCloseAck.get() && remoteClose.get();
+ }
+
+ @Override
+ public String toString() {
+ return "Channel:" + channelId + "[localClose: " + localClose + " localCloseAck: " + localCloseAck
+ + " remoteClose: " + remoteClose + " readCredits: " + ri.credits + " writeCredits: " + wi.credits + "]";
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java
new file mode 100644
index 0000000..ffdb5c6
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/ChannelSet.java
@@ -0,0 +1,222 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.util.Arrays;
+import java.util.BitSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+
+public class ChannelSet {
+ private static final Logger LOGGER = Logger.getLogger(ChannelSet.class.getName());
+
+ private static final int MAX_OPEN_CHANNELS = 1024;
+
+ private static final int INITIAL_SIZE = 16;
+
+ private final MultiplexedConnection mConn;
+
+ private ChannelControlBlock[] ccbArray;
+
+ private final BitSet allocationBitmap;
+
+ private final BitSet pendingChannelWriteBitmap;
+
+ private final BitSet pendingChannelCreditsBitmap;
+
+ private final BitSet pendingChannelSynBitmap;
+
+ private final BitSet pendingEOSAckBitmap;
+
+ private int openChannelCount;
+
+ private final IEventCounter pendingWriteEventsCounter;
+
+ ChannelSet(MultiplexedConnection mConn, IEventCounter pendingWriteEventsCounter) {
+ this.mConn = mConn;
+ ccbArray = new ChannelControlBlock[INITIAL_SIZE];
+ allocationBitmap = new BitSet();
+ pendingChannelWriteBitmap = new BitSet();
+ pendingChannelCreditsBitmap = new BitSet();
+ pendingChannelSynBitmap = new BitSet();
+ pendingEOSAckBitmap = new BitSet();
+ this.pendingWriteEventsCounter = pendingWriteEventsCounter;
+ openChannelCount = 0;
+ }
+
+ ChannelControlBlock allocateChannel() throws NetException {
+ synchronized (mConn) {
+ int idx = allocationBitmap.nextClearBit(0);
+ if (idx < 0 || idx == ccbArray.length) {
+ cleanupClosedChannels();
+ idx = allocationBitmap.nextClearBit(0);
+ if (idx < 0 || idx == ccbArray.length) {
+ idx = ccbArray.length;
+ }
+ }
+ return createChannel(idx);
+ }
+ }
+
+ private void cleanupClosedChannels() {
+ for (int i = 0; i < ccbArray.length; ++i) {
+ ChannelControlBlock ccb = ccbArray[i];
+ if (ccb != null) {
+ if (ccb.completelyClosed()) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Cleaning free channel: " + ccb);
+ }
+ freeChannel(ccb);
+ }
+ }
+ }
+ }
+
+ ChannelControlBlock registerChannel(int channelId) throws NetException {
+ synchronized (mConn) {
+ return createChannel(channelId);
+ }
+ }
+
+ private void freeChannel(ChannelControlBlock channel) {
+ int idx = channel.getChannelId();
+ ccbArray[idx] = null;
+ allocationBitmap.clear(idx);
+ pendingChannelWriteBitmap.clear(idx);
+ pendingChannelCreditsBitmap.clear(idx);
+ pendingChannelSynBitmap.clear(idx);
+ pendingEOSAckBitmap.clear(idx);
+ --openChannelCount;
+ }
+
+ ChannelControlBlock getCCB(int channelId) {
+ return ccbArray[channelId];
+ }
+
+ BitSet getPendingChannelWriteBitmap() {
+ return pendingChannelWriteBitmap;
+ }
+
+ BitSet getPendingChannelCreditsBitmap() {
+ return pendingChannelCreditsBitmap;
+ }
+
+ BitSet getPendingChannelSynBitmap() {
+ return pendingChannelSynBitmap;
+ }
+
+ BitSet getPendingEOSAckBitmap() {
+ return pendingEOSAckBitmap;
+ }
+
+ int getOpenChannelCount() {
+ return openChannelCount;
+ }
+
+ void initiateChannelSyn(int channelId) {
+ synchronized (mConn) {
+ assert !pendingChannelSynBitmap.get(channelId);
+ pendingChannelSynBitmap.set(channelId);
+ pendingWriteEventsCounter.increment();
+ }
+ }
+
+ void addPendingCredits(int channelId, int delta) {
+ if (delta <= 0) {
+ return;
+ }
+ synchronized (mConn) {
+ ChannelControlBlock ccb = ccbArray[channelId];
+ if (ccb != null) {
+ if (ccb.getRemoteEOS()) {
+ return;
+ }
+ int oldCredits = ccb.getReadCredits();
+ ccb.setReadCredits(oldCredits + delta);
+ if (oldCredits == 0) {
+ assert !pendingChannelCreditsBitmap.get(channelId);
+ pendingChannelCreditsBitmap.set(channelId);
+ pendingWriteEventsCounter.increment();
+ }
+ }
+ }
+ }
+
+ void unmarkPendingCredits(int channelId) {
+ synchronized (mConn) {
+ if (pendingChannelCreditsBitmap.get(channelId)) {
+ pendingChannelCreditsBitmap.clear(channelId);
+ pendingWriteEventsCounter.decrement();
+ }
+ }
+ }
+
+ void markPendingWrite(int channelId) {
+ synchronized (mConn) {
+ assert !pendingChannelWriteBitmap.get(channelId);
+ pendingChannelWriteBitmap.set(channelId);
+ pendingWriteEventsCounter.increment();
+ }
+ }
+
+ void unmarkPendingWrite(int channelId) {
+ synchronized (mConn) {
+ assert pendingChannelWriteBitmap.get(channelId);
+ pendingChannelWriteBitmap.clear(channelId);
+ pendingWriteEventsCounter.decrement();
+ }
+ }
+
+ void markEOSAck(int channelId) {
+ synchronized (mConn) {
+ assert !pendingEOSAckBitmap.get(channelId);
+ pendingEOSAckBitmap.set(channelId);
+ pendingWriteEventsCounter.increment();
+ }
+ }
+
+ private ChannelControlBlock createChannel(int idx) throws NetException {
+ if (idx >= ccbArray.length) {
+ expand(idx);
+ }
+ if (idx > MAX_OPEN_CHANNELS) {
+ throw new NetException("More than " + MAX_OPEN_CHANNELS + " opened concurrently");
+ }
+ if (ccbArray[idx] != null) {
+ assert ccbArray[idx].completelyClosed();
+ if (ccbArray[idx].completelyClosed()) {
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Cleaning free channel: " + ccbArray[idx]);
+ }
+ freeChannel(ccbArray[idx]);
+ }
+ }
+ assert idx < ccbArray.length;
+ assert !allocationBitmap.get(idx);
+ ChannelControlBlock channel = new ChannelControlBlock(this, idx);
+ ccbArray[idx] = channel;
+ allocationBitmap.set(idx);
+ ++openChannelCount;
+ return channel;
+ }
+
+ private void expand(int idx) {
+ while (idx >= ccbArray.length) {
+ ccbArray = Arrays.copyOf(ccbArray, ccbArray.length * 2);
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java
new file mode 100644
index 0000000..0fc9b2a
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelOpenListener.java
@@ -0,0 +1,19 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+public interface IChannelOpenListener {
+ public void channelOpened(ChannelControlBlock channel);
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java
new file mode 100644
index 0000000..468a617
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelReadInterface.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+
+public interface IChannelReadInterface {
+ public void setFullBufferAcceptor(ICloseableBufferAcceptor fullBufferAcceptor);
+
+ public IBufferAcceptor getEmptyBufferAcceptor();
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java
new file mode 100644
index 0000000..1e53d71
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IChannelWriteInterface.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+
+public interface IChannelWriteInterface {
+ public void setEmptyBufferAcceptor(IBufferAcceptor emptyBufferAcceptor);
+
+ public ICloseableBufferAcceptor getFullBufferAcceptor();
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IEventCounter.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IEventCounter.java
new file mode 100644
index 0000000..148078c
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/IEventCounter.java
@@ -0,0 +1,21 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+public interface IEventCounter {
+ public void increment();
+
+ public void decrement();
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java
new file mode 100644
index 0000000..c905f57
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MultiplexedConnection.java
@@ -0,0 +1,384 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.SocketChannel;
+import java.util.BitSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+import edu.uci.ics.hyracks.net.protocols.tcp.ITCPConnectionEventListener;
+import edu.uci.ics.hyracks.net.protocols.tcp.TCPConnection;
+
+public class MultiplexedConnection implements ITCPConnectionEventListener {
+ private static final Logger LOGGER = Logger.getLogger(MultiplexedConnection.class.getName());
+
+ private static final int MAX_CHUNKS_READ_PER_CYCLE = 4;
+
+ private final MuxDemux muxDemux;
+
+ private final IEventCounter pendingWriteEventsCounter;
+
+ private final ChannelSet cSet;
+
+ private final ReaderState readerState;
+
+ private final WriterState writerState;
+
+ private TCPConnection tcpConnection;
+
+ private int lastChannelWritten;
+
+ private int nConnectionAttempts;
+
+ private boolean connectionFailure;
+
+ public MultiplexedConnection(MuxDemux muxDemux) {
+ this.muxDemux = muxDemux;
+ pendingWriteEventsCounter = new IEventCounter() {
+ private int counter;
+
+ @Override
+ public synchronized void increment() {
+ ++counter;
+ if (counter == 1) {
+ tcpConnection.enable(SelectionKey.OP_WRITE);
+ }
+ }
+
+ @Override
+ public synchronized void decrement() {
+ --counter;
+ if (counter == 0) {
+ tcpConnection.disable(SelectionKey.OP_WRITE);
+ }
+ if (counter < 0) {
+ throw new IllegalStateException();
+ }
+ }
+ };
+ cSet = new ChannelSet(this, pendingWriteEventsCounter);
+ readerState = new ReaderState();
+ writerState = new WriterState();
+ lastChannelWritten = -1;
+ connectionFailure = false;
+ }
+
+ int getConnectionAttempts() {
+ return nConnectionAttempts;
+ }
+
+ void setConnectionAttempts(int nConnectionAttempts) {
+ this.nConnectionAttempts = nConnectionAttempts;
+ }
+
+ synchronized void setTCPConnection(TCPConnection tcpConnection) {
+ this.tcpConnection = tcpConnection;
+ tcpConnection.enable(SelectionKey.OP_READ);
+ notifyAll();
+ }
+
+ synchronized void setConnectionFailure() {
+ this.connectionFailure = true;
+ notifyAll();
+ }
+
+ synchronized void waitUntilConnected() throws InterruptedException, NetException {
+ while (tcpConnection == null && !connectionFailure) {
+ wait();
+ }
+ if (connectionFailure) {
+ throw new NetException("Connection failure");
+ }
+ }
+
+ @Override
+ public void notifyIOReady(TCPConnection connection, boolean readable, boolean writable) throws IOException,
+ NetException {
+ if (readable) {
+ driveReaderStateMachine();
+ }
+ if (writable) {
+ driveWriterStateMachine();
+ }
+ }
+
+ public ChannelControlBlock openChannel() throws NetException, InterruptedException {
+ ChannelControlBlock channel = cSet.allocateChannel();
+ int channelId = channel.getChannelId();
+ cSet.initiateChannelSyn(channelId);
+ return channel;
+ }
+
+ class WriterState {
+ private final ByteBuffer writeBuffer;
+
+ final MuxDemuxCommand command;
+
+ private ByteBuffer pendingBuffer;
+
+ private int pendingWriteSize;
+
+ private ChannelControlBlock ccb;
+
+ public WriterState() {
+ writeBuffer = ByteBuffer.allocateDirect(MuxDemuxCommand.COMMAND_SIZE);
+ writeBuffer.flip();
+ command = new MuxDemuxCommand();
+ ccb = null;
+ }
+
+ boolean writePending() {
+ return writeBuffer.remaining() > 0 || (pendingBuffer != null && pendingWriteSize > 0);
+ }
+
+ void reset(ByteBuffer pendingBuffer, int pendingWriteSize, ChannelControlBlock ccb) {
+ writeBuffer.clear();
+ command.write(writeBuffer);
+ writeBuffer.flip();
+ this.pendingBuffer = pendingBuffer;
+ this.pendingWriteSize = pendingWriteSize;
+ this.ccb = ccb;
+ }
+
+ boolean performPendingWrite(SocketChannel sc) throws IOException {
+ int len = writeBuffer.remaining();
+ if (len > 0) {
+ int written = sc.write(writeBuffer);
+ muxDemux.getPerformanceCounters().addSignalingBytesWritten(written);
+ if (written < len) {
+ return false;
+ }
+ }
+ if (pendingBuffer != null) {
+ if (pendingWriteSize > 0) {
+ assert pendingWriteSize <= pendingBuffer.remaining();
+ int oldLimit = pendingBuffer.limit();
+ try {
+ pendingBuffer.limit(pendingWriteSize + pendingBuffer.position());
+ int written = sc.write(pendingBuffer);
+ muxDemux.getPerformanceCounters().addPayloadBytesWritten(written);
+ pendingWriteSize -= written;
+ } finally {
+ pendingBuffer.limit(oldLimit);
+ }
+ }
+ if (pendingWriteSize > 0) {
+ return false;
+ }
+ pendingBuffer = null;
+ pendingWriteSize = 0;
+ }
+ if (ccb != null) {
+ ccb.writeComplete();
+ ccb = null;
+ }
+ return true;
+ }
+ }
+
+ void driveWriterStateMachine() throws IOException, NetException {
+ SocketChannel sc = tcpConnection.getSocketChannel();
+ if (writerState.writePending()) {
+ if (!writerState.performPendingWrite(sc)) {
+ return;
+ }
+ pendingWriteEventsCounter.decrement();
+ }
+ int numCycles;
+
+ synchronized (MultiplexedConnection.this) {
+ numCycles = cSet.getOpenChannelCount();
+ }
+
+ for (int i = 0; i < numCycles; ++i) {
+ ChannelControlBlock writeCCB = null;
+ synchronized (MultiplexedConnection.this) {
+ BitSet pendingChannelSynBitmap = cSet.getPendingChannelSynBitmap();
+ for (int j = pendingChannelSynBitmap.nextSetBit(0); j >= 0; j = pendingChannelSynBitmap.nextSetBit(j)) {
+ pendingChannelSynBitmap.clear(j);
+ writerState.command.setChannelId(j);
+ writerState.command.setCommandType(MuxDemuxCommand.CommandType.OPEN_CHANNEL);
+ writerState.command.setData(0);
+ writerState.reset(null, 0, null);
+ if (!writerState.performPendingWrite(sc)) {
+ return;
+ }
+ pendingWriteEventsCounter.decrement();
+ }
+ BitSet pendingChannelCreditsBitmap = cSet.getPendingChannelCreditsBitmap();
+ for (int j = pendingChannelCreditsBitmap.nextSetBit(0); j >= 0; j = pendingChannelCreditsBitmap
+ .nextSetBit(j)) {
+ writerState.command.setChannelId(j);
+ writerState.command.setCommandType(MuxDemuxCommand.CommandType.ADD_CREDITS);
+ ChannelControlBlock ccb = cSet.getCCB(j);
+ int credits = ccb.getReadCredits();
+ int effectiveCredits;
+ if (credits <= MuxDemuxCommand.MAX_DATA_VALUE) {
+ effectiveCredits = credits;
+ ccb.setReadCredits(0);
+ pendingChannelCreditsBitmap.clear(j);
+ } else {
+ effectiveCredits = MuxDemuxCommand.MAX_DATA_VALUE;
+ ccb.setReadCredits(credits - effectiveCredits);
+ }
+ writerState.command.setData(effectiveCredits);
+ writerState.reset(null, 0, null);
+ if (!writerState.performPendingWrite(sc)) {
+ return;
+ }
+ if (credits == effectiveCredits) {
+ pendingWriteEventsCounter.decrement();
+ }
+ }
+ BitSet pendingEOSAckBitmap = cSet.getPendingEOSAckBitmap();
+ for (int j = pendingEOSAckBitmap.nextSetBit(0); j >= 0; j = pendingEOSAckBitmap.nextSetBit(j)) {
+ pendingEOSAckBitmap.clear(j);
+ writerState.command.setChannelId(j);
+ writerState.command.setCommandType(MuxDemuxCommand.CommandType.CLOSE_CHANNEL_ACK);
+ writerState.command.setData(0);
+ writerState.reset(null, 0, null);
+ if (!writerState.performPendingWrite(sc)) {
+ return;
+ }
+ pendingWriteEventsCounter.decrement();
+ }
+ BitSet pendingChannelWriteBitmap = cSet.getPendingChannelWriteBitmap();
+ lastChannelWritten = pendingChannelWriteBitmap.nextSetBit(lastChannelWritten + 1);
+ if (lastChannelWritten < 0) {
+ lastChannelWritten = pendingChannelWriteBitmap.nextSetBit(0);
+ if (lastChannelWritten < 0) {
+ return;
+ }
+ }
+ writeCCB = cSet.getCCB(lastChannelWritten);
+ }
+ writeCCB.write(writerState);
+ if (writerState.writePending()) {
+ pendingWriteEventsCounter.increment();
+ if (!writerState.performPendingWrite(sc)) {
+ return;
+ }
+ pendingWriteEventsCounter.decrement();
+ }
+ }
+ }
+
+ class ReaderState {
+ private final ByteBuffer readBuffer;
+
+ final MuxDemuxCommand command;
+
+ private int pendingReadSize;
+
+ private ChannelControlBlock ccb;
+
+ ReaderState() {
+ readBuffer = ByteBuffer.allocateDirect(MuxDemuxCommand.COMMAND_SIZE);
+ command = new MuxDemuxCommand();
+ }
+
+ void reset() {
+ readBuffer.clear();
+ pendingReadSize = 0;
+ ccb = null;
+ }
+
+ private ChannelControlBlock getCCBInCommand() {
+ synchronized (MultiplexedConnection.this) {
+ return cSet.getCCB(command.getChannelId());
+ }
+ }
+ }
+
+ void driveReaderStateMachine() throws IOException, NetException {
+ SocketChannel sc = tcpConnection.getSocketChannel();
+ int chunksRead = 0;
+ while (chunksRead < MAX_CHUNKS_READ_PER_CYCLE) {
+ if (readerState.readBuffer.remaining() > 0) {
+ int read = sc.read(readerState.readBuffer);
+ if (read < 0) {
+ throw new NetException("Socket Closed");
+ }
+ muxDemux.getPerformanceCounters().addSignalingBytesRead(read);
+ if (readerState.readBuffer.remaining() > 0) {
+ return;
+ }
+ readerState.readBuffer.flip();
+ readerState.command.read(readerState.readBuffer);
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Received command: " + readerState.command);
+ }
+ ChannelControlBlock ccb = null;
+ switch (readerState.command.getCommandType()) {
+ case ADD_CREDITS: {
+ ccb = readerState.getCCBInCommand();
+ ccb.addWriteCredits(readerState.command.getData());
+ break;
+ }
+ case CLOSE_CHANNEL: {
+ ccb = readerState.getCCBInCommand();
+ ccb.reportRemoteEOS();
+ int channelId = ccb.getChannelId();
+ cSet.markEOSAck(channelId);
+ cSet.unmarkPendingCredits(channelId);
+ break;
+ }
+ case CLOSE_CHANNEL_ACK: {
+ ccb = readerState.getCCBInCommand();
+ ccb.reportLocalEOSAck();
+ break;
+ }
+ case DATA: {
+ ccb = readerState.getCCBInCommand();
+ readerState.pendingReadSize = readerState.command.getData();
+ readerState.ccb = ccb;
+ break;
+ }
+ case ERROR: {
+ ccb = readerState.getCCBInCommand();
+ ccb.reportRemoteError(readerState.command.getData());
+ int channelId = ccb.getChannelId();
+ cSet.markEOSAck(channelId);
+ cSet.unmarkPendingCredits(channelId);
+ break;
+ }
+ case OPEN_CHANNEL: {
+ int channelId = readerState.command.getChannelId();
+ ccb = cSet.registerChannel(channelId);
+ muxDemux.getChannelOpenListener().channelOpened(ccb);
+ }
+ }
+ if (LOGGER.isLoggable(Level.FINE)) {
+ LOGGER.fine("Applied command: " + readerState.command + " on " + ccb);
+ }
+ }
+ if (readerState.pendingReadSize > 0) {
+ ++chunksRead;
+ int newPendingReadSize = readerState.ccb.read(sc, readerState.pendingReadSize);
+ muxDemux.getPerformanceCounters().addPayloadBytesRead(readerState.pendingReadSize - newPendingReadSize);
+ readerState.pendingReadSize = newPendingReadSize;
+ if (readerState.pendingReadSize > 0) {
+ return;
+ }
+ }
+ readerState.reset();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java
new file mode 100644
index 0000000..8548bb8
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemux.java
@@ -0,0 +1,112 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.HashMap;
+import java.util.Map;
+
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+import edu.uci.ics.hyracks.net.protocols.tcp.ITCPConnectionListener;
+import edu.uci.ics.hyracks.net.protocols.tcp.TCPConnection;
+import edu.uci.ics.hyracks.net.protocols.tcp.TCPEndpoint;
+
+public class MuxDemux {
+ private final InetSocketAddress localAddress;
+
+ private final IChannelOpenListener channelOpenListener;
+
+ private final Map<InetSocketAddress, MultiplexedConnection> connectionMap;
+
+ private final TCPEndpoint tcpEndpoint;
+
+ private final MuxDemuxPerformanceCounters perfCounters;
+
+ public MuxDemux(InetSocketAddress localAddress, IChannelOpenListener listener, int nThreads) {
+ this.localAddress = localAddress;
+ this.channelOpenListener = listener;
+ connectionMap = new HashMap<InetSocketAddress, MultiplexedConnection>();
+ this.tcpEndpoint = new TCPEndpoint(new ITCPConnectionListener() {
+ @Override
+ public void connectionEstablished(TCPConnection connection) {
+ MultiplexedConnection mConn;
+ synchronized (MuxDemux.this) {
+ mConn = connectionMap.get(connection.getRemoteAddress());
+ }
+ assert mConn != null;
+ mConn.setTCPConnection(connection);
+ connection.setEventListener(mConn);
+ connection.setAttachment(mConn);
+ }
+
+ @Override
+ public void acceptedConnection(TCPConnection connection) {
+ MultiplexedConnection mConn = new MultiplexedConnection(MuxDemux.this);
+ mConn.setTCPConnection(connection);
+ connection.setEventListener(mConn);
+ connection.setAttachment(mConn);
+ }
+
+ @Override
+ public void connectionFailure(InetSocketAddress remoteAddress) {
+ MultiplexedConnection mConn;
+ synchronized (MuxDemux.this) {
+ mConn = connectionMap.get(remoteAddress);
+ assert mConn != null;
+ int nConnectionAttempts = mConn.getConnectionAttempts();
+ if (nConnectionAttempts > 5) {
+ connectionMap.remove(remoteAddress);
+ mConn.setConnectionFailure();
+ } else {
+ mConn.setConnectionAttempts(nConnectionAttempts + 1);
+ tcpEndpoint.initiateConnection(remoteAddress);
+ }
+ }
+ }
+ }, nThreads);
+ perfCounters = new MuxDemuxPerformanceCounters();
+ }
+
+ public void start() throws IOException {
+ tcpEndpoint.start(localAddress);
+ }
+
+ public MultiplexedConnection connect(InetSocketAddress remoteAddress) throws InterruptedException, NetException {
+ MultiplexedConnection mConn = null;
+ synchronized (this) {
+ mConn = connectionMap.get(remoteAddress);
+ if (mConn == null) {
+ mConn = new MultiplexedConnection(this);
+ connectionMap.put(remoteAddress, mConn);
+ tcpEndpoint.initiateConnection(remoteAddress);
+ }
+ }
+ mConn.waitUntilConnected();
+ return mConn;
+ }
+
+ IChannelOpenListener getChannelOpenListener() {
+ return channelOpenListener;
+ }
+
+ public InetSocketAddress getLocalAddress() {
+ return tcpEndpoint.getLocalAddress();
+ }
+
+ public MuxDemuxPerformanceCounters getPerformanceCounters() {
+ return perfCounters;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java
new file mode 100644
index 0000000..2e2636b
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxCommand.java
@@ -0,0 +1,75 @@
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+
+class MuxDemuxCommand {
+ static final int MAX_CHANNEL_ID = 0x3ff;
+
+ static final int COMMAND_SIZE = 4;
+
+ static final int MAX_DATA_VALUE = 0x7ffff;
+
+ enum CommandType {
+ OPEN_CHANNEL,
+ CLOSE_CHANNEL,
+ CLOSE_CHANNEL_ACK,
+ ERROR,
+ ADD_CREDITS,
+ DATA,
+ }
+
+ private int channelId;
+
+ private CommandType type;
+
+ private int data;
+
+ public int getChannelId() {
+ return channelId;
+ }
+
+ public void setChannelId(int channelId) throws NetException {
+ if (channelId > MAX_CHANNEL_ID) {
+ throw new NetException("channelId " + channelId + " exceeds " + MAX_CHANNEL_ID);
+ }
+ this.channelId = channelId;
+ }
+
+ public CommandType getCommandType() {
+ return type;
+ }
+
+ public void setCommandType(CommandType type) {
+ this.type = type;
+ }
+
+ public int getData() {
+ return data;
+ }
+
+ public void setData(int data) throws NetException {
+ if (data > MAX_DATA_VALUE) {
+ throw new NetException("data " + data + " exceeds " + MAX_DATA_VALUE);
+ }
+ this.data = data;
+ }
+
+ public void write(ByteBuffer buffer) {
+ int cmd = (channelId << 22) | (type.ordinal() << 19) | (data & 0x7ffff);
+ buffer.putInt(cmd);
+ }
+
+ public void read(ByteBuffer buffer) {
+ int cmd = buffer.getInt();
+ channelId = (cmd >> 22) & 0x3ff;
+ type = CommandType.values()[(cmd >> 19) & 0x7];
+ data = cmd & 0x7ffff;
+ }
+
+ @Override
+ public String toString() {
+ return channelId + ":" + type + ":" + data;
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxPerformanceCounters.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxPerformanceCounters.java
new file mode 100644
index 0000000..6bf21aa
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/muxdemux/MuxDemuxPerformanceCounters.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.muxdemux;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+public class MuxDemuxPerformanceCounters {
+ private final AtomicLong payloadBytesRead;
+
+ private final AtomicLong payloadBytesWritten;
+
+ private final AtomicLong signalingBytesRead;
+
+ private final AtomicLong signalingBytesWritten;
+
+ public MuxDemuxPerformanceCounters() {
+ payloadBytesRead = new AtomicLong();
+ payloadBytesWritten = new AtomicLong();
+ signalingBytesRead = new AtomicLong();
+ signalingBytesWritten = new AtomicLong();
+ }
+
+ public void addPayloadBytesRead(long delta) {
+ payloadBytesRead.addAndGet(delta);
+ }
+
+ public long getPayloadBytesRead() {
+ return payloadBytesRead.get();
+ }
+
+ public void addPayloadBytesWritten(long delta) {
+ payloadBytesWritten.addAndGet(delta);
+ }
+
+ public long getPayloadBytesWritten() {
+ return payloadBytesWritten.get();
+ }
+
+ public void addSignalingBytesRead(long delta) {
+ signalingBytesRead.addAndGet(delta);
+ }
+
+ public long getSignalingBytesRead() {
+ return signalingBytesRead.get();
+ }
+
+ public void addSignalingBytesWritten(long delta) {
+ signalingBytesWritten.addAndGet(delta);
+ }
+
+ public long getSignalingBytesWritten() {
+ return signalingBytesWritten.get();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java
new file mode 100644
index 0000000..607bf31
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionEventListener.java
@@ -0,0 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.tcp;
+
+import java.io.IOException;
+
+import edu.uci.ics.hyracks.net.exceptions.NetException;
+
+public interface ITCPConnectionEventListener {
+ public void notifyIOReady(TCPConnection connection, boolean readable, boolean writable) throws IOException,
+ NetException;
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionListener.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionListener.java
new file mode 100644
index 0000000..ead2a1f
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/ITCPConnectionListener.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.tcp;
+
+import java.net.InetSocketAddress;
+
+public interface ITCPConnectionListener {
+ public void acceptedConnection(TCPConnection connection);
+
+ public void connectionEstablished(TCPConnection connection);
+
+ public void connectionFailure(InetSocketAddress remoteAddress);
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPConnection.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPConnection.java
new file mode 100644
index 0000000..210508b
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPConnection.java
@@ -0,0 +1,93 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.tcp;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.SocketChannel;
+
+public class TCPConnection {
+ private final TCPEndpoint endpoint;
+
+ private final SocketChannel channel;
+
+ private final SelectionKey key;
+
+ private final Selector selector;
+
+ private ITCPConnectionEventListener eventListener;
+
+ private Object attachment;
+
+ public TCPConnection(TCPEndpoint endpoint, SocketChannel channel, SelectionKey key, Selector selector) {
+ this.endpoint = endpoint;
+ this.channel = channel;
+ this.key = key;
+ this.selector = selector;
+ }
+
+ public TCPEndpoint getEndpoint() {
+ return endpoint;
+ }
+
+ public SocketChannel getSocketChannel() {
+ return channel;
+ }
+
+ public InetSocketAddress getLocalAddress() {
+ return (InetSocketAddress) channel.socket().getLocalSocketAddress();
+ }
+
+ public InetSocketAddress getRemoteAddress() {
+ return (InetSocketAddress) channel.socket().getRemoteSocketAddress();
+ }
+
+ public void enable(int ops) {
+ key.interestOps(key.interestOps() | ops);
+ selector.wakeup();
+ }
+
+ public void disable(int ops) {
+ key.interestOps(key.interestOps() & ~(ops));
+ selector.wakeup();
+ }
+
+ public ITCPConnectionEventListener getEventListener() {
+ return eventListener;
+ }
+
+ public void setEventListener(ITCPConnectionEventListener eventListener) {
+ this.eventListener = eventListener;
+ }
+
+ public Object getAttachment() {
+ return attachment;
+ }
+
+ public void setAttachment(Object attachment) {
+ this.attachment = attachment;
+ }
+
+ public void close() {
+ key.cancel();
+ try {
+ channel.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java
new file mode 100644
index 0000000..1a73bdc
--- /dev/null
+++ b/hyracks-net/src/main/java/edu/uci/ics/hyracks/net/protocols/tcp/TCPEndpoint.java
@@ -0,0 +1,217 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.protocols.tcp;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.ServerSocket;
+import java.nio.channels.SelectableChannel;
+import java.nio.channels.SelectionKey;
+import java.nio.channels.Selector;
+import java.nio.channels.ServerSocketChannel;
+import java.nio.channels.SocketChannel;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+public class TCPEndpoint {
+ private final ITCPConnectionListener connectionListener;
+
+ private final int nThreads;
+
+ private ServerSocketChannel serverSocketChannel;
+
+ private InetSocketAddress localAddress;
+
+ private IOThread[] ioThreads;
+
+ private int nextThread;
+
+ public TCPEndpoint(ITCPConnectionListener connectionListener, int nThreads) {
+ this.connectionListener = connectionListener;
+ this.nThreads = nThreads;
+ }
+
+ public void start(InetSocketAddress localAddress) throws IOException {
+ serverSocketChannel = ServerSocketChannel.open();
+ ServerSocket serverSocket = serverSocketChannel.socket();
+ serverSocket.bind(localAddress);
+ this.localAddress = (InetSocketAddress) serverSocket.getLocalSocketAddress();
+ ioThreads = new IOThread[nThreads];
+ for (int i = 0; i < ioThreads.length; ++i) {
+ ioThreads[i] = new IOThread();
+ }
+ ioThreads[0].registerServerSocket(serverSocketChannel);
+ for (int i = 0; i < ioThreads.length; ++i) {
+ ioThreads[i].start();
+ }
+ }
+
+ private synchronized int getNextThread() {
+ int result = nextThread;
+ nextThread = (nextThread + 1) % nThreads;
+ return result;
+ }
+
+ public void initiateConnection(InetSocketAddress remoteAddress) {
+ int targetThread = getNextThread();
+ ioThreads[targetThread].initiateConnection(remoteAddress);
+ }
+
+ private void distributeIncomingConnection(SocketChannel channel) {
+ int targetThread = getNextThread();
+ ioThreads[targetThread].addIncomingConnection(channel);
+ }
+
+ public InetSocketAddress getLocalAddress() {
+ return localAddress;
+ }
+
+ private class IOThread extends Thread {
+ private final List<InetSocketAddress> pendingConnections;
+
+ private final List<InetSocketAddress> workingPendingConnections;
+
+ private final List<SocketChannel> incomingConnections;
+
+ private final List<SocketChannel> workingIncomingConnections;
+
+ private Selector selector;
+
+ public IOThread() throws IOException {
+ super("TCPEndpoint IO Thread");
+ setPriority(MAX_PRIORITY);
+ this.pendingConnections = new ArrayList<InetSocketAddress>();
+ this.workingPendingConnections = new ArrayList<InetSocketAddress>();
+ this.incomingConnections = new ArrayList<SocketChannel>();
+ this.workingIncomingConnections = new ArrayList<SocketChannel>();
+ selector = Selector.open();
+ }
+
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ int n = selector.select();
+ collectOutstandingWork();
+ if (!workingPendingConnections.isEmpty()) {
+ for (InetSocketAddress address : workingPendingConnections) {
+ SocketChannel channel = SocketChannel.open();
+ channel.configureBlocking(false);
+ boolean connect = false;
+ boolean failure = false;
+ try {
+ connect = channel.connect(address);
+ } catch (IOException e) {
+ failure = true;
+ connectionListener.connectionFailure(address);
+ }
+ if (!failure) {
+ if (!connect) {
+ SelectionKey key = channel.register(selector, SelectionKey.OP_CONNECT);
+ key.attach(address);
+ } else {
+ SelectionKey key = channel.register(selector, 0);
+ createConnection(key, channel);
+ }
+ }
+ }
+ workingPendingConnections.clear();
+ }
+ if (!workingIncomingConnections.isEmpty()) {
+ for (SocketChannel channel : workingIncomingConnections) {
+ channel.configureBlocking(false);
+ SelectionKey sKey = channel.register(selector, 0);
+ TCPConnection connection = new TCPConnection(TCPEndpoint.this, channel, sKey, selector);
+ sKey.attach(connection);
+ synchronized (connectionListener) {
+ connectionListener.acceptedConnection(connection);
+ }
+ }
+ workingIncomingConnections.clear();
+ }
+ if (n > 0) {
+ Iterator<SelectionKey> i = selector.selectedKeys().iterator();
+ while (i.hasNext()) {
+ SelectionKey key = i.next();
+ i.remove();
+ SelectableChannel sc = key.channel();
+ boolean readable = key.isReadable();
+ boolean writable = key.isWritable();
+
+ if (readable || writable) {
+ TCPConnection connection = (TCPConnection) key.attachment();
+ connection.getEventListener().notifyIOReady(connection, readable, writable);
+ }
+ if (key.isAcceptable()) {
+ assert sc == serverSocketChannel;
+ SocketChannel channel = serverSocketChannel.accept();
+ distributeIncomingConnection(channel);
+ } else if (key.isConnectable()) {
+ SocketChannel channel = (SocketChannel) sc;
+ boolean finishConnect = false;
+ try {
+ finishConnect = channel.finishConnect();
+ } catch (Exception e) {
+ e.printStackTrace();
+ key.cancel();
+ connectionListener.connectionFailure((InetSocketAddress) key.attachment());
+ }
+ if (finishConnect) {
+ createConnection(key, channel);
+ }
+ }
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ private void createConnection(SelectionKey key, SocketChannel channel) {
+ TCPConnection connection = new TCPConnection(TCPEndpoint.this, channel, key, selector);
+ key.attach(connection);
+ key.interestOps(0);
+ connectionListener.connectionEstablished(connection);
+ }
+
+ synchronized void initiateConnection(InetSocketAddress remoteAddress) {
+ pendingConnections.add(remoteAddress);
+ selector.wakeup();
+ }
+
+ synchronized void addIncomingConnection(SocketChannel channel) {
+ incomingConnections.add(channel);
+ selector.wakeup();
+ }
+
+ void registerServerSocket(ServerSocketChannel serverSocketChannel) throws IOException {
+ serverSocketChannel.configureBlocking(false);
+ serverSocketChannel.register(selector, SelectionKey.OP_ACCEPT);
+ }
+
+ private synchronized void collectOutstandingWork() {
+ if (!pendingConnections.isEmpty()) {
+ workingPendingConnections.addAll(pendingConnections);
+ pendingConnections.clear();
+ }
+ if (!incomingConnections.isEmpty()) {
+ workingIncomingConnections.addAll(incomingConnections);
+ incomingConnections.clear();
+ }
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java b/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java
new file mode 100644
index 0000000..31bc2df
--- /dev/null
+++ b/hyracks-net/src/test/java/edu/uci/ics/hyracks/net/tests/NetTest.java
@@ -0,0 +1,213 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.net.tests;
+
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.LinkedList;
+import java.util.Queue;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.net.buffers.IBufferAcceptor;
+import edu.uci.ics.hyracks.net.buffers.ICloseableBufferAcceptor;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.ChannelControlBlock;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.IChannelOpenListener;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MultiplexedConnection;
+import edu.uci.ics.hyracks.net.protocols.muxdemux.MuxDemux;
+
+public class NetTest {
+ @Test
+ public void test() throws Exception {
+ AtomicBoolean failFlag = new AtomicBoolean();
+
+ MuxDemux md1 = createMuxDemux("md1", failFlag);
+ md1.start();
+ MuxDemux md2 = createMuxDemux("md2", failFlag);
+ md2.start();
+ InetSocketAddress md2Address = md2.getLocalAddress();
+
+ MultiplexedConnection md1md2 = md1.connect(md2Address);
+
+ Thread t1 = createThread(md1md2, 1);
+ Thread t2 = createThread(md1md2, -1);
+ t1.start();
+ t2.start();
+
+ t1.join();
+ t2.join();
+
+ Assert.assertFalse("Failure flag was set to true", failFlag.get());
+ }
+
+ private Thread createThread(final MultiplexedConnection md1md2, final int factor) {
+ return new Thread() {
+ @Override
+ public void run() {
+ try {
+ ChannelControlBlock md1md2c1 = md1md2.openChannel();
+
+ final Semaphore sem = new Semaphore(1);
+ sem.acquire();
+ md1md2c1.getWriteInterface().setEmptyBufferAcceptor(new IBufferAcceptor() {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ }
+ });
+
+ md1md2c1.getReadInterface().setFullBufferAcceptor(new ICloseableBufferAcceptor() {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ }
+
+ @Override
+ public void error(int ecode) {
+ }
+
+ @Override
+ public void close() {
+ sem.release();
+ }
+ });
+
+ ICloseableBufferAcceptor fba = md1md2c1.getWriteInterface().getFullBufferAcceptor();
+ for (int i = 0; i < 10000; ++i) {
+ ByteBuffer buffer = ByteBuffer.allocate(1024);
+ for (int j = 0; j < 256; ++j) {
+ buffer.putInt(factor * (i + j));
+ }
+ buffer.flip();
+ fba.accept(buffer);
+ }
+ fba.close();
+ sem.acquire();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ };
+
+ }
+
+ private MuxDemux createMuxDemux(final String label, final AtomicBoolean failFlag) {
+ IChannelOpenListener md1OpenListener = new IChannelOpenListener() {
+ @Override
+ public void channelOpened(final ChannelControlBlock channel) {
+ final ChannelIO cio = new ChannelIO(label, channel);
+ channel.getReadInterface().setFullBufferAcceptor(cio.rifba);
+ channel.getWriteInterface().setEmptyBufferAcceptor(cio.wieba);
+
+ final IBufferAcceptor rieba = channel.getReadInterface().getEmptyBufferAcceptor();
+ for (int i = 0; i < 50; ++i) {
+ rieba.accept(ByteBuffer.allocate(1024));
+ }
+ new Thread() {
+ private int prevTotal = 0;
+
+ @Override
+ public void run() {
+ while (true) {
+ ByteBuffer fbuf = null;
+ synchronized (channel) {
+ while (!cio.eos && cio.ecode == 0 && cio.rifq.isEmpty()) {
+ try {
+ channel.wait();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ if (!cio.rifq.isEmpty()) {
+ fbuf = cio.rifq.poll();
+ } else if (cio.ecode != 0) {
+ throw new RuntimeException("Error: " + cio.ecode);
+ } else if (cio.eos) {
+ channel.getWriteInterface().getFullBufferAcceptor().close();
+ return;
+ }
+ }
+ int counter = 0;
+ while (fbuf.remaining() > 0) {
+ counter += fbuf.getInt();
+ }
+ if (prevTotal != 0) {
+ if (Math.abs(counter - prevTotal) != 256) {
+ failFlag.set(true);
+ }
+ }
+ prevTotal = counter;
+ fbuf.compact();
+ rieba.accept(fbuf);
+ }
+ }
+ }.start();
+ }
+ };
+ return new MuxDemux(new InetSocketAddress("127.0.0.1", 0), md1OpenListener, 1);
+ }
+
+ private class ChannelIO {
+ private ChannelControlBlock channel;
+
+ private Queue<ByteBuffer> rifq;
+
+ private Queue<ByteBuffer> wieq;
+
+ private boolean eos;
+
+ private int ecode;
+
+ private ICloseableBufferAcceptor rifba;
+
+ private IBufferAcceptor wieba;
+
+ public ChannelIO(final String label, ChannelControlBlock channel) {
+ this.channel = channel;
+ this.rifq = new LinkedList<ByteBuffer>();
+ this.wieq = new LinkedList<ByteBuffer>();
+
+ rifba = new ICloseableBufferAcceptor() {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ rifq.add(buffer);
+ ChannelIO.this.channel.notifyAll();
+ }
+
+ @Override
+ public void error(int ecode) {
+ ChannelIO.this.ecode = ecode;
+ ChannelIO.this.channel.notifyAll();
+ }
+
+ @Override
+ public void close() {
+ eos = true;
+ ChannelIO.this.channel.notifyAll();
+ }
+ };
+
+ wieba = new IBufferAcceptor() {
+ @Override
+ public void accept(ByteBuffer buffer) {
+ wieq.add(buffer);
+ ChannelIO.this.channel.notifyAll();
+ }
+ };
+ }
+ }
+}
\ No newline at end of file
diff --git a/hyracks-server/.classpath b/hyracks-server/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-server/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-server/.project b/hyracks-server/.project
deleted file mode 100644
index 8b4eef9..0000000
--- a/hyracks-server/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-server</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-server/.settings/org.eclipse.jdt.core.prefs b/hyracks-server/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 611528f..0000000
--- a/hyracks-server/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Tue Sep 20 17:16:43 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-server/.settings/org.maven.ide.eclipse.prefs b/hyracks-server/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index e6f9a9e..0000000
--- a/hyracks-server/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Sun Aug 29 19:38:10 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-server/pom.xml b/hyracks-server/pom.xml
index 8732ca4..8a81c7b 100644
--- a/hyracks-server/pom.xml
+++ b/hyracks-server/pom.xml
@@ -1,13 +1,10 @@
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
- <groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-server</artifactId>
- <version>0.1.8-SNAPSHOT</version>
-
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -74,23 +71,16 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-cc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>edu.uci.ics.hyracks</groupId>
- <artifactId>hyracks-admin-console</artifactId>
- <version>0.1.8-SNAPSHOT</version>
- <type>war</type>
- <scope>compile</scope>
- </dependency>
</dependencies>
</project>
diff --git a/hyracks-server/src/main/assembly/binary-assembly.xml b/hyracks-server/src/main/assembly/binary-assembly.xml
index 72af28b..cd598d9 100644
--- a/hyracks-server/src/main/assembly/binary-assembly.xml
+++ b/hyracks-server/src/main/assembly/binary-assembly.xml
@@ -5,16 +5,6 @@
<format>dir</format>
</formats>
<includeBaseDirectory>false</includeBaseDirectory>
- <dependencySets>
- <dependencySet>
- <outputDirectory>console</outputDirectory>
- <fileMode>0644</fileMode>
- <includes>
- <include>edu.uci.ics.hyracks:hyracks-admin-console</include>
- </includes>
- <outputFileNameMapping>${artifact.artifactId}.${artifact.extension}</outputFileNameMapping>
- </dependencySet>
- </dependencySets>
<fileSets>
<fileSet>
<directory>target/appassembler/bin</directory>
diff --git a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
index 92a1ad5..8833fed 100644
--- a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
+++ b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/drivers/VirtualClusterDriver.java
@@ -27,8 +27,14 @@
@Option(name = "-n", required = false, usage = "Number of node controllers (default: 2)")
public int n = 2;
- @Option(name = "-cc-port", required = false, usage = "CC Port (default: 1099)")
- public int ccPort = 1099;
+ @Option(name = "-cc-client-net-port", required = false, usage = "CC Port (default: 1098)")
+ public int ccClientNetPort = 1098;
+
+ @Option(name = "-cc-cluster-net-port", required = false, usage = "CC Port (default: 1099)")
+ public int ccClusterNetPort = 1099;
+
+ @Option(name = "-cc-http-port", required = false, usage = "CC Port (default: 16001)")
+ public int ccHttpPort = 16001;
}
public static void main(String[] args) throws Exception {
@@ -43,16 +49,22 @@
}
CCConfig ccConfig = new CCConfig();
- ccConfig.port = options.ccPort;
+ ccConfig.clusterNetIpAddress = "127.0.0.1";
+ ccConfig.clusterNetPort = options.ccClusterNetPort;
+ ccConfig.clientNetIpAddress = "127.0.0.1";
+ ccConfig.clientNetPort = options.ccClientNetPort;
+ ccConfig.httpPort = options.ccHttpPort;
HyracksCCProcess ccp = new HyracksCCProcess(ccConfig);
ccp.start();
- Thread.sleep(2000);
+ Thread.sleep(5000);
HyracksNCProcess ncps[] = new HyracksNCProcess[options.n];
for (int i = 0; i < options.n; ++i) {
NCConfig ncConfig = new NCConfig();
- ncConfig.ccHost = "localhost";
+ ncConfig.ccHost = "127.0.0.1";
+ ncConfig.ccPort = options.ccClusterNetPort;
+ ncConfig.clusterNetIPAddress = "127.0.0.1";
ncConfig.nodeId = "nc" + i;
ncConfig.dataIPAddress = "127.0.0.1";
ncps[i] = new HyracksNCProcess(ncConfig);
diff --git a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java
index 17169ac..6a0e45f 100644
--- a/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java
+++ b/hyracks-server/src/main/java/edu/uci/ics/hyracks/server/process/HyracksServerProcess.java
@@ -62,6 +62,8 @@
private String[] buildCommand() {
List<String> cList = new ArrayList<String>();
cList.add(getJavaCommand());
+ cList.add("-Dbasedir=" + System.getProperty("basedir"));
+ cList.add("-Djava.rmi.server.hostname=127.0.0.1");
cList.add("-classpath");
cList.add(getClasspath());
cList.add(getMainClassName());
diff --git a/hyracks-storage-am-btree/.classpath b/hyracks-storage-am-btree/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-storage-am-btree/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-storage-am-btree/.project b/hyracks-storage-am-btree/.project
deleted file mode 100644
index 754745f..0000000
--- a/hyracks-storage-am-btree/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-storage-am-btree</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-storage-am-btree/.settings/org.eclipse.jdt.core.prefs b/hyracks-storage-am-btree/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 451c926..0000000
--- a/hyracks-storage-am-btree/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,264 +0,0 @@
-#Fri May 20 19:34:05 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=48
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=48
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=true
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true
-org.eclipse.jdt.core.formatter.comment.format_line_comments=true
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=true
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_member=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=120
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
diff --git a/hyracks-storage-am-btree/.settings/org.maven.ide.eclipse.prefs b/hyracks-storage-am-btree/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index ecf0da7..0000000
--- a/hyracks-storage-am-btree/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Sun Aug 29 21:59:34 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-storage-am-btree/pom.xml b/hyracks-storage-am-btree/pom.xml
index efffe5d..8ad45ac 100644
--- a/hyracks-storage-am-btree/pom.xml
+++ b/hyracks-storage-am-btree/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,35 +27,35 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java
new file mode 100644
index 0000000..a24c4d7
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeFrame.java
@@ -0,0 +1,16 @@
+package edu.uci.ics.hyracks.storage.am.btree.api;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public interface IBTreeFrame extends ITreeIndexFrame {
+ public int findUpdateTupleIndex(ITupleReference tuple) throws TreeIndexException;
+ public int findInsertTupleIndex(ITupleReference tuple) throws TreeIndexException;
+ public int findDeleteTupleIndex(ITupleReference tuple) throws TreeIndexException;
+ public void insertSorted(ITupleReference tuple);
+ public boolean getSmFlag();
+ public void setSmFlag(boolean smFlag);
+ public void setMultiComparator(MultiComparator cmp);
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeInteriorFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeInteriorFrame.java
index 25ab167..23fdcf5 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeInteriorFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeInteriorFrame.java
@@ -15,22 +15,16 @@
package edu.uci.ics.hyracks.storage.am.btree.api;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-public interface IBTreeInteriorFrame extends ITreeIndexFrame {
- public void insertSorted(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException;
-
- public int getChildPageId(RangePredicate pred, MultiComparator srcCmp);
+public interface IBTreeInteriorFrame extends IBTreeFrame {
+ public int getChildPageId(RangePredicate pred);
- public int getLeftmostChildPageId(MultiComparator cmp);
+ public int getLeftmostChildPageId();
- public int getRightmostChildPageId(MultiComparator cmp);
+ public int getRightmostChildPageId();
public void setRightmostChildPageId(int pageId);
- public void deleteGreatest(MultiComparator cmp);
+ public void deleteGreatest();
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
index 53d892e..a079527 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
@@ -17,25 +17,20 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-public interface IBTreeLeafFrame extends ITreeIndexFrame {
- public void insertSorted(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException;
-
+public interface IBTreeLeafFrame extends IBTreeFrame {
public void setNextLeaf(int nextPage);
public int getNextLeaf();
public void setPrevLeaf(int prevPage);
- public int getPrevLeaf();
-
- public ITreeIndexTupleReference createTupleReference();
+ public int getPrevLeaf();
public int findTupleIndex(ITupleReference searchKey, ITreeIndexTupleReference pageTuple, MultiComparator cmp,
- FindTupleMode ftm, FindTupleNoExactMatchPolicy ftp);
+ FindTupleMode ftm, FindTupleNoExactMatchPolicy ftp) throws HyracksDataException;
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IFrameCompressor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IFrameCompressor.java
deleted file mode 100644
index 8e1d0a2..0000000
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IFrameCompressor.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree.api;
-
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-
-public interface IFrameCompressor {
- public boolean compress(BTreeFieldPrefixNSMLeafFrame frame, MultiComparator cmp) throws Exception;
-}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IPrefixSlotManager.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IPrefixSlotManager.java
index d01db11..0636968 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IPrefixSlotManager.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IPrefixSlotManager.java
@@ -16,7 +16,7 @@
package edu.uci.ics.hyracks.storage.am.btree.api;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ISlotManager;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
@@ -42,9 +42,9 @@
// potentially all tuples slots would have to change their prefix slot pointers
// all prefixes are recomputed during a reorg or compaction
-public interface IPrefixSlotManager {
- public void setFrame(BTreeFieldPrefixNSMLeafFrame frame);
-
+public interface IPrefixSlotManager extends ISlotManager {
+ // TODO: Clean up interface after extending ISlotManager.
+
public int decodeFirstSlotField(int slot);
public int decodeSecondSlotField(int slot);
@@ -58,7 +58,7 @@
public int insertSlot(int slot, int tupleOff);
// returns prefix slot number, returns TUPLE_UNCOMPRESSED if none found
- public int findPrefix(ITupleReference tuple, ITreeIndexTupleReference framePrefixTuple, MultiComparator multiCmp);
+ public int findPrefix(ITupleReference tuple, ITreeIndexTupleReference framePrefixTuple);
public int getTupleSlotStartOff();
@@ -78,4 +78,6 @@
// functions for testing
public void setPrefixSlot(int tupleIndex, int slot);
+
+ public void setMultiComparator(MultiComparator cmp);
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
index 1292ff2..4486205 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
@@ -21,16 +21,17 @@
import java.util.Comparator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.storage.am.btree.api.IFrameCompressor;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.btree.api.IPrefixSlotManager;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.impls.FieldPrefixSlotManager;
import edu.uci.ics.hyracks.storage.am.btree.impls.FieldPrefixTupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameCompressor;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
-public class FieldPrefixCompressor implements IFrameCompressor {
+public class FieldPrefixCompressor implements ITreeIndexFrameCompressor {
// minimum ratio of uncompressed tuples to total tuple to consider
// re-compression
@@ -40,17 +41,18 @@
// them
private int occurrenceThreshold;
- private ITypeTrait[] typeTraits;
+ private ITypeTraits[] typeTraits;
- public FieldPrefixCompressor(ITypeTrait[] typeTraits, float ratioThreshold, int occurrenceThreshold) {
+ public FieldPrefixCompressor(ITypeTraits[] typeTraits, float ratioThreshold, int occurrenceThreshold) {
this.typeTraits = typeTraits;
this.ratioThreshold = ratioThreshold;
this.occurrenceThreshold = occurrenceThreshold;
}
@Override
- public boolean compress(BTreeFieldPrefixNSMLeafFrame frame, MultiComparator cmp) throws Exception {
- int tupleCount = frame.getTupleCount();
+ public boolean compress(ITreeIndexFrame indexFrame, MultiComparator cmp) throws Exception {
+ BTreeFieldPrefixNSMLeafFrame frame = (BTreeFieldPrefixNSMLeafFrame)indexFrame;
+ int tupleCount = frame.getTupleCount();
if (tupleCount <= 0) {
frame.setPrefixTupleCount(0);
frame.setFreeSpaceOff(frame.getOrigFreeSpaceOff());
@@ -58,13 +60,17 @@
return false;
}
+ if (cmp.getKeyFieldCount() == 1) {
+ return false;
+ }
+
int uncompressedTupleCount = frame.getUncompressedTupleCount();
float ratio = (float) uncompressedTupleCount / (float) tupleCount;
if (ratio < ratioThreshold)
return false;
IBinaryComparator[] cmps = cmp.getComparators();
- int fieldCount = cmp.getKeyFieldCount();
+ int fieldCount = typeTraits.length;
ByteBuffer buf = frame.getBuffer();
byte[] pageArray = buf.array();
@@ -102,7 +108,7 @@
// we use a greedy heuristic to solve this "knapsack"-like problem
// (every keyPartition has a space savings and a number of slots
- // required, but we the number of slots are constrained by
+ // required, but the number of slots are constrained by
// MAX_PREFIX_SLOTS)
// we sort the keyPartitions by maxBenefitMinusCost / prefixSlotsNeeded
// and later choose the top MAX_PREFIX_SLOTS
@@ -161,7 +167,6 @@
uncompressedTupleCount = 0;
TypeAwareTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
-
FieldPrefixTupleReference tupleToWrite = new FieldPrefixTupleReference(tupleWriter.createTupleReference());
tupleToWrite.setFieldCount(fieldCount);
@@ -173,14 +178,14 @@
// number of fields we decided to use for compression of
// this keyPartition
- int numFieldsToCompress = keyPartitions.get(kpIndex).maxPmiIndex + 1;
+ int fieldCountToCompress = keyPartitions.get(kpIndex).maxPmiIndex + 1;
int segmentStart = keyPartitions.get(kpIndex).firstTupleIndex;
int tuplesInSegment = 1;
// System.out.println("PROCESSING KEYPARTITION: " + kpIndex
// + " RANGE: " + keyPartitions.get(kpIndex).firstRecSlotNum
// + " " + keyPartitions.get(kpIndex).lastRecSlotNum +
- // " FIELDSTOCOMPRESS: " + numFieldsToCompress);
+ // " FIELDSTOCOMPRESS: " + fieldCountToCompress);
FieldPrefixTupleReference prevTuple = new FieldPrefixTupleReference(tupleWriter
.createTupleReference());
@@ -193,10 +198,10 @@
prevTuple.resetByTupleIndex(frame, i - 1);
tuple.resetByTupleIndex(frame, i);
- // check if tuples match in numFieldsToCompress of their
+ // check if tuples match in fieldCountToCompress of their
// first fields
int prefixFieldsMatch = 0;
- for (int j = 0; j < numFieldsToCompress; j++) {
+ for (int j = 0; j < fieldCountToCompress; j++) {
if (cmps[j].compare(pageArray, prevTuple.getFieldStart(j), prevTuple.getFieldLength(j),
pageArray, tuple.getFieldStart(j), tuple.getFieldLength(j)) == 0)
prefixFieldsMatch++;
@@ -207,7 +212,7 @@
// the two tuples must match in exactly the number of
// fields we decided to compress for this keyPartition
int processSegments = 0;
- if (prefixFieldsMatch == numFieldsToCompress)
+ if (prefixFieldsMatch == fieldCountToCompress)
tuplesInSegment++;
else
processSegments++;
@@ -218,7 +223,7 @@
for (int r = 0; r < processSegments; r++) {
// compress current segment and then start new
// segment
- if (tuplesInSegment < occurrenceThreshold || numFieldsToCompress <= 0) {
+ if (tuplesInSegment < occurrenceThreshold || fieldCountToCompress <= 0) {
// segment does not have at least
// occurrenceThreshold tuples, so write tuples
// uncompressed
@@ -235,19 +240,19 @@
// extract prefix, write prefix tuple to buffer,
// and set prefix slot
newPrefixSlots[newPrefixSlots.length - 1 - prefixTupleIndex] = slotManager
- .encodeSlotFields(numFieldsToCompress, prefixFreeSpace);
+ .encodeSlotFields(fieldCountToCompress, prefixFreeSpace);
// int tmp = freeSpace;
// prevRec.reset();
// System.out.println("SOURCE CONTENTS: " +
// buf.getInt(prevRec.getFieldOff()) + " " +
// buf.getInt(prevRec.getFieldOff()+4));
- prefixFreeSpace += tupleWriter.writeTupleFields(prevTuple, 0, numFieldsToCompress,
+ prefixFreeSpace += tupleWriter.writeTupleFields(prevTuple, 0, fieldCountToCompress,
byteBuffer, prefixFreeSpace);
// System.out.println("WRITING PREFIX RECORD " +
// prefixSlotNum + " AT " + tmp + " " +
// freeSpace);
// System.out.print("CONTENTS: ");
- // for(int x = 0; x < numFieldsToCompress; x++)
+ // for(int x = 0; x < fieldCountToCompress; x++)
// System.out.print(buf.getInt(tmp + x*4) +
// " ");
// System.out.println();
@@ -259,8 +264,8 @@
tupleToWrite.resetByTupleIndex(frame, currTupleIndex);
newTupleSlots[tupleCount - 1 - currTupleIndex] = slotManager.encodeSlotFields(
prefixTupleIndex, tupleFreeSpace);
- tupleFreeSpace += tupleWriter.writeTupleFields(tupleToWrite, numFieldsToCompress,
- fieldCount - numFieldsToCompress, byteBuffer, tupleFreeSpace);
+ tupleFreeSpace += tupleWriter.writeTupleFields(tupleToWrite, fieldCountToCompress,
+ fieldCount - fieldCountToCompress, byteBuffer, tupleFreeSpace);
}
prefixTupleIndex++;
@@ -312,8 +317,7 @@
// copy new tuple and new slots into original page
int freeSpaceAfterInit = frame.getOrigFreeSpaceOff();
- System
- .arraycopy(buffer, freeSpaceAfterInit, pageArray, freeSpaceAfterInit, tupleFreeSpace
+ System.arraycopy(buffer, freeSpaceAfterInit, pageArray, freeSpaceAfterInit, tupleFreeSpace
- freeSpaceAfterInit);
// copy prefix slots
@@ -368,7 +372,7 @@
private ArrayList<KeyPartition> getKeyPartitions(BTreeFieldPrefixNSMLeafFrame frame, MultiComparator cmp,
int occurrenceThreshold) {
IBinaryComparator[] cmps = cmp.getComparators();
- int fieldCount = cmp.getKeyFieldCount();
+ int fieldCount = typeTraits.length;
int maxCmps = cmps.length - 1;
ByteBuffer buf = frame.getBuffer();
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
new file mode 100644
index 0000000..3e235b3
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelper.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class BTreeDataflowHelper extends TreeIndexDataflowHelper {
+ public BTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
+ boolean createIfNotExists) {
+ super(opDesc, ctx, partition, createIfNotExists);
+ }
+
+ @Override
+ public ITreeIndex createIndexInstance() throws HyracksDataException {
+ MultiComparator cmp = IndexUtils.createMultiComparator(treeOpDesc.getTreeIndexComparatorFactories());
+ IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
+ ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, indexFileId, 0,
+ metaDataFrameFactory);
+ return new BTree(bufferCache, treeOpDesc.getTreeIndexTypeTraits().length, cmp, freePageManager,
+ treeOpDesc.getTreeIndexInteriorFactory(), treeOpDesc.getTreeIndexLeafFactory());
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
new file mode 100644
index 0000000..308e305
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeDataflowHelperFactory.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
+
+public class BTreeDataflowHelperFactory implements IIndexDataflowHelperFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+ int partition, boolean createIfNotExists) {
+ return new BTreeDataflowHelper(opDesc, ctx, partition, createIfNotExists);
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeOpHelper.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeOpHelper.java
deleted file mode 100644
index eaf5a81..0000000
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeOpHelper.java
+++ /dev/null
@@ -1,40 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.btree.dataflow;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptorHelper;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexHelperOpenMode;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexOpHelper;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-
-public class BTreeOpHelper extends TreeIndexOpHelper {
-
- protected ITreeIndexOperatorDescriptorHelper opDesc;
-
- public BTreeOpHelper(ITreeIndexOperatorDescriptorHelper opDesc, IHyracksTaskContext ctx, int partition,
- IndexHelperOpenMode mode) {
- super(opDesc, ctx, partition, mode);
- this.opDesc = opDesc;
- }
-
- public ITreeIndex createTreeIndex() throws HyracksDataException {
- IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
- ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, indexFileId, 0,
- metaDataFrameFactory);
- return new BTree(bufferCache, freePageManager, opDesc.getTreeIndexInteriorFactory(),
- opDesc.getTreeIndexLeafFactory(), cmp);
- }
-
- public MultiComparator createMultiComparator(IBinaryComparator[] comparators) throws HyracksDataException {
- return new MultiComparator(opDesc.getTreeIndexTypeTraits(), comparators);
- }
-}
\ No newline at end of file
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeOpHelperFactory.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeOpHelperFactory.java
deleted file mode 100644
index 6cc79ab..0000000
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeOpHelperFactory.java
+++ /dev/null
@@ -1,19 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.btree.dataflow;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptorHelper;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexHelperOpenMode;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexOpHelper;
-
-public class BTreeOpHelperFactory implements ITreeIndexOpHelperFactory {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public TreeIndexOpHelper createTreeIndexOpHelper(ITreeIndexOperatorDescriptorHelper opDesc,
- IHyracksTaskContext ctx, int partition, IndexHelperOpenMode mode) {
- return new BTreeOpHelper(opDesc, ctx, partition, mode);
- }
-
-}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
index c542bb0..6314260 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorDescriptor.java
@@ -19,37 +19,36 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public class BTreeSearchOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor {
private static final long serialVersionUID = 1L;
- private boolean isForward;
- private int[] lowKeyFields; // fields in input tuple to be used as low keys
- private int[] highKeyFields; // fields in input tuple to be used as high
+ protected boolean isForward;
+ protected int[] lowKeyFields; // fields in input tuple to be used as low keys
+ protected int[] highKeyFields; // fields in input tuple to be used as high
// keys
- private boolean lowKeyInclusive;
- private boolean highKeyInclusive;
+ protected boolean lowKeyInclusive;
+ protected boolean highKeyInclusive;
public BTreeSearchOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc,
- IStorageManagerInterface storageManager, IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
+ IStorageManagerInterface storageManager, IIndexRegistryProvider<IIndex> indexRegistryProvider,
IFileSplitProvider fileSplitProvider, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] typeTraits,
+ ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] comparatorFactories, boolean isForward, int[] lowKeyFields, int[] highKeyFields,
- boolean lowKeyInclusive, boolean highKeyInclusive, ITreeIndexOpHelperFactory opHelperFactory) {
- super(spec, 1, 1, recDesc, storageManager, treeIndexRegistryProvider, fileSplitProvider, interiorFrameFactory,
- leafFrameFactory, typeTraits, comparatorFactories, null, opHelperFactory);
+ boolean lowKeyInclusive, boolean highKeyInclusive, IIndexDataflowHelperFactory dataflowHelperFactory) {
+ super(spec, 1, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
+ leafFrameFactory, typeTraits, comparatorFactories, dataflowHelperFactory);
this.isForward = isForward;
this.lowKeyFields = lowKeyFields;
this.highKeyFields = highKeyFields;
@@ -58,8 +57,8 @@
}
@Override
- public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, final IOperatorEnvironment env,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
+ int partition, int nPartitions) {
return new BTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, isForward, lowKeyFields,
highKeyFields, lowKeyInclusive, highKeyInclusive);
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
index f53bfd6..8a8c5ce 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeSearchOperatorNodePushable.java
@@ -18,7 +18,6 @@
import java.nio.ByteBuffer;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
@@ -30,47 +29,46 @@
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexHelperOpenMode;
import edu.uci.ics.hyracks.storage.am.common.dataflow.PermutingFrameTupleReference;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexOpHelper;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
public class BTreeSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
- private TreeIndexOpHelper treeIndexOpHelper;
- private FrameTupleAccessor accessor;
+ protected TreeIndexDataflowHelper treeIndexHelper;
+ protected FrameTupleAccessor accessor;
- private ByteBuffer writeBuffer;
- private FrameTupleAppender appender;
- private ArrayTupleBuilder tb;
- private DataOutput dos;
+ protected ByteBuffer writeBuffer;
+ protected FrameTupleAppender appender;
+ protected ArrayTupleBuilder tb;
+ protected DataOutput dos;
- private BTree btree;
- private boolean isForward;
- private PermutingFrameTupleReference lowKey;
- private PermutingFrameTupleReference highKey;
- private boolean lowKeyInclusive;
- private boolean highKeyInclusive;
- private RangePredicate rangePred;
- private MultiComparator lowKeySearchCmp;
- private MultiComparator highKeySearchCmp;
- private ITreeIndexCursor cursor;
- private ITreeIndexFrame cursorFrame;
- private BTreeOpContext opCtx;
+ protected BTree btree;
+ protected boolean isForward;
+ protected PermutingFrameTupleReference lowKey;
+ protected PermutingFrameTupleReference highKey;
+ protected boolean lowKeyInclusive;
+ protected boolean highKeyInclusive;
+ protected RangePredicate rangePred;
+ protected MultiComparator lowKeySearchCmp;
+ protected MultiComparator highKeySearchCmp;
+ protected ITreeIndexCursor cursor;
+ protected ITreeIndexFrame cursorFrame;
+ protected ITreeIndexAccessor indexAccessor;
- private RecordDescriptor recDesc;
+ protected RecordDescriptor recDesc;
public BTreeSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, IRecordDescriptorProvider recordDescProvider, boolean isForward, int[] lowKeyFields,
int[] highKeyFields, boolean lowKeyInclusive, boolean highKeyInclusive) {
- treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory().createTreeIndexOpHelper(opDesc, ctx, partition,
- IndexHelperOpenMode.OPEN);
+ treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
+ opDesc, ctx, partition, false);
this.isForward = isForward;
this.lowKeyInclusive = lowKeyInclusive;
this.highKeyInclusive = highKeyInclusive;
@@ -87,71 +85,48 @@
@Override
public void open() throws HyracksDataException {
- AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexOpHelper
+ AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexHelper
.getOperatorDescriptor();
- accessor = new FrameTupleAccessor(treeIndexOpHelper.getHyracksTaskContext().getFrameSize(), recDesc);
+ accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), recDesc);
cursorFrame = opDesc.getTreeIndexLeafFactory().createFrame();
- cursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) cursorFrame);
+ setCursor();
writer.open();
try {
- treeIndexOpHelper.init();
- btree = (BTree) treeIndexOpHelper.getTreeIndex();
+ treeIndexHelper.init();
+ btree = (BTree) treeIndexHelper.getIndex();
- // construct range predicate
-
- int lowKeySearchFields = btree.getMultiComparator().getComparators().length;
- int highKeySearchFields = btree.getMultiComparator().getComparators().length;
- if (lowKey != null)
- lowKeySearchFields = lowKey.getFieldCount();
- if (highKey != null)
- highKeySearchFields = highKey.getFieldCount();
-
- IBinaryComparator[] lowKeySearchComparators = new IBinaryComparator[lowKeySearchFields];
- for (int i = 0; i < lowKeySearchFields; i++) {
- lowKeySearchComparators[i] = btree.getMultiComparator().getComparators()[i];
- }
- lowKeySearchCmp = new MultiComparator(btree.getMultiComparator().getTypeTraits(), lowKeySearchComparators);
-
- if (lowKeySearchFields == highKeySearchFields) {
- highKeySearchCmp = lowKeySearchCmp;
- } else {
- IBinaryComparator[] highKeySearchComparators = new IBinaryComparator[highKeySearchFields];
- for (int i = 0; i < highKeySearchFields; i++) {
- highKeySearchComparators[i] = btree.getMultiComparator().getComparators()[i];
- }
- highKeySearchCmp = new MultiComparator(btree.getMultiComparator().getTypeTraits(),
- highKeySearchComparators);
-
- }
-
+ // Construct range predicate.
+ lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), lowKey);
+ highKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), highKey);
rangePred = new RangePredicate(isForward, null, null, lowKeyInclusive, highKeyInclusive, lowKeySearchCmp,
highKeySearchCmp);
- writeBuffer = treeIndexOpHelper.getHyracksTaskContext().allocateFrame();
- tb = new ArrayTupleBuilder(btree.getMultiComparator().getFieldCount());
+ writeBuffer = treeIndexHelper.getHyracksTaskContext().allocateFrame();
+ tb = new ArrayTupleBuilder(btree.getFieldCount());
dos = tb.getDataOutput();
- appender = new FrameTupleAppender(treeIndexOpHelper.getHyracksTaskContext().getFrameSize());
+ appender = new FrameTupleAppender(treeIndexHelper.getHyracksTaskContext().getFrameSize());
appender.reset(writeBuffer, true);
-
- opCtx = btree.createOpContext(IndexOp.SEARCH, treeIndexOpHelper.getLeafFrame(),
- treeIndexOpHelper.getInteriorFrame(), null);
-
+ indexAccessor = btree.createAccessor();
} catch (Exception e) {
- treeIndexOpHelper.deinit();
+ treeIndexHelper.deinit();
throw new HyracksDataException(e);
}
}
- private void writeSearchResults() throws Exception {
+ protected void setCursor() {
+ cursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) cursorFrame, false);
+ }
+
+ protected void writeSearchResults() throws Exception {
while (cursor.hasNext()) {
tb.reset();
cursor.next();
- ITupleReference frameTuple = cursor.getTuple();
- for (int i = 0; i < frameTuple.getFieldCount(); i++) {
- dos.write(frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
+ ITupleReference tuple = cursor.getTuple();
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ dos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
tb.addFieldEndOffset();
}
@@ -168,19 +143,19 @@
@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
accessor.reset(buffer);
-
int tupleCount = accessor.getTupleCount();
try {
for (int i = 0; i < tupleCount; i++) {
- if (lowKey != null)
+ if (lowKey != null) {
lowKey.reset(accessor, i);
- if (highKey != null)
+ }
+ if (highKey != null) {
highKey.reset(accessor, i);
+ }
rangePred.setLowKey(lowKey, lowKeyInclusive);
rangePred.setHighKey(highKey, highKeyInclusive);
-
cursor.reset();
- btree.search(cursor, rangePred, opCtx);
+ indexAccessor.search(cursor, rangePred);
writeSearchResults();
}
} catch (Exception e) {
@@ -201,7 +176,7 @@
throw new HyracksDataException(e);
}
} finally {
- treeIndexOpHelper.deinit();
+ treeIndexHelper.deinit();
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
new file mode 100644
index 0000000..8cde863
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorDescriptor.java
@@ -0,0 +1,43 @@
+package edu.uci.ics.hyracks.storage.am.btree.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITupleUpdaterFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
+import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+
+public class BTreeUpdateSearchOperatorDescriptor extends BTreeSearchOperatorDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ private final ITupleUpdaterFactory tupleUpdaterFactory;
+
+ public BTreeUpdateSearchOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc,
+ IStorageManagerInterface storageManager, IIndexRegistryProvider<IIndex> indexRegistryProvider,
+ IFileSplitProvider fileSplitProvider, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] comparatorFactories, boolean isForward, int[] lowKeyFields, int[] highKeyFields,
+ boolean lowKeyInclusive, boolean highKeyInclusive, IIndexDataflowHelperFactory dataflowHelperFactory,
+ ITupleUpdaterFactory tupleUpdaterFactory) {
+ super(spec, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
+ leafFrameFactory, typeTraits, comparatorFactories, isForward, lowKeyFields, highKeyFields, lowKeyInclusive,
+ highKeyInclusive, dataflowHelperFactory);
+ this.tupleUpdaterFactory = tupleUpdaterFactory;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
+ int partition, int nPartitions) {
+ return new BTreeUpdateSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, isForward, lowKeyFields,
+ highKeyFields, lowKeyInclusive, highKeyInclusive, tupleUpdaterFactory.createTupleUpdater());
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
new file mode 100644
index 0000000..06a42bd
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/dataflow/BTreeUpdateSearchOperatorNodePushable.java
@@ -0,0 +1,51 @@
+package edu.uci.ics.hyracks.storage.am.btree.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITupleUpdater;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
+
+public class BTreeUpdateSearchOperatorNodePushable extends BTreeSearchOperatorNodePushable {
+ private final ITupleUpdater tupleUpdater;
+
+ public BTreeUpdateSearchOperatorNodePushable(
+ AbstractTreeIndexOperatorDescriptor opDesc,
+ IHyracksTaskContext ctx, int partition,
+ IRecordDescriptorProvider recordDescProvider, boolean isForward,
+ int[] lowKeyFields, int[] highKeyFields, boolean lowKeyInclusive,
+ boolean highKeyInclusive, ITupleUpdater tupleUpdater) {
+ super(opDesc, ctx, partition, recordDescProvider, isForward, lowKeyFields,
+ highKeyFields, lowKeyInclusive, highKeyInclusive);
+ this.tupleUpdater = tupleUpdater;
+ }
+
+ @Override
+ protected void setCursor() {
+ cursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) cursorFrame, true);
+ }
+
+ @Override
+ protected void writeSearchResults() throws Exception {
+ while (cursor.hasNext()) {
+ tb.reset();
+ cursor.next();
+ ITupleReference tuple = cursor.getTuple();
+ tupleUpdater.updateTuple(tuple);
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ dos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ FrameUtils.flushFrame(writeBuffer, writer);
+ appender.reset(writeBuffer, true);
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeDuplicateKeyException.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeDuplicateKeyException.java
new file mode 100644
index 0000000..d6d945f
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeDuplicateKeyException.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.exceptions;
+
+public class BTreeDuplicateKeyException extends BTreeException {
+ private static final long serialVersionUID = 1L;
+
+ public BTreeDuplicateKeyException(Exception e) {
+ super(e);
+ }
+
+ public BTreeDuplicateKeyException(String message) {
+ super(message);
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeException.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeException.java
new file mode 100644
index 0000000..c568f6f
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeException.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.exceptions;
+
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+
+public class BTreeException extends TreeIndexException {
+
+ protected static final long serialVersionUID = 1L;
+
+ public BTreeException(Exception e) {
+ super(e);
+ }
+
+ public BTreeException(String message) {
+ super(message);
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeNonExistentKeyException.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeNonExistentKeyException.java
new file mode 100644
index 0000000..81a0e79
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeNonExistentKeyException.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.exceptions;
+
+public class BTreeNonExistentKeyException extends BTreeException {
+
+ private static final long serialVersionUID = 1L;
+
+ public BTreeNonExistentKeyException(Exception e) {
+ super(e);
+ }
+
+ public BTreeNonExistentKeyException(String message) {
+ super(message);
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeNotUpdateableException.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeNotUpdateableException.java
new file mode 100644
index 0000000..73b22d8
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/exceptions/BTreeNotUpdateableException.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.exceptions;
+
+public class BTreeNotUpdateableException extends BTreeException {
+ private static final long serialVersionUID = 1L;
+
+ public BTreeNotUpdateableException(Exception e) {
+ super(e);
+ }
+
+ public BTreeNotUpdateableException(String message) {
+ super(message);
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
index bd4947c..0aba70b 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
@@ -15,30 +15,28 @@
package edu.uci.ics.hyracks.storage.am.btree.frames;
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.api.IFrameCompressor;
import edu.uci.ics.hyracks.storage.am.btree.api.IPrefixSlotManager;
import edu.uci.ics.hyracks.storage.am.btree.compressors.FieldPrefixCompressor;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNonExistentKeyException;
import edu.uci.ics.hyracks.storage.am.btree.impls.FieldPrefixPrefixTupleReference;
import edu.uci.ics.hyracks.storage.am.btree.impls.FieldPrefixSlotManager;
import edu.uci.ics.hyracks.storage.am.btree.impls.FieldPrefixTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ISlotManager;
import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameCompressor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
@@ -52,32 +50,34 @@
public class BTreeFieldPrefixNSMLeafFrame implements IBTreeLeafFrame {
protected static final int pageLsnOff = 0; // 0
- protected static final int tupleCountOff = pageLsnOff + 4; // 4
- protected static final int freeSpaceOff = tupleCountOff + 4; // 8
- protected static final int totalFreeSpaceOff = freeSpaceOff + 4; // 12
- protected static final int levelOff = totalFreeSpaceOff + 4; // 16
- protected static final int smFlagOff = levelOff + 1; // 17
- protected static final int uncompressedTupleCountOff = smFlagOff + 1; // 18
- protected static final int prefixTupleCountOff = uncompressedTupleCountOff + 4; // 21
+ protected static final int tupleCountOff = pageLsnOff + 8; // 8
+ protected static final int freeSpaceOff = tupleCountOff + 4; // 12
+ protected static final int totalFreeSpaceOff = freeSpaceOff + 4; // 16
+ protected static final int levelOff = totalFreeSpaceOff + 4; // 20
+ protected static final int smFlagOff = levelOff + 1; // 21
+ protected static final int uncompressedTupleCountOff = smFlagOff + 1; // 22
+ protected static final int prefixTupleCountOff = uncompressedTupleCountOff + 4; // 26
- protected static final int prevLeafOff = prefixTupleCountOff + 4; // 22
- protected static final int nextLeafOff = prevLeafOff + 4; // 26
+ protected static final int prevLeafOff = prefixTupleCountOff + 4; // 30
+ protected static final int nextLeafOff = prevLeafOff + 4; // 34
protected ICachedPage page = null;
protected ByteBuffer buf = null;
- public IFrameCompressor compressor;
- public IPrefixSlotManager slotManager; // TODO: should be protected, but
- // will trigger some refactoring
- private ITreeIndexTupleWriter tupleWriter;
+ public final ITreeIndexFrameCompressor compressor;
+ // TODO: Should be protected, but will trigger some refactoring.
+ public final IPrefixSlotManager slotManager;
- private FieldPrefixTupleReference frameTuple;
- private FieldPrefixPrefixTupleReference framePrefixTuple;
+ private final ITreeIndexTupleWriter tupleWriter;
+ private MultiComparator cmp;
+
+ private final FieldPrefixTupleReference frameTuple;
+ private final FieldPrefixPrefixTupleReference framePrefixTuple;
public BTreeFieldPrefixNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) {
this.tupleWriter = tupleWriter;
this.frameTuple = new FieldPrefixTupleReference(tupleWriter.createTupleReference());
- ITypeTrait[] typeTraits = ((TypeAwareTupleWriter) tupleWriter).getTypeTraits();
+ ITypeTraits[] typeTraits = ((TypeAwareTupleWriter) tupleWriter).getTypeTraits();
this.framePrefixTuple = new FieldPrefixPrefixTupleReference(typeTraits);
this.slotManager = new FieldPrefixSlotManager();
this.compressor = new FieldPrefixCompressor(typeTraits, 0.001f, 2);
@@ -101,7 +101,7 @@
}
@Override
- public boolean compress(MultiComparator cmp) throws HyracksDataException {
+ public boolean compress() throws HyracksDataException {
try {
return compressor.compress(this, cmp);
} catch (Exception e) {
@@ -115,11 +115,9 @@
// 3. prefix tuple are sorted (last prefix tuple is at highest offset)
// this procedure will not move prefix tuples
@Override
- public boolean compact(MultiComparator cmp) {
+ public boolean compact() {
resetSpaceParams();
- frameTuple.setFieldCount(cmp.getFieldCount());
-
int tupleCount = buf.getInt(tupleCountOff);
// determine start of target free space (depends on assumptions stated
@@ -168,82 +166,63 @@
slotManager.setSlot(sortedTupleOffs.get(i).slotOff, slotManager.encodeSlotFields(prefixSlotNum, freeSpace));
freeSpace += tupleLength;
}
-
+
buf.putInt(freeSpaceOff, freeSpace);
int totalFreeSpace = buf.capacity() - buf.getInt(freeSpaceOff)
- ((buf.getInt(tupleCountOff) + buf.getInt(prefixTupleCountOff)) * slotManager.getSlotSize());
buf.putInt(totalFreeSpaceOff, totalFreeSpace);
-
+
return false;
}
@Override
- public void delete(ITupleReference tuple, MultiComparator cmp, boolean exactDelete) throws Exception {
- int slot = slotManager.findSlot(tuple, frameTuple, framePrefixTuple, cmp, FindTupleMode.FTM_EXACT,
- FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY);
+ public void delete(ITupleReference tuple, int slot) {
int tupleIndex = slotManager.decodeSecondSlotField(slot);
- if (tupleIndex == FieldPrefixSlotManager.GREATEST_SLOT) {
- throw new BTreeException("Key to be deleted does not exist.");
+ int prefixSlotNum = slotManager.decodeFirstSlotField(slot);
+ int tupleSlotOff = slotManager.getTupleSlotOff(tupleIndex);
+
+ // perform deletion (we just do a memcpy to overwrite the slot)
+ int slotEndOff = slotManager.getTupleSlotEndOff();
+ int length = tupleSlotOff - slotEndOff;
+ System.arraycopy(buf.array(), slotEndOff, buf.array(), slotEndOff + slotManager.getSlotSize(), length);
+
+ // maintain space information, get size of tuple suffix (suffix
+ // could be entire tuple)
+ int tupleSize = 0;
+ int suffixFieldStart = 0;
+ if (prefixSlotNum == FieldPrefixSlotManager.TUPLE_UNCOMPRESSED) {
+ suffixFieldStart = 0;
+ buf.putInt(uncompressedTupleCountOff, buf.getInt(uncompressedTupleCountOff) - 1);
} else {
- int prefixSlotNum = slotManager.decodeFirstSlotField(slot);
- int tupleSlotOff = slotManager.getTupleSlotOff(tupleIndex);
-
- if (exactDelete) {
- frameTuple.setFieldCount(cmp.getFieldCount());
- frameTuple.resetByTupleIndex(this, tupleIndex);
-
- int comparison = cmp.fieldRangeCompare(tuple, frameTuple, cmp.getKeyFieldCount() - 1, cmp
- .getFieldCount()
- - cmp.getKeyFieldCount());
- if (comparison != 0) {
- throw new BTreeException("Cannot delete tuple. Byte-by-byte comparison failed to prove equality.");
- }
- }
-
- // perform deletion (we just do a memcpy to overwrite the slot)
- int slotEndOff = slotManager.getTupleSlotEndOff();
- int length = tupleSlotOff - slotEndOff;
- System.arraycopy(buf.array(), slotEndOff, buf.array(), slotEndOff + slotManager.getSlotSize(), length);
-
- // maintain space information, get size of tuple suffix (suffix
- // could be entire tuple)
- int tupleSize = 0;
- int suffixFieldStart = 0;
- if (prefixSlotNum == FieldPrefixSlotManager.TUPLE_UNCOMPRESSED) {
- suffixFieldStart = 0;
- buf.putInt(uncompressedTupleCountOff, buf.getInt(uncompressedTupleCountOff) - 1);
- } else {
- int prefixSlot = buf.getInt(slotManager.getPrefixSlotOff(prefixSlotNum));
- suffixFieldStart = slotManager.decodeFirstSlotField(prefixSlot);
- }
-
- frameTuple.resetByTupleIndex(this, tupleIndex);
- tupleSize = tupleWriter.bytesRequired(frameTuple, suffixFieldStart, frameTuple.getFieldCount()
- - suffixFieldStart);
-
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + tupleSize + slotManager.getSlotSize());
+ int prefixSlot = buf.getInt(slotManager.getPrefixSlotOff(prefixSlotNum));
+ suffixFieldStart = slotManager.decodeFirstSlotField(prefixSlot);
}
+
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ tupleSize = tupleWriter.bytesRequired(frameTuple, suffixFieldStart, frameTuple.getFieldCount()
+ - suffixFieldStart);
+
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + tupleSize + slotManager.getSlotSize());
}
@Override
- public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple, MultiComparator cmp) {
+ public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) {
int freeContiguous = buf.capacity() - buf.getInt(freeSpaceOff)
- ((buf.getInt(tupleCountOff) + buf.getInt(prefixTupleCountOff)) * slotManager.getSlotSize());
int bytesRequired = tupleWriter.bytesRequired(tuple);
- // see if the tuple would fit uncompressed
+ // See if the tuple would fit uncompressed.
if (bytesRequired + slotManager.getSlotSize() <= freeContiguous)
return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
- // see if tuple would fit into remaining space after compaction
+ // See if tuple would fit into remaining space after compaction.
if (bytesRequired + slotManager.getSlotSize() <= buf.getInt(totalFreeSpaceOff))
return FrameOpSpaceStatus.SUFFICIENT_SPACE;
- // see if the tuple matches a prefix and will fit after truncating the
- // prefix
- int prefixSlotNum = slotManager.findPrefix(tuple, framePrefixTuple, cmp);
+ // See if the tuple matches a prefix and will fit after truncating the prefix.
+ int prefixSlotNum = slotManager.findPrefix(tuple, framePrefixTuple);
if (prefixSlotNum != FieldPrefixSlotManager.TUPLE_UNCOMPRESSED) {
int prefixSlotOff = slotManager.getPrefixSlotOff(prefixSlotNum);
int prefixSlot = buf.getInt(prefixSlotOff);
@@ -259,11 +238,96 @@
}
@Override
- public FrameOpSpaceStatus hasSpaceUpdate(int rid, ITupleReference tuple, MultiComparator cmp) {
- // TODO Auto-generated method stub
+ public void insert(ITupleReference tuple, int tupleIndex) {
+ int slot = slotManager.insertSlot(tupleIndex, buf.getInt(freeSpaceOff));
+ int prefixSlotNum = slotManager.decodeFirstSlotField(slot);
+ int numPrefixFields = 0;
+ if (prefixSlotNum != FieldPrefixSlotManager.TUPLE_UNCOMPRESSED) {
+ int prefixSlotOff = slotManager.getPrefixSlotOff(prefixSlotNum);
+ int prefixSlot = buf.getInt(prefixSlotOff);
+ numPrefixFields = slotManager.decodeFirstSlotField(prefixSlot);
+ } else {
+ buf.putInt(uncompressedTupleCountOff, buf.getInt(uncompressedTupleCountOff) + 1);
+ }
+
+ int freeSpace = buf.getInt(freeSpaceOff);
+ int bytesWritten = tupleWriter.writeTupleFields(tuple, numPrefixFields,
+ tuple.getFieldCount() - numPrefixFields, buf, freeSpace);
+
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
+ buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - bytesWritten - slotManager.getSlotSize());
+ }
+
+ @Override
+ public FrameOpSpaceStatus hasSpaceUpdate(ITupleReference newTuple, int oldTupleIndex) {
+ int tupleIndex = slotManager.decodeSecondSlotField(oldTupleIndex);
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+
+ int oldTupleBytes = 0;
+ int newTupleBytes = 0;
+
+ int numPrefixFields = frameTuple.getNumPrefixFields();
+ int fieldCount = frameTuple.getFieldCount();
+ if (numPrefixFields != 0) {
+ // Check the space requirements for updating the suffix of the original tuple.
+ oldTupleBytes = frameTuple.getSuffixTupleSize();
+ newTupleBytes = tupleWriter.bytesRequired(newTuple, numPrefixFields, fieldCount - numPrefixFields);
+ } else {
+ // The original tuple is uncompressed.
+ oldTupleBytes = frameTuple.getTupleSize();
+ newTupleBytes = tupleWriter.bytesRequired(newTuple);
+ }
+
+ int additionalBytesRequired = newTupleBytes - oldTupleBytes;
+ // Enough space for an in-place update?
+ if (additionalBytesRequired <= 0) {
+ return FrameOpSpaceStatus.SUFFICIENT_INPLACE_SPACE;
+ }
+
+ int freeContiguous = buf.capacity() - buf.getInt(freeSpaceOff)
+ - ((buf.getInt(tupleCountOff) + buf.getInt(prefixTupleCountOff)) * slotManager.getSlotSize());
+
+ // Enough space if we delete the old tuple and insert the new one without compaction?
+ if (newTupleBytes <= freeContiguous) {
+ return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
+ }
+ // Enough space if we delete the old tuple and compact?
+ if (additionalBytesRequired <= buf.getInt(totalFreeSpaceOff)) {
+ return FrameOpSpaceStatus.SUFFICIENT_SPACE;
+ }
return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
}
+ @Override
+ public void update(ITupleReference newTuple, int oldTupleIndex, boolean inPlace) {
+ int tupleIndex = slotManager.decodeSecondSlotField(oldTupleIndex);
+ int tupleSlotOff = slotManager.getTupleSlotOff(tupleIndex);
+ int tupleSlot = buf.getInt(tupleSlotOff);
+ int prefixSlotNum = slotManager.decodeFirstSlotField(tupleSlot);
+ int suffixTupleStartOff = slotManager.decodeSecondSlotField(tupleSlot);
+
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ int fieldCount = frameTuple.getFieldCount();
+ int numPrefixFields = frameTuple.getNumPrefixFields();
+ int oldTupleBytes = frameTuple.getSuffixTupleSize();
+ int bytesWritten = 0;
+
+ if (inPlace) {
+ // Overwrite the old tuple suffix in place.
+ bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf, suffixTupleStartOff);
+ } else {
+ // Insert the new tuple suffix at the end of the free space, and change the slot value (effectively "deleting" the old tuple).
+ int newSuffixTupleStartOff = buf.getInt(freeSpaceOff);
+ bytesWritten = tupleWriter.writeTupleFields(newTuple, numPrefixFields, fieldCount - numPrefixFields, buf, newSuffixTupleStartOff);
+ // Update slot value using the same prefix slot num.
+ slotManager.setSlot(tupleSlotOff, slotManager.encodeSlotFields(prefixSlotNum, newSuffixTupleStartOff));
+ // Update contiguous free space pointer.
+ buf.putInt(freeSpaceOff, newSuffixTupleStartOff + bytesWritten);
+ }
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + oldTupleBytes - bytesWritten);
+ }
+
protected void resetSpaceParams() {
buf.putInt(freeSpaceOff, getOrigFreeSpaceOff());
buf.putInt(totalFreeSpaceOff, getOrigTotalFreeSpace());
@@ -271,7 +335,7 @@
@Override
public void initBuffer(byte level) {
- buf.putInt(pageLsnOff, 0); // TODO: might to set to a different lsn
+ buf.putLong(pageLsnOff, 0);
// during creation
buf.putInt(tupleCountOff, 0);
resetSpaceParams();
@@ -292,43 +356,55 @@
}
@Override
- public int findTupleIndex(ITupleReference tuple, MultiComparator cmp) throws Exception {
- return slotManager.findSlot(tuple, frameTuple, framePrefixTuple, cmp, FindTupleMode.FTM_INCLUSIVE,
- FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY);
+ public int findInsertTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ int slot = slotManager.findSlot(tuple, frameTuple, framePrefixTuple, cmp, FindTupleMode.EXCLUSIVE_ERROR_IF_EXISTS,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
+ int tupleIndex = slotManager.decodeSecondSlotField(slot);
+ // Error indicator is set if there is an exact match.
+ if (tupleIndex == slotManager.getErrorIndicator()) {
+ throw new BTreeDuplicateKeyException("Trying to insert duplicate key into leaf node.");
+ }
+ return slot;
}
@Override
- public void insert(ITupleReference tuple, MultiComparator cmp, int tupleIndex) throws Exception {
- int slot = slotManager.insertSlot(tupleIndex, buf.getInt(freeSpaceOff));
- int prefixSlotNum = slotManager.decodeFirstSlotField(slot);
- int numPrefixFields = 0;
- if (prefixSlotNum != FieldPrefixSlotManager.TUPLE_UNCOMPRESSED) {
- int prefixSlotOff = slotManager.getPrefixSlotOff(prefixSlotNum);
- int prefixSlot = buf.getInt(prefixSlotOff);
- numPrefixFields = slotManager.decodeFirstSlotField(prefixSlot);
- } else {
- buf.putInt(uncompressedTupleCountOff, buf.getInt(uncompressedTupleCountOff) + 1);
- }
-
- int freeSpace = buf.getInt(freeSpaceOff);
- int bytesWritten = tupleWriter.writeTupleFields(tuple, numPrefixFields,
- tuple.getFieldCount() - numPrefixFields, buf, freeSpace);
-
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
- buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - bytesWritten - slotManager.getSlotSize());
+ public int findUpdateTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ int slot = slotManager.findSlot(tuple, frameTuple, framePrefixTuple, cmp, FindTupleMode.EXACT,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
+ int tupleIndex = slotManager.decodeSecondSlotField(slot);
+ // Error indicator is set if there is no exact match.
+ if (tupleIndex == slotManager.getErrorIndicator()) {
+ throw new BTreeNonExistentKeyException("Trying to update a tuple with a nonexistent key in leaf node.");
+ }
+ return slot;
}
-
+
@Override
- public void update(int rid, ITupleReference tuple) throws Exception {
- // TODO Auto-generated method stub
-
+ public int findDeleteTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ int slot = slotManager.findSlot(tuple, frameTuple, framePrefixTuple, cmp, FindTupleMode.EXACT,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
+ int tupleIndex = slotManager.decodeSecondSlotField(slot);
+ // Error indicator is set if there is no exact match.
+ if (tupleIndex == slotManager.getErrorIndicator()) {
+ throw new BTreeNonExistentKeyException("Trying to delete a tuple with a nonexistent key in leaf node.");
+ }
+ return slot;
}
-
+
@Override
- public void printHeader() {
- // TODO Auto-generated method stub
-
+ public String printHeader() {
+ StringBuilder strBuilder = new StringBuilder();
+ strBuilder.append("pageLsnOff: " + pageLsnOff + "\n");
+ strBuilder.append("tupleCountOff: " + tupleCountOff + "\n");
+ strBuilder.append("freeSpaceOff: " + freeSpaceOff + "\n");
+ strBuilder.append("totalFreeSpaceOff: " + totalFreeSpaceOff + "\n");
+ strBuilder.append("levelOff: " + levelOff + "\n");
+ strBuilder.append("smFlagOff: " + smFlagOff + "\n");
+ strBuilder.append("uncompressedTupleCountOff: " + uncompressedTupleCountOff + "\n");
+ strBuilder.append("prefixTupleCountOff: " + prefixTupleCountOff + "\n");
+ strBuilder.append("prevLeafOff: " + prevLeafOff + "\n");
+ strBuilder.append("nextLeafOff: " + nextLeafOff + "\n");
+ return strBuilder.toString();
}
@Override
@@ -341,26 +417,6 @@
}
@Override
- public String printKeys(MultiComparator cmp, ISerializerDeserializer[] fields) throws HyracksDataException {
- StringBuilder strBuilder = new StringBuilder();
- int tupleCount = buf.getInt(tupleCountOff);
- frameTuple.setFieldCount(fields.length);
- for (int i = 0; i < tupleCount; i++) {
- frameTuple.resetByTupleIndex(this, i);
- for (int j = 0; j < cmp.getKeyFieldCount(); j++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(j), frameTuple
- .getFieldStart(j), frameTuple.getFieldLength(j));
- DataInput dataIn = new DataInputStream(inStream);
- Object o = fields[j].deserialize(dataIn);
- strBuilder.append(o.toString() + " ");
- }
- strBuilder.append(" | ");
- }
- strBuilder.append("\n");
- return strBuilder.toString();
- }
-
- @Override
public int getTupleOffset(int slotNum) {
int tupleSlotOff = slotManager.getTupleSlotOff(slotNum);
int tupleSlot = buf.getInt(tupleSlotOff);
@@ -368,13 +424,13 @@
}
@Override
- public int getPageLsn() {
- return buf.getInt(pageLsnOff);
+ public long getPageLsn() {
+ return buf.getLong(pageLsnOff);
}
@Override
- public void setPageLsn(int pageLsn) {
- buf.putInt(pageLsnOff, pageLsn);
+ public void setPageLsn(long pageLsn) {
+ buf.putLong(pageLsnOff, pageLsn);
}
@Override
@@ -386,7 +442,7 @@
public boolean isLeaf() {
return buf.get(levelOff) == 0;
}
-
+
@Override
public boolean isInterior() {
return buf.get(levelOff) > 0;
@@ -424,7 +480,7 @@
}
@Override
- public void insertSorted(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException {
+ public void insertSorted(ITupleReference tuple) {
int freeSpace = buf.getInt(freeSpaceOff);
int fieldsToTruncate = 0;
@@ -445,7 +501,7 @@
prefixSlotNum = buf.getInt(prefixTupleCountOff) - 1;
else
buf.putInt(uncompressedTupleCountOff, buf.getInt(uncompressedTupleCountOff) + 1);
- int insSlot = slotManager.encodeSlotFields(prefixSlotNum, FieldPrefixSlotManager.GREATEST_SLOT);
+ int insSlot = slotManager.encodeSlotFields(prefixSlotNum, FieldPrefixSlotManager.GREATEST_KEY_INDICATOR);
slotManager.insertSlot(insSlot, freeSpace);
// update page metadata
@@ -455,17 +511,16 @@
}
@Override
- public int split(ITreeIndexFrame rightFrame, ITupleReference tuple, MultiComparator cmp, ISplitKey splitKey)
- throws Exception {
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey)
+ throws TreeIndexException {
- BTreeFieldPrefixNSMLeafFrame rf = (BTreeFieldPrefixNSMLeafFrame) rightFrame;
+ BTreeFieldPrefixNSMLeafFrame rf = (BTreeFieldPrefixNSMLeafFrame)rightFrame;
- frameTuple.setFieldCount(cmp.getFieldCount());
-
ByteBuffer right = rf.getBuffer();
int tupleCount = getTupleCount();
int prefixTupleCount = getPrefixTupleCount();
+ // Find split point, and determine into which frame the new tuple should be inserted into.
int tuplesToLeft;
int midSlotNum = tupleCount / 2;
ITreeIndexFrame targetFrame = null;
@@ -522,7 +577,7 @@
int bytesWritten = 0;
if (lastPrefixSlotNum != prefixSlotNum) {
- bytesWritten = tupleWriter.writeTuple(framePrefixTuple, right, freeSpace);
+ bytesWritten = tupleWriter.writeTuple(framePrefixTuple, right.array(), freeSpace);
int newPrefixSlot = rf.slotManager
.encodeSlotFields(framePrefixTuple.getFieldCount(), freeSpace);
int prefixSlotOff = rf.slotManager.getPrefixSlotOff(prefixSlotNum);
@@ -567,12 +622,12 @@
buf.putInt(prefixTupleCountOff, prefixesToLeft);
// compact both pages
- compact(cmp);
- rightFrame.compact(cmp);
+ compact();
+ rightFrame.compact();
// insert last key
- int targetTupleIndex = targetFrame.findTupleIndex(tuple, cmp);
- targetFrame.insert(tuple, cmp, targetTupleIndex);
+ int targetTupleIndex = ((IBTreeLeafFrame)targetFrame).findInsertTupleIndex(tuple);
+ targetFrame.insert(tuple, targetTupleIndex);
// set split key to be highest value in left page
frameTuple.resetByTupleIndex(this, getTupleCount() - 1);
@@ -581,8 +636,6 @@
splitKey.initData(splitKeySize);
tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
-
- return 0;
}
@Override
@@ -632,11 +685,6 @@
return slotManager.getSlotSize();
}
- @Override
- public void setPageTupleFieldCount(int fieldCount) {
- frameTuple.setFieldCount(fieldCount);
- }
-
public ITreeIndexTupleWriter getTupleWriter() {
return tupleWriter;
}
@@ -651,14 +699,21 @@
FindTupleMode ftm, FindTupleNoExactMatchPolicy ftp) {
int slot = slotManager.findSlot(searchKey, pageTuple, framePrefixTuple, cmp, ftm, ftp);
int tupleIndex = slotManager.decodeSecondSlotField(slot);
- if (tupleIndex == FieldPrefixSlotManager.GREATEST_SLOT)
+ // TODO: Revisit this one. Maybe there is a cleaner way to solve this in the RangeSearchCursor.
+ if (tupleIndex == FieldPrefixSlotManager.GREATEST_KEY_INDICATOR || tupleIndex == FieldPrefixSlotManager.ERROR_INDICATOR)
return -1;
else
return tupleIndex;
}
@Override
- public int getPageHeaderSize() {
- return nextLeafOff;
+ public int getPageHeaderSize() {
+ return nextLeafOff;
+ }
+
+ @Override
+ public void setMultiComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ this.slotManager.setMultiComparator(cmp);
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
index 05b43d3..5defb27 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
@@ -22,8 +22,8 @@
public class BTreeFieldPrefixNSMLeafFrameFactory implements ITreeIndexFrameFactory {
private static final long serialVersionUID = 1L;
- private ITreeIndexTupleWriterFactory tupleWriterFactory;
-
+ private final ITreeIndexTupleWriterFactory tupleWriterFactory;
+
public BTreeFieldPrefixNSMLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
this.tupleWriterFactory = tupleWriterFactory;
}
@@ -32,4 +32,9 @@
public IBTreeLeafFrame createFrame() {
return new BTreeFieldPrefixNSMLeafFrame(tupleWriterFactory.createTupleWriter());
}
+
+ @Override
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return tupleWriterFactory;
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeLeafFrameType.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeLeafFrameType.java
new file mode 100644
index 0000000..6ff44be
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeLeafFrameType.java
@@ -0,0 +1,6 @@
+package edu.uci.ics.hyracks.storage.am.btree.frames;
+
+public enum BTreeLeafFrameType {
+ REGULAR_NSM,
+ FIELD_PREFIX_COMPRESSED_NSM
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
index 6025003..6173440 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
@@ -15,23 +15,19 @@
package edu.uci.ics.hyracks.storage.am.btree.frames;
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.frames.TreeIndexNSMFrame;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
@@ -42,20 +38,14 @@
public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeInteriorFrame {
private static final int rightLeafOff = smFlagOff + 1;
-
private static final int childPtrSize = 4;
-
- // private SimpleTupleReference cmpFrameTuple = new SimpleTupleReference();
- private ITreeIndexTupleReference cmpFrameTuple;
+
+ private final ITreeIndexTupleReference cmpFrameTuple;
+ private MultiComparator cmp;
public BTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter) {
super(tupleWriter, new OrderedSlotManager());
cmpFrameTuple = tupleWriter.createTupleReference();
-
- }
-
- private int getLeftChildPageOff(ITupleReference tuple, MultiComparator cmp) {
- return tuple.getFieldStart(cmp.getKeyFieldCount() - 1) + tuple.getFieldLength(cmp.getKeyFieldCount() - 1);
}
@Override
@@ -65,99 +55,138 @@
}
@Override
- public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple, MultiComparator cmp) {
- int bytesRequired = tupleWriter.bytesRequired(tuple) + 8; // for the two
- // childpointers
- if (bytesRequired + slotManager.getSlotSize() <= buf.capacity() - buf.getInt(freeSpaceOff)
- - (buf.getInt(tupleCountOff) * slotManager.getSlotSize()))
- return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
- else if (bytesRequired + slotManager.getSlotSize() <= buf.getInt(totalFreeSpaceOff))
- return FrameOpSpaceStatus.SUFFICIENT_SPACE;
- else
- return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
- }
-
- public int findTupleIndex(ITupleReference tuple, MultiComparator cmp) throws Exception {
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.FTM_INCLUSIVE,
- FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY);
- int slotOff = slotManager.getSlotOff(tupleIndex);
- boolean isDuplicate = true;
-
- if (tupleIndex < 0)
- isDuplicate = false; // greater than all existing keys
- else {
- frameTuple.resetByTupleOffset(buf, slotManager.getTupleOff(slotOff));
- if (cmp.compare(tuple, frameTuple) != 0)
- isDuplicate = false;
- }
- if (isDuplicate) {
- throw new BTreeException("Trying to insert duplicate value into interior node.");
- }
- return tupleIndex;
+ public int findInsertTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ return slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.INCLUSIVE,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
}
@Override
- public void insert(ITupleReference tuple, MultiComparator cmp, int tupleIndex) throws Exception {
- int slotOff = slotManager.insertSlot(tupleIndex, buf.getInt(freeSpaceOff));
- int freeSpace = buf.getInt(freeSpaceOff);
- int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, cmp.getKeyFieldCount(), buf, freeSpace);
- System.arraycopy(tuple.getFieldData(cmp.getKeyFieldCount() - 1), getLeftChildPageOff(tuple, cmp), buf
- .array(), freeSpace + bytesWritten, childPtrSize);
- int tupleSize = bytesWritten + childPtrSize;
-
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
- buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + tupleSize);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - tupleSize - slotManager.getSlotSize());
-
- // did insert into the rightmost slot?
- if (slotOff == slotManager.getSlotEndOff()) {
- System.arraycopy(tuple.getFieldData(cmp.getKeyFieldCount() - 1), getLeftChildPageOff(tuple, cmp)
- + childPtrSize, buf.array(), rightLeafOff, childPtrSize);
- } else {
- // if slotOff has a right (slot-)neighbor then update its child
- // pointer
- // the only time when this is NOT the case, is when this is the
- // first tuple
- // (or when the splitkey goes into the rightmost slot but that
- // case was handled in the if above)
- if (buf.getInt(tupleCountOff) > 1) {
- int rightNeighborOff = slotOff - slotManager.getSlotSize();
- frameTuple.resetByTupleOffset(buf, slotManager.getTupleOff(rightNeighborOff));
- System.arraycopy(tuple.getFieldData(0), getLeftChildPageOff(tuple, cmp) + childPtrSize,
- buf.array(), getLeftChildPageOff(frameTuple, cmp), childPtrSize);
- }
- }
+ public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) {
+ // Tuple bytes + child pointer + slot.
+ int bytesRequired = tupleWriter.bytesRequired(tuple) + childPtrSize + slotManager.getSlotSize();
+ if (bytesRequired <= getFreeContiguousSpace()) {
+ return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
+ }
+ if (bytesRequired <= getTotalFreeSpace()) {
+ return FrameOpSpaceStatus.SUFFICIENT_SPACE;
+ }
+ return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
}
-
@Override
- public void insertSorted(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException {
+ public void insert(ITupleReference tuple, int tupleIndex) {
+ int slotOff = slotManager.insertSlot(tupleIndex, buf.getInt(freeSpaceOff));
int freeSpace = buf.getInt(freeSpaceOff);
- slotManager.insertSlot(-1, freeSpace);
- int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, cmp.getKeyFieldCount(), buf, freeSpace);
- System.arraycopy(tuple.getFieldData(cmp.getKeyFieldCount() - 1), getLeftChildPageOff(tuple, cmp), buf.array(),
+ int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf, freeSpace);
+ System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), getLeftChildPageOff(tuple), buf.array(),
freeSpace + bytesWritten, childPtrSize);
int tupleSize = bytesWritten + childPtrSize;
buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + tupleSize);
buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - tupleSize - slotManager.getSlotSize());
- System.arraycopy(tuple.getFieldData(0), getLeftChildPageOff(tuple, cmp) + childPtrSize, buf.array(),
- rightLeafOff, childPtrSize);
+ // Did we insert into the rightmost slot?
+ if (slotOff == slotManager.getSlotEndOff()) {
+ System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), getLeftChildPageOff(tuple) + childPtrSize,
+ buf.array(), rightLeafOff, childPtrSize);
+ } else {
+ // If slotOff has a right (slot-)neighbor then update its child
+ // pointer.
+ // The only time when this is NOT the case, is when this is the
+ // very first tuple (or when the splitkey goes into the rightmost
+ // slot but that
+ // case is handled in the if above).
+ if (buf.getInt(tupleCountOff) > 1) {
+ int rightNeighborOff = slotOff - slotManager.getSlotSize();
+ frameTuple.resetByTupleOffset(buf, slotManager.getTupleOff(rightNeighborOff));
+ System.arraycopy(tuple.getFieldData(0), getLeftChildPageOff(tuple) + childPtrSize, buf.array(),
+ getLeftChildPageOff(frameTuple), childPtrSize);
+ }
+ }
+ }
+
+ @Override
+ public int findDeleteTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ return slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.INCLUSIVE,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
}
@Override
- public int split(ITreeIndexFrame rightFrame, ITupleReference tuple, MultiComparator cmp, ISplitKey splitKey)
- throws Exception {
- // before doing anything check if key already exists
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
-
- ByteBuffer right = rightFrame.getBuffer();
- int tupleCount = buf.getInt(tupleCountOff);
+ public void delete(ITupleReference tuple, int tupleIndex) {
+ int slotOff = slotManager.getSlotOff(tupleIndex);
+ int tupleOff;
+ int keySize;
+ if (tupleIndex == slotManager.getGreatestKeyIndicator()) {
+ tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ keySize = frameTuple.getTupleSize();
+ // Copy new rightmost pointer.
+ System.arraycopy(buf.array(), tupleOff + keySize, buf.array(), rightLeafOff, childPtrSize);
+ } else {
+ tupleOff = slotManager.getTupleOff(slotOff);
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ keySize = frameTuple.getTupleSize();
+ // Perform deletion (we just do a memcpy to overwrite the slot).
+ int slotStartOff = slotManager.getSlotEndOff();
+ int length = slotOff - slotStartOff;
+ System.arraycopy(buf.array(), slotStartOff, buf.array(), slotStartOff + slotManager.getSlotSize(), length);
+ }
+ // Maintain space information.
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
+ buf.putInt(totalFreeSpaceOff,
+ buf.getInt(totalFreeSpaceOff) + keySize + childPtrSize + slotManager.getSlotSize());
+ }
+
+ @Override
+ public void deleteGreatest() {
+ int slotOff = slotManager.getSlotEndOff();
+ int tupleOff = slotManager.getTupleOff(slotOff);
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ int keySize = tupleWriter.bytesRequired(frameTuple);
+ System.arraycopy(buf.array(), tupleOff + keySize, buf.array(), rightLeafOff, childPtrSize);
+ // Maintain space information.
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
+ buf.putInt(totalFreeSpaceOff,
+ buf.getInt(totalFreeSpaceOff) + keySize + childPtrSize + slotManager.getSlotSize());
+ int freeSpace = buf.getInt(freeSpaceOff);
+ if (freeSpace == tupleOff + keySize + childPtrSize) {
+ buf.putInt(freeSpace, freeSpace - (keySize + childPtrSize));
+ }
+ }
+
+ @Override
+ public FrameOpSpaceStatus hasSpaceUpdate(ITupleReference tuple, int oldTupleIndex) {
+ throw new UnsupportedOperationException("Cannot update tuples in interior node.");
+ }
+
+ @Override
+ public int findUpdateTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ throw new UnsupportedOperationException("Cannot update tuples in interior node.");
+ }
+ @Override
+ public void insertSorted(ITupleReference tuple) {
+ int freeSpace = buf.getInt(freeSpaceOff);
+ slotManager.insertSlot(slotManager.getGreatestKeyIndicator(), freeSpace);
+ int bytesWritten = tupleWriter.writeTuple(tuple, buf, freeSpace);
+ System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), getLeftChildPageOff(tuple), buf.array(),
+ freeSpace + bytesWritten, childPtrSize);
+ int tupleSize = bytesWritten + childPtrSize;
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
+ buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + tupleSize);
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - tupleSize - slotManager.getSlotSize());
+ System.arraycopy(tuple.getFieldData(0), getLeftChildPageOff(tuple) + childPtrSize, buf.array(), rightLeafOff,
+ childPtrSize);
+ }
+
+ @Override
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException {
+ ByteBuffer right = rightFrame.getBuffer();
+ int tupleCount = getTupleCount();
+
+ // Find split point, and determine into which frame the new tuple should be inserted into.
int tuplesToLeft = (tupleCount / 2) + (tupleCount % 2);
ITreeIndexFrame targetFrame = null;
- frameTuple.resetByTupleOffset(buf, getTupleOffset(tuplesToLeft - 1));
+ frameTuple.resetByTupleIndex(this, tuplesToLeft - 1);
if (cmp.compare(tuple, frameTuple) <= 0) {
targetFrame = this;
} else {
@@ -165,10 +194,10 @@
}
int tuplesToRight = tupleCount - tuplesToLeft;
- // copy entire page
+ // Copy entire page.
System.arraycopy(buf.array(), 0, right.array(), 0, buf.capacity());
- // on right page we need to copy rightmost slots to left
+ // On the right page we need to copy rightmost slots to left.
int src = rightFrame.getSlotManager().getSlotEndOff();
int dest = rightFrame.getSlotManager().getSlotEndOff() + tuplesToLeft
* rightFrame.getSlotManager().getSlotSize();
@@ -176,48 +205,44 @@
System.arraycopy(right.array(), src, right.array(), dest, length);
right.putInt(tupleCountOff, tuplesToRight);
- // on left page, remove highest key and make its childpointer the
- // rightmost childpointer
+ // On the left page, remove the highest key and make its child pointer
+ // the rightmost child pointer.
buf.putInt(tupleCountOff, tuplesToLeft);
- // copy data to be inserted, we need this because creating the splitkey
- // will overwrite the data param (data points to same memory as
- // splitKey.getData())
+ // Copy the split key to be inserted.
+ // We must do so because setting the new split key will overwrite the
+ // old split key, and we cannot insert the existing split key at this point.
ISplitKey savedSplitKey = splitKey.duplicate(tupleWriter.createTupleReference());
- // set split key to be highest value in left page
+ // Set split key to be highest value in left page.
int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
frameTuple.resetByTupleOffset(buf, tupleOff);
int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
splitKey.initData(splitKeySize);
- tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
+ tupleWriter.writeTuple(frameTuple, splitKey.getBuffer(), 0);
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
int deleteTupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
frameTuple.resetByTupleOffset(buf, deleteTupleOff);
- buf.putInt(rightLeafOff, buf.getInt(getLeftChildPageOff(frameTuple, cmp)));
+ buf.putInt(rightLeafOff, buf.getInt(getLeftChildPageOff(frameTuple)));
buf.putInt(tupleCountOff, tuplesToLeft - 1);
- // compact both pages
- rightFrame.compact(cmp);
- compact(cmp);
+ // Compact both pages.
+ rightFrame.compact();
+ compact();
- // insert key
- int targetTupleIndex = targetFrame.findTupleIndex(savedSplitKey.getTuple(), cmp);
- targetFrame.insert(savedSplitKey.getTuple(), cmp, targetTupleIndex);
-
- return 0;
+ // Insert the saved split key.
+ int targetTupleIndex = ((BTreeNSMInteriorFrame) targetFrame)
+ .findInsertTupleIndex(savedSplitKey.getTuple());
+ targetFrame.insert(savedSplitKey.getTuple(), targetTupleIndex);
}
@Override
- public boolean compact(MultiComparator cmp) {
+ public boolean compact() {
resetSpaceParams();
-
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
-
int tupleCount = buf.getInt(tupleCountOff);
int freeSpace = buf.getInt(freeSpaceOff);
-
+ // Sort the slots by the tuple offset they point to.
ArrayList<SlotOffTupleOff> sortedTupleOffs = new ArrayList<SlotOffTupleOff>();
sortedTupleOffs.ensureCapacity(tupleCount);
for (int i = 0; i < tupleCount; i++) {
@@ -226,133 +251,109 @@
sortedTupleOffs.add(new SlotOffTupleOff(i, slotOff, tupleOff));
}
Collections.sort(sortedTupleOffs);
-
+ // Iterate over the sorted slots, and move their corresponding tuples to
+ // the left, reclaiming free space.
for (int i = 0; i < sortedTupleOffs.size(); i++) {
int tupleOff = sortedTupleOffs.get(i).tupleOff;
frameTuple.resetByTupleOffset(buf, tupleOff);
-
int tupleEndOff = frameTuple.getFieldStart(frameTuple.getFieldCount() - 1)
+ frameTuple.getFieldLength(frameTuple.getFieldCount() - 1);
int tupleLength = tupleEndOff - tupleOff + childPtrSize;
System.arraycopy(buf.array(), tupleOff, buf.array(), freeSpace, tupleLength);
-
slotManager.setSlot(sortedTupleOffs.get(i).slotOff, freeSpace);
freeSpace += tupleLength;
}
-
+ // Update contiguous free space pointer and total free space indicator.
buf.putInt(freeSpaceOff, freeSpace);
buf.putInt(totalFreeSpaceOff, buf.capacity() - freeSpace - tupleCount * slotManager.getSlotSize());
-
return false;
}
@Override
- public int getChildPageId(RangePredicate pred, MultiComparator srcCmp) {
- // check for trivial case where there is only a child pointer (and no
- // key)
+ public int getChildPageId(RangePredicate pred) {
+ // Trivial case where there is only a child pointer (and no key).
if (buf.getInt(tupleCountOff) == 0) {
return buf.getInt(rightLeafOff);
}
-
- cmpFrameTuple.setFieldCount(srcCmp.getKeyFieldCount());
- frameTuple.setFieldCount(srcCmp.getKeyFieldCount());
-
- // check for trivial cases where no low key or high key exists (e.g.
- // during an index scan)
+ // Trivial cases where no low key or high key was given (e.g.
+ // during an index scan).
ITupleReference tuple = null;
FindTupleMode fsm = null;
+ // The target comparator may be on a prefix of the BTree key fields.
MultiComparator targetCmp = null;
- if (pred.isForward()) {
+ if (pred.isForward()) {
tuple = pred.getLowKey();
if (tuple == null) {
- return getLeftmostChildPageId(srcCmp);
+ return getLeftmostChildPageId();
}
- if (pred.isLowKeyInclusive())
- fsm = FindTupleMode.FTM_INCLUSIVE;
- else
- fsm = FindTupleMode.FTM_EXCLUSIVE;
+ if (pred.isLowKeyInclusive()) {
+ fsm = FindTupleMode.INCLUSIVE;
+ } else {
+ fsm = FindTupleMode.EXCLUSIVE;
+ }
targetCmp = pred.getLowKeyComparator();
} else {
tuple = pred.getHighKey();
if (tuple == null) {
- return getRightmostChildPageId(srcCmp);
+ return getRightmostChildPageId();
}
- if (pred.isHighKeyInclusive())
- fsm = FindTupleMode.FTM_EXCLUSIVE;
- else
- fsm = FindTupleMode.FTM_INCLUSIVE;
+ if (pred.isHighKeyInclusive()) {
+ fsm = FindTupleMode.EXCLUSIVE;
+ } else {
+ fsm = FindTupleMode.INCLUSIVE;
+ }
targetCmp = pred.getHighKeyComparator();
}
-
+ // Search for a matching key.
int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, targetCmp, fsm,
- FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY);
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
int slotOff = slotManager.getSlotOff(tupleIndex);
- if (tupleIndex < 0) {
+ // Follow the rightmost (greatest) child pointer.
+ if (tupleIndex == slotManager.getGreatestKeyIndicator()) {
return buf.getInt(rightLeafOff);
- } else {
- int origTupleOff = slotManager.getTupleOff(slotOff);
- cmpFrameTuple.resetByTupleOffset(buf, origTupleOff);
- int cmpTupleOff = origTupleOff;
- if (pred.isForward()) {
- int maxSlotOff = buf.capacity();
- slotOff += slotManager.getSlotSize();
- while (slotOff < maxSlotOff) {
- cmpTupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.resetByTupleOffset(buf, cmpTupleOff);
- if (targetCmp.compare(cmpFrameTuple, frameTuple) != 0)
- break;
- slotOff += slotManager.getSlotSize();
- }
- slotOff -= slotManager.getSlotSize();
- } else {
- int minSlotOff = slotManager.getSlotEndOff() - slotManager.getSlotSize();
- slotOff -= slotManager.getSlotSize();
- while (slotOff > minSlotOff) {
- cmpTupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.resetByTupleOffset(buf, cmpTupleOff);
- if (targetCmp.compare(cmpFrameTuple, frameTuple) != 0)
- break;
- slotOff -= slotManager.getSlotSize();
+ }
+ // Deal with prefix searches.
+ // slotManager.findTupleIndex() will return an arbitrary tuple matching
+ // the given field prefix (according to the target comparator).
+ // To make sure we traverse the right path, we must find the
+ // leftmost or rightmost tuple that matches the prefix.
+ int origTupleOff = slotManager.getTupleOff(slotOff);
+ cmpFrameTuple.resetByTupleOffset(buf, origTupleOff);
+ int cmpTupleOff = origTupleOff;
+ if (pred.isForward()) {
+ // The answer set begins with the lowest key matching the prefix.
+ // We must follow the child pointer of the lowest (leftmost) key
+ // matching the given prefix.
+ int maxSlotOff = buf.capacity();
+ slotOff += slotManager.getSlotSize();
+ while (slotOff < maxSlotOff) {
+ cmpTupleOff = slotManager.getTupleOff(slotOff);
+ frameTuple.resetByTupleOffset(buf, cmpTupleOff);
+ if (targetCmp.compare(cmpFrameTuple, frameTuple) != 0) {
+ break;
}
slotOff += slotManager.getSlotSize();
}
-
- frameTuple.resetByTupleOffset(buf, slotManager.getTupleOff(slotOff));
- int childPageOff = getLeftChildPageOff(frameTuple, srcCmp);
- return buf.getInt(childPageOff);
- }
- }
-
- @Override
- public void delete(ITupleReference tuple, MultiComparator cmp, boolean exactDelete) throws Exception {
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.FTM_INCLUSIVE,
- FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY);
- int slotOff = slotManager.getSlotOff(tupleIndex);
- int tupleOff;
- int keySize;
-
- if (tupleIndex < 0) {
- tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
- frameTuple.resetByTupleOffset(buf, tupleOff);
- keySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
-
- // copy new rightmost pointer
- System.arraycopy(buf.array(), tupleOff + keySize, buf.array(), rightLeafOff, childPtrSize);
+ slotOff -= slotManager.getSlotSize();
} else {
- tupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.resetByTupleOffset(buf, tupleOff);
- keySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
- // perform deletion (we just do a memcpy to overwrite the slot)
- int slotStartOff = slotManager.getSlotEndOff();
- int length = slotOff - slotStartOff;
- System.arraycopy(buf.array(), slotStartOff, buf.array(), slotStartOff + slotManager.getSlotSize(), length);
+ // The answer set begins with the highest key matching the prefix.
+ // We must follow the child pointer of the highest (rightmost) key
+ // matching the given prefix.
+ int minSlotOff = slotManager.getSlotEndOff() - slotManager.getSlotSize();
+ slotOff -= slotManager.getSlotSize();
+ while (slotOff > minSlotOff) {
+ cmpTupleOff = slotManager.getTupleOff(slotOff);
+ frameTuple.resetByTupleOffset(buf, cmpTupleOff);
+ if (targetCmp.compare(cmpFrameTuple, frameTuple) != 0) {
+ break;
+ }
+ slotOff -= slotManager.getSlotSize();
+ }
+ slotOff += slotManager.getSlotSize();
}
-
- // maintain space information
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + keySize + childPtrSize
- + slotManager.getSlotSize());
+ frameTuple.resetByTupleOffset(buf, slotManager.getTupleOff(slotOff));
+ int childPageOff = getLeftChildPageOff(frameTuple);
+ return buf.getInt(childPageOff);
}
@Override
@@ -362,16 +363,15 @@
}
@Override
- public int getLeftmostChildPageId(MultiComparator cmp) {
+ public int getLeftmostChildPageId() {
int tupleOff = slotManager.getTupleOff(slotManager.getSlotStartOff());
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
frameTuple.resetByTupleOffset(buf, tupleOff);
- int childPageOff = getLeftChildPageOff(frameTuple, cmp);
+ int childPageOff = getLeftChildPageOff(frameTuple);
return buf.getInt(childPageOff);
}
@Override
- public int getRightmostChildPageId(MultiComparator cmp) {
+ public int getRightmostChildPageId() {
return buf.getInt(rightLeafOff);
}
@@ -380,7 +380,44 @@
buf.putInt(rightLeafOff, pageId);
}
- // for debugging
+ @Override
+ public int getPageHeaderSize() {
+ return rightLeafOff;
+ }
+
+ private int getLeftChildPageOff(ITupleReference tuple) {
+ return tuple.getFieldStart(tuple.getFieldCount() - 1) + tuple.getFieldLength(tuple.getFieldCount() - 1);
+ }
+
+ @Override
+ public boolean getSmFlag() {
+ return buf.get(smFlagOff) != 0;
+ }
+
+ @Override
+ public void setSmFlag(boolean smFlag) {
+ if (smFlag) {
+ buf.put(smFlagOff, (byte) 1);
+ } else {
+ buf.put(smFlagOff, (byte) 0);
+ }
+ }
+
+ @Override
+ public void setMultiComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ cmpFrameTuple.setFieldCount(cmp.getKeyFieldCount());
+ frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ }
+
+ @Override
+ public ITreeIndexTupleReference createTupleReference() {
+ ITreeIndexTupleReference tuple = tupleWriter.createTupleReference();
+ tuple.setFieldCount(cmp.getKeyFieldCount());
+ return tuple;
+ }
+
+ // For debugging.
public ArrayList<Integer> getChildren(MultiComparator cmp) {
ArrayList<Integer> ret = new ArrayList<Integer>();
frameTuple.setFieldCount(cmp.getKeyFieldCount());
@@ -388,8 +425,10 @@
for (int i = 0; i < tupleCount; i++) {
int tupleOff = slotManager.getTupleOff(slotManager.getSlotOff(i));
frameTuple.resetByTupleOffset(buf, tupleOff);
- int intVal = getInt(buf.array(), frameTuple.getFieldStart(frameTuple.getFieldCount() - 1)
- + frameTuple.getFieldLength(frameTuple.getFieldCount() - 1));
+ int intVal = IntegerSerializerDeserializer.getInt(
+ buf.array(),
+ frameTuple.getFieldStart(frameTuple.getFieldCount() - 1)
+ + frameTuple.getFieldLength(frameTuple.getFieldCount() - 1));
ret.add(intVal);
}
if (!isLeaf()) {
@@ -399,54 +438,4 @@
}
return ret;
}
-
- @Override
- public void deleteGreatest(MultiComparator cmp) {
- int slotOff = slotManager.getSlotEndOff();
- int tupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- frameTuple.resetByTupleOffset(buf, tupleOff);
- int keySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
- System.arraycopy(buf.array(), tupleOff + keySize, buf.array(), rightLeafOff, childPtrSize);
-
- // maintain space information
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + keySize + childPtrSize
- + slotManager.getSlotSize());
-
- int freeSpace = buf.getInt(freeSpaceOff);
- if (freeSpace == tupleOff + keySize + childPtrSize) {
- buf.putInt(freeSpace, freeSpace - (keySize + childPtrSize));
- }
- }
-
- private int getInt(byte[] bytes, int offset) {
- return ((bytes[offset] & 0xff) << 24) + ((bytes[offset + 1] & 0xff) << 16) + ((bytes[offset + 2] & 0xff) << 8)
- + ((bytes[offset + 3] & 0xff) << 0);
- }
-
- @Override
- public String printKeys(MultiComparator cmp, ISerializerDeserializer[] fields) throws HyracksDataException {
- StringBuilder strBuilder = new StringBuilder();
- int tupleCount = buf.getInt(tupleCountOff);
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- for (int i = 0; i < tupleCount; i++) {
- frameTuple.resetByTupleIndex(this, i);
- for (int j = 0; j < cmp.getKeyFieldCount(); j++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(j), frameTuple
- .getFieldStart(j), frameTuple.getFieldLength(j));
- DataInput dataIn = new DataInputStream(inStream);
- Object o = fields[j].deserialize(dataIn);
- strBuilder.append(o.toString() + " ");
- }
- strBuilder.append(" | ");
- }
- strBuilder.append("\n");
- return strBuilder.toString();
- }
-
- @Override
- public int getPageHeaderSize() {
- return rightLeafOff;
- }
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
index 6b30ee0..8618df8 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
@@ -22,7 +22,7 @@
public class BTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory {
private static final long serialVersionUID = 1L;
- private ITreeIndexTupleWriterFactory tupleWriterFactory;
+ private final ITreeIndexTupleWriterFactory tupleWriterFactory;
public BTreeNSMInteriorFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
this.tupleWriterFactory = tupleWriterFactory;
@@ -32,4 +32,9 @@
public IBTreeInteriorFrame createFrame() {
return new BTreeNSMInteriorFrame(tupleWriterFactory.createTupleWriter());
}
+
+ @Override
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return tupleWriterFactory;
+ }
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
index 85fbec9..4856595 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
@@ -17,14 +17,15 @@
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNonExistentKeyException;
import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.TreeIndexNSMFrame;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
@@ -33,7 +34,8 @@
public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFrame {
protected static final int prevLeafOff = smFlagOff + 1;
protected static final int nextLeafOff = prevLeafOff + 4;
-
+ private MultiComparator cmp;
+
public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) {
super(tupleWriter, new OrderedSlotManager());
}
@@ -66,56 +68,59 @@
}
@Override
- public int findTupleIndex(ITupleReference tuple, MultiComparator cmp) throws Exception {
- frameTuple.setFieldCount(cmp.getFieldCount());
- int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.FTM_INCLUSIVE,
- FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY);
- int slotOff = slotManager.getSlotOff(tupleIndex);
- boolean isDuplicate = true;
-
- if (tupleIndex < 0)
- isDuplicate = false; // greater than all existing keys
- else {
- frameTuple.resetByTupleOffset(buf, slotManager.getTupleOff(slotOff));
- if (cmp.compare(tuple, frameTuple) != 0)
- isDuplicate = false;
+ public int findInsertTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.EXCLUSIVE_ERROR_IF_EXISTS,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
+ // Error indicator is set if there is an exact match.
+ if (tupleIndex == slotManager.getErrorIndicator()) {
+ throw new BTreeDuplicateKeyException("Trying to insert duplicate key into leaf node.");
}
- if (isDuplicate) {
- throw new BTreeException("Trying to insert duplicate value into leaf of unique index");
- }
-
return tupleIndex;
}
@Override
- public void insert(ITupleReference tuple, MultiComparator cmp, int tupleIndex) throws Exception {
- slotManager.insertSlot(tupleIndex, buf.getInt(freeSpaceOff));
- int freeSpace = buf.getInt(freeSpaceOff);
- int bytesWritten = tupleWriter.writeTuple(tuple, buf, freeSpace);
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
- buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - bytesWritten - slotManager.getSlotSize());
+ public int findUpdateTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.EXACT,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
+ // Error indicator is set if there is no exact match.
+ if (tupleIndex == slotManager.getErrorIndicator()) {
+ throw new BTreeNonExistentKeyException("Trying to update a tuple with a nonexistent key in leaf node.");
+ }
+ return tupleIndex;
+ }
+
+ @Override
+ public int findDeleteTupleIndex(ITupleReference tuple) throws TreeIndexException {
+ int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp, FindTupleMode.EXACT,
+ FindTupleNoExactMatchPolicy.HIGHER_KEY);
+ // Error indicator is set if there is no exact match.
+ if (tupleIndex == slotManager.getErrorIndicator()) {
+ throw new BTreeNonExistentKeyException("Trying to delete a tuple with a nonexistent key in leaf node.");
+ }
+ return tupleIndex;
}
@Override
- public void insertSorted(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException {
+ public void insert(ITupleReference tuple, int tupleIndex) {
int freeSpace = buf.getInt(freeSpaceOff);
- slotManager.insertSlot(-1, freeSpace);
- int bytesWritten = tupleWriter.writeTuple(tuple, buf, freeSpace);
+ slotManager.insertSlot(tupleIndex, freeSpace);
+ int bytesWritten = tupleWriter.writeTuple(tuple, buf.array(), freeSpace);
buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - bytesWritten - slotManager.getSlotSize());
}
@Override
- public int split(ITreeIndexFrame rightFrame, ITupleReference tuple, MultiComparator cmp, ISplitKey splitKey)
- throws Exception {
+ public void insertSorted(ITupleReference tuple) {
+ insert(tuple, slotManager.getGreatestKeyIndicator());
+ }
- frameTuple.setFieldCount(cmp.getFieldCount());
-
+ @Override
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException {
ByteBuffer right = rightFrame.getBuffer();
- int tupleCount = getTupleCount();
-
+ int tupleCount = getTupleCount();
+
+ // Find split point, and determine into which frame the new tuple should be inserted into.
int tuplesToLeft;
int mid = tupleCount / 2;
ITreeIndexFrame targetFrame = null;
@@ -130,10 +135,10 @@
}
int tuplesToRight = tupleCount - tuplesToLeft;
- // copy entire page
+ // Copy entire page.
System.arraycopy(buf.array(), 0, right.array(), 0, buf.capacity());
- // on right page we need to copy rightmost slots to left
+ // On the right page we need to copy rightmost slots to the left.
int src = rightFrame.getSlotManager().getSlotEndOff();
int dest = rightFrame.getSlotManager().getSlotEndOff() + tuplesToLeft
* rightFrame.getSlotManager().getSlotSize();
@@ -141,27 +146,24 @@
System.arraycopy(right.array(), src, right.array(), dest, length);
right.putInt(tupleCountOff, tuplesToRight);
- // on left page only change the tupleCount indicator
+ // On left page only change the tupleCount indicator.
buf.putInt(tupleCountOff, tuplesToLeft);
- // compact both pages
- rightFrame.compact(cmp);
- compact(cmp);
+ // Compact both pages.
+ rightFrame.compact();
+ compact();
- // insert last key
- int targetTupleIndex = targetFrame.findTupleIndex(tuple, cmp);
- targetFrame.insert(tuple, cmp, targetTupleIndex);
+ // Insert the new tuple.
+ int targetTupleIndex = ((BTreeNSMLeafFrame)targetFrame).findInsertTupleIndex(tuple);
+ targetFrame.insert(tuple, targetTupleIndex);
- // set split key to be highest value in left page
+ // Set the split key to be highest key in the left page.
tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
frameTuple.resetByTupleOffset(buf, tupleOff);
-
int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, cmp.getKeyFieldCount());
splitKey.initData(splitKeySize);
tupleWriter.writeTupleFields(frameTuple, 0, cmp.getKeyFieldCount(), splitKey.getBuffer(), 0);
splitKey.getTuple().resetByTupleOffset(splitKey.getBuffer(), 0);
-
- return 0;
}
@Override
@@ -181,8 +183,27 @@
return slotManager.findTupleIndex(searchKey, pageTuple, cmp, ftm, ftp);
}
+ @Override
+ public int getPageHeaderSize() {
+ return nextLeafOff;
+ }
+
+ @Override
+ public boolean getSmFlag() {
+ return buf.get(smFlagOff) != 0;
+ }
+
+ @Override
+ public void setSmFlag(boolean smFlag) {
+ if (smFlag) {
+ buf.put(smFlagOff, (byte) 1);
+ } else {
+ buf.put(smFlagOff, (byte) 0);
+ }
+ }
+
@Override
- public int getPageHeaderSize() {
- return nextLeafOff;
+ public void setMultiComparator(MultiComparator cmp) {
+ this.cmp = cmp;
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
index d59b391..9508df5 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
@@ -22,8 +22,8 @@
public class BTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory {
private static final long serialVersionUID = 1L;
- private ITreeIndexTupleWriterFactory tupleWriterFactory;
-
+ private final ITreeIndexTupleWriterFactory tupleWriterFactory;
+
public BTreeNSMLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
this.tupleWriterFactory = tupleWriterFactory;
}
@@ -32,4 +32,9 @@
public IBTreeLeafFrame createFrame() {
return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter());
}
+
+ @Override
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return tupleWriterFactory;
+ }
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/OrderedSlotManager.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/OrderedSlotManager.java
index 89a70e0..5f507f5 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/OrderedSlotManager.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/OrderedSlotManager.java
@@ -15,12 +15,7 @@
package edu.uci.ics.hyracks.storage.am.btree.frames;
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.frames.AbstractSlotManager;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
@@ -32,8 +27,9 @@
@Override
public int findTupleIndex(ITupleReference searchKey, ITreeIndexTupleReference frameTuple, MultiComparator multiCmp,
FindTupleMode mode, FindTupleNoExactMatchPolicy matchPolicy) {
- if (frame.getTupleCount() <= 0)
- return -1;
+ if (frame.getTupleCount() <= 0) {
+ return GREATEST_KEY_INDICATOR;
+ }
int mid;
int begin = 0;
@@ -49,43 +45,53 @@
} else if (cmp > 0) {
begin = mid + 1;
} else {
- if (mode == FindTupleMode.FTM_EXCLUSIVE) {
- if (matchPolicy == FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY)
+ if (mode == FindTupleMode.EXCLUSIVE) {
+ if (matchPolicy == FindTupleNoExactMatchPolicy.HIGHER_KEY) {
begin = mid + 1;
- else
+ } else {
end = mid - 1;
+ }
} else {
- return mid;
+ if (mode == FindTupleMode.EXCLUSIVE_ERROR_IF_EXISTS) {
+ return ERROR_INDICATOR;
+ } else {
+ return mid;
+ }
}
}
}
- if (mode == FindTupleMode.FTM_EXACT)
- return -1;
+ if (mode == FindTupleMode.EXACT) {
+ return ERROR_INDICATOR;
+ }
- if (matchPolicy == FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY) {
- if (begin > frame.getTupleCount() - 1)
- return -1;
+ if (matchPolicy == FindTupleNoExactMatchPolicy.HIGHER_KEY) {
+ if (begin > frame.getTupleCount() - 1) {
+ return GREATEST_KEY_INDICATOR;
+ }
frameTuple.resetByTupleIndex(frame, begin);
- if (multiCmp.compare(searchKey, frameTuple) < 0)
+ if (multiCmp.compare(searchKey, frameTuple) < 0) {
return begin;
- else
- return -1;
+ } else {
+ return GREATEST_KEY_INDICATOR;
+ }
} else {
- if (end < 0)
- return -1;
+ if (end < 0) {
+ return GREATEST_KEY_INDICATOR;
+ }
frameTuple.resetByTupleIndex(frame, end);
- if (multiCmp.compare(searchKey, frameTuple) > 0)
+ if (multiCmp.compare(searchKey, frameTuple) > 0) {
return end;
- else
- return -1;
+ } else {
+ return GREATEST_KEY_INDICATOR;
+ }
}
}
@Override
public int insertSlot(int tupleIndex, int tupleOff) {
int slotOff = getSlotOff(tupleIndex);
- if (tupleIndex < 0) {
+ if (tupleIndex == GREATEST_KEY_INDICATOR) {
slotOff = getSlotEndOff() - slotSize;
setSlot(slotOff, tupleOff);
return slotOff;
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index 1869a1b..4113e9f 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -22,24 +22,30 @@
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeNotUpdateableException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexUtils;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
@@ -48,235 +54,114 @@
public static final float DEFAULT_FILL_FACTOR = 0.7f;
- private final static int RESTART_OP = Integer.MIN_VALUE;
+ private final static long RESTART_OP = Long.MIN_VALUE;
private final static int MAX_RESTARTS = 10;
-
- // the root page never changes
- private final int rootPage = 1;
-
+ private final static int rootPage = 1;
+
private final IFreePageManager freePageManager;
-
- private boolean created = false;
- private boolean loaded = false;
-
- private final IBufferCache bufferCache;
- private int fileId;
+ private final IBufferCache bufferCache;
private final ITreeIndexFrameFactory interiorFrameFactory;
private final ITreeIndexFrameFactory leafFrameFactory;
+ private final int fieldCount;
private final MultiComparator cmp;
private final ReadWriteLock treeLatch;
private final RangePredicate diskOrderScanPredicate;
+ private int fileId;
- public int rootSplits = 0;
- public int[] splitsByLevel = new int[500];
- public long readLatchesAcquired = 0;
- public long readLatchesReleased = 0;
- public long writeLatchesAcquired = 0;
- public long writeLatchesReleased = 0;
- public long pins = 0;
- public long unpins = 0;
-
- public long treeLatchesAcquired = 0;
- public long treeLatchesReleased = 0;
-
- public byte currentLevel = 0;
-
- public int usefulCompression = 0;
- public int uselessCompression = 0;
-
- public void treeLatchStatus() {
- System.out.println(treeLatch.writeLock().toString());
- }
-
- public String printStats() {
- StringBuilder strBuilder = new StringBuilder();
- strBuilder.append("\n");
- strBuilder.append("ROOTSPLITS: " + rootSplits + "\n");
- strBuilder.append("SPLITS BY LEVEL\n");
- for (int i = 0; i < currentLevel; i++) {
- strBuilder.append(String.format("%3d ", i) + String.format("%8d ", splitsByLevel[i]) + "\n");
- }
- strBuilder.append(String.format("READ LATCHES: %10d %10d\n", readLatchesAcquired, readLatchesReleased));
- strBuilder.append(String.format("WRITE LATCHES: %10d %10d\n", writeLatchesAcquired, writeLatchesReleased));
- strBuilder.append(String.format("PINS: %10d %10d\n", pins, unpins));
- return strBuilder.toString();
- }
-
- public BTree(IBufferCache bufferCache, IFreePageManager freePageManager,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory, MultiComparator cmp) {
+ public BTree(IBufferCache bufferCache, int fieldCount, MultiComparator cmp, IFreePageManager freePageManager,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
this.bufferCache = bufferCache;
- this.interiorFrameFactory = interiorFrameFactory;
- this.leafFrameFactory = leafFrameFactory;
+ this.fieldCount = fieldCount;
this.cmp = cmp;
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
this.freePageManager = freePageManager;
this.treeLatch = new ReentrantReadWriteLock(true);
this.diskOrderScanPredicate = new RangePredicate(true, null, null, true, true, cmp, cmp);
}
@Override
- public void create(int fileId, ITreeIndexFrame leafFrame, ITreeIndexMetaDataFrame metaFrame) throws Exception {
-
- if (created)
- return;
-
+ public void create(int fileId) throws HyracksDataException {
treeLatch.writeLock().lock();
try {
-
- // check if another thread beat us to it
- if (created)
- return;
-
+ ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
+ ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
+ this.fileId = fileId;
freePageManager.init(metaFrame, rootPage);
-
- // initialize root page
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
- pins++;
-
- rootNode.acquireWriteLatch();
- writeLatchesAcquired++;
- try {
- leafFrame.setPage(rootNode);
- leafFrame.initBuffer((byte) 0);
- } finally {
- rootNode.releaseWriteLatch();
- writeLatchesReleased++;
- bufferCache.unpin(rootNode);
- unpins++;
- }
- currentLevel = 0;
-
- created = true;
+ initRoot(leafFrame, true);
} finally {
treeLatch.writeLock().unlock();
}
}
+ @Override
public void open(int fileId) {
- this.fileId = fileId;
+ this.fileId = fileId;
}
+ @Override
public void close() {
fileId = -1;
}
- private void addFreePages(BTreeOpContext ctx) throws Exception {
+ private void addFreePages(BTreeOpContext ctx) throws HyracksDataException {
for (int i = 0; i < ctx.freePages.size(); i++) {
- // root page is special, don't add it to free pages
+ // Root page is special, never add it to free pages.
if (ctx.freePages.get(i) != rootPage) {
freePageManager.addFreePage(ctx.metaFrame, ctx.freePages.get(i));
}
}
ctx.freePages.clear();
}
-
- public void printTree(IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame, ISerializerDeserializer[] fields)
- throws Exception {
- printTree(rootPage, null, false, leafFrame, interiorFrame, fields);
- }
-
- public void printTree(int pageId, ICachedPage parent, boolean unpin, IBTreeLeafFrame leafFrame,
- IBTreeInteriorFrame interiorFrame, ISerializerDeserializer[] fields) throws Exception {
-
- ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pins++;
- node.acquireReadLatch();
- readLatchesAcquired++;
-
- try {
- if (parent != null && unpin == true) {
- parent.releaseReadLatch();
- readLatchesReleased++;
-
- bufferCache.unpin(parent);
- unpins++;
- }
-
- interiorFrame.setPage(node);
- int level = interiorFrame.getLevel();
-
- System.out.format("%1d ", level);
- System.out.format("%3d ", pageId);
- for (int i = 0; i < currentLevel - level; i++)
- System.out.format(" ");
-
- String keyString;
- if (interiorFrame.isLeaf()) {
- leafFrame.setPage(node);
- keyString = leafFrame.printKeys(cmp, fields);
- } else {
- keyString = interiorFrame.printKeys(cmp, fields);
- }
-
- System.out.format(keyString);
- if (!interiorFrame.isLeaf()) {
- ArrayList<Integer> children = ((BTreeNSMInteriorFrame) (interiorFrame)).getChildren(cmp);
-
- for (int i = 0; i < children.size(); i++) {
- printTree(children.get(i), node, i == children.size() - 1, leafFrame, interiorFrame, fields);
- }
- } else {
- node.releaseReadLatch();
- readLatchesReleased++;
-
- bufferCache.unpin(node);
- unpins++;
- }
- } catch (Exception e) {
- node.releaseReadLatch();
- readLatchesReleased++;
-
- bufferCache.unpin(node);
- unpins++;
- e.printStackTrace();
- }
- }
-
- @Override
- public void diskOrderScan(ITreeIndexCursor icursor, ITreeIndexFrame leafFrame, ITreeIndexMetaDataFrame metaFrame,
- IndexOpContext ictx) throws HyracksDataException {
+
+ private void diskOrderScan(ITreeIndexCursor icursor, BTreeOpContext ctx) throws HyracksDataException {
TreeDiskOrderScanCursor cursor = (TreeDiskOrderScanCursor) icursor;
- BTreeOpContext ctx = (BTreeOpContext) ictx;
ctx.reset();
- int currentPageId = rootPage + 1;
- int maxPageId = freePageManager.getMaxPage(metaFrame);
+ int currentPageId = rootPage;
+ int maxPageId = freePageManager.getMaxPage(ctx.metaFrame);
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
page.acquireReadLatch();
- cursor.setBufferCache(bufferCache);
- cursor.setFileId(fileId);
- cursor.setCurrentPageId(currentPageId);
- cursor.setMaxPageId(maxPageId);
- ctx.cursorInitialState.setPage(page);
- cursor.open(ctx.cursorInitialState, diskOrderScanPredicate);
+ try {
+ cursor.setBufferCache(bufferCache);
+ cursor.setFileId(fileId);
+ cursor.setCurrentPageId(currentPageId);
+ cursor.setMaxPageId(maxPageId);
+ ctx.cursorInitialState.setPage(page);
+ cursor.open(ctx.cursorInitialState, diskOrderScanPredicate);
+ } catch (Exception e) {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ throw new HyracksDataException(e);
+ }
}
- public void search(ITreeIndexCursor cursor, RangePredicate pred, BTreeOpContext ctx) throws Exception {
+ private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, BTreeOpContext ctx)
+ throws TreeIndexException, HyracksDataException, PageAllocationException {
ctx.reset();
- ctx.pred = pred;
+ ctx.pred = (RangePredicate) searchPred;
ctx.cursor = cursor;
// simple index scan
- if (ctx.pred.getLowKeyComparator() == null)
+ if (ctx.pred.getLowKeyComparator() == null) {
ctx.pred.setLowKeyComparator(cmp);
- if (ctx.pred.getHighKeyComparator() == null)
+ }
+ if (ctx.pred.getHighKeyComparator() == null) {
ctx.pred.setHighKeyComparator(cmp);
-
- boolean repeatOp = true;
+ }
// we use this loop to deal with possibly multiple operation restarts
// due to ongoing structure modifications during the descent
+ boolean repeatOp = true;
while (repeatOp && ctx.opRestarts < MAX_RESTARTS) {
performOp(rootPage, null, ctx);
-
// if we reach this stage then we need to restart from the (possibly
// new) root
if (!ctx.pageLsns.isEmpty() && ctx.pageLsns.getLast() == RESTART_OP) {
ctx.pageLsns.removeLast(); // pop the restart op indicator
continue;
}
-
repeatOp = false;
}
-
cursor.setBufferCache(bufferCache);
cursor.setFileId(fileId);
}
@@ -286,99 +171,81 @@
for (int i = 0; i < ctx.smPages.size(); i++) {
int pageId = ctx.smPages.get(i);
ICachedPage smPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pins++;
- smPage.acquireWriteLatch(); // TODO: would like to set page dirty
- // without latching
- writeLatchesAcquired++;
+ smPage.acquireWriteLatch();
try {
ctx.interiorFrame.setPage(smPage);
ctx.interiorFrame.setSmFlag(false);
} finally {
smPage.releaseWriteLatch();
- writeLatchesReleased++;
bufferCache.unpin(smPage);
- unpins++;
}
}
if (ctx.smPages.size() > 0) {
treeLatch.writeLock().unlock();
- treeLatchesReleased++;
ctx.smPages.clear();
}
ctx.interiorFrame.setPage(originalPage);
}
- private void createNewRoot(BTreeOpContext ctx) throws Exception {
- rootSplits++; // debug
- splitsByLevel[currentLevel]++;
- currentLevel++;
-
- // make sure the root is always at the same level
+ private void initRoot(ITreeIndexFrame leafFrame, boolean firstInit) throws HyracksDataException {
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), firstInit);
+ rootNode.acquireWriteLatch();
+ try {
+ leafFrame.setPage(rootNode);
+ leafFrame.initBuffer((byte) 0);
+ } finally {
+ rootNode.releaseWriteLatch();
+ bufferCache.unpin(rootNode);
+ }
+ }
+
+ private void createNewRoot(BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ // Make sure the root is always in the same page.
ICachedPage leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, ctx.splitKey.getLeftPage()),
false);
- pins++;
- leftNode.acquireWriteLatch(); // TODO: think about whether latching is
- // really required
- writeLatchesAcquired++;
+ leftNode.acquireWriteLatch();
try {
ICachedPage rightNode = bufferCache.pin(
BufferedFileHandle.getDiskPageId(fileId, ctx.splitKey.getRightPage()), false);
- pins++;
- rightNode.acquireWriteLatch(); // TODO: think about whether latching
- // is really required
- writeLatchesAcquired++;
+ rightNode.acquireWriteLatch();
try {
int newLeftId = freePageManager.getFreePage(ctx.metaFrame);
ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId), true);
- pins++;
- newLeftNode.acquireWriteLatch(); // TODO: think about whether
- // latching is really
- // required
- writeLatchesAcquired++;
+ newLeftNode.acquireWriteLatch();
try {
- // copy left child to new left child
+ // Copy left child to new left child.
System.arraycopy(leftNode.getBuffer().array(), 0, newLeftNode.getBuffer().array(), 0, newLeftNode
.getBuffer().capacity());
ctx.interiorFrame.setPage(newLeftNode);
ctx.interiorFrame.setSmFlag(false);
-
- // change sibling pointer if children are leaves
+ // Change sibling pointer if children are leaves.
ctx.leafFrame.setPage(rightNode);
if (ctx.leafFrame.isLeaf()) {
ctx.leafFrame.setPrevLeaf(newLeftId);
}
-
- // initialize new root (leftNode becomes new root)
+ // Initialize new root (leftNode becomes new root).
ctx.interiorFrame.setPage(leftNode);
ctx.interiorFrame.initBuffer((byte) (ctx.leafFrame.getLevel() + 1));
- ctx.interiorFrame.setSmFlag(true); // will be cleared later
- // in unsetSmPages
+ // Will be cleared later in unsetSmPages.
+ ctx.interiorFrame.setSmFlag(true);
ctx.splitKey.setLeftPage(newLeftId);
- int targetTupleIndex = ctx.interiorFrame.findTupleIndex(ctx.splitKey.getTuple(), cmp);
- ctx.interiorFrame.insert(ctx.splitKey.getTuple(), cmp, targetTupleIndex);
+ int targetTupleIndex = ctx.interiorFrame.findInsertTupleIndex(ctx.splitKey.getTuple());
+ ctx.interiorFrame.insert(ctx.splitKey.getTuple(), targetTupleIndex);
} finally {
newLeftNode.releaseWriteLatch();
- writeLatchesReleased++;
bufferCache.unpin(newLeftNode);
- unpins++;
}
} finally {
rightNode.releaseWriteLatch();
- writeLatchesReleased++;
bufferCache.unpin(rightNode);
- unpins++;
}
} finally {
leftNode.releaseWriteLatch();
- writeLatchesReleased++;
bufferCache.unpin(leftNode);
- unpins++;
}
}
-
- @Override
- public void insert(ITupleReference tuple, IndexOpContext ictx) throws Exception {
- BTreeOpContext ctx = (BTreeOpContext) ictx;
+
+ private void insertUpdateOrDelete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
ctx.reset();
ctx.pred.setLowKeyComparator(cmp);
ctx.pred.setHighKeyComparator(cmp);
@@ -386,409 +253,364 @@
ctx.pred.setHighKey(tuple, true);
ctx.splitKey.reset();
ctx.splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
-
+ // We use this loop to deal with possibly multiple operation restarts
+ // due to ongoing structure modifications during the descent.
boolean repeatOp = true;
- // we use this loop to deal with possibly multiple operation restarts
- // due to ongoing structure modifications during the descent
while (repeatOp && ctx.opRestarts < MAX_RESTARTS) {
performOp(rootPage, null, ctx);
-
- // if we reach this stage then we need to restart from the (possibly
- // new) root
+ // Do we need to restart from the (possibly new) root?
if (!ctx.pageLsns.isEmpty() && ctx.pageLsns.getLast() == RESTART_OP) {
ctx.pageLsns.removeLast(); // pop the restart op indicator
continue;
}
-
- // we split the root, here is the key for a new root
+ // Split key propagated?
if (ctx.splitKey.getBuffer() != null) {
- createNewRoot(ctx);
+ if (ctx.op == IndexOp.DELETE) {
+ // Reset level of root to zero.
+ initRoot(ctx.leafFrame, false);
+ } else {
+ // Insert or update op. Create a new root.
+ createNewRoot(ctx);
+ }
}
-
unsetSmPages(ctx);
-
+ if (ctx.op == IndexOp.DELETE) {
+ addFreePages(ctx);
+ }
repeatOp = false;
}
}
+
+ private void insert(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ insertUpdateOrDelete(tuple, ctx);
+ }
- public long uselessCompressionTime = 0;
-
- private void insertLeaf(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
+ private void update(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ // This call only allows updating of non-key fields.
+ // Updating a tuple's key necessitates deleting the old entry, and inserting the new entry.
+ // The user of the BTree is responsible for dealing with non-key updates (i.e., doing a delete + insert).
+ if (fieldCount == cmp.getKeyFieldCount()) {
+ throw new BTreeNotUpdateableException("Cannot perform updates when the entire tuple forms the key.");
+ }
+ insertUpdateOrDelete(tuple, ctx);
+ }
+
+ private void delete(ITupleReference tuple, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ insertUpdateOrDelete(tuple, ctx);
+ }
+
+ private boolean insertLeaf(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
ctx.leafFrame.setPage(node);
- ctx.leafFrame.setPageTupleFieldCount(cmp.getFieldCount());
-
- int targetTupleIndex = ctx.leafFrame.findTupleIndex(tuple, cmp);
- FrameOpSpaceStatus spaceStatus = ctx.leafFrame.hasSpaceInsert(tuple, cmp);
+ boolean restartOp = false;
+ int targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple);
+ FrameOpSpaceStatus spaceStatus = ctx.leafFrame.hasSpaceInsert(tuple);
switch (spaceStatus) {
-
case SUFFICIENT_CONTIGUOUS_SPACE: {
- // System.out.println("SUFFICIENT_CONTIGUOUS_SPACE");
- ctx.leafFrame.insert(tuple, cmp, targetTupleIndex);
+ ctx.leafFrame.insert(tuple, targetTupleIndex);
ctx.splitKey.reset();
- }
break;
-
+ }
case SUFFICIENT_SPACE: {
- // System.out.println("SUFFICIENT_SPACE");
- boolean slotsChanged = ctx.leafFrame.compact(cmp);
- if (slotsChanged)
- targetTupleIndex = ctx.leafFrame.findTupleIndex(tuple, cmp);
- ctx.leafFrame.insert(tuple, cmp, targetTupleIndex);
+ boolean slotsChanged = ctx.leafFrame.compact();
+ if (slotsChanged) {
+ targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple);
+ }
+ ctx.leafFrame.insert(tuple, targetTupleIndex);
ctx.splitKey.reset();
- }
break;
-
- case INSUFFICIENT_SPACE: {
- // System.out.println("INSUFFICIENT_SPACE");
-
- // try compressing the page first and see if there is space
- // available
- long start = System.currentTimeMillis();
- boolean reCompressed = ctx.leafFrame.compress(cmp);
- long end = System.currentTimeMillis();
- if (reCompressed)
- spaceStatus = ctx.leafFrame.hasSpaceInsert(tuple, cmp);
-
+ }
+ case INSUFFICIENT_SPACE: {
+ // Try compressing the page first and see if there is space available.
+ boolean reCompressed = ctx.leafFrame.compress();
+ if (reCompressed) {
+ // Compression could have changed the target tuple index, find it again.
+ targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple);
+ spaceStatus = ctx.leafFrame.hasSpaceInsert(tuple);
+ }
if (spaceStatus == FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE) {
- ctx.leafFrame.insert(tuple, cmp, targetTupleIndex);
+ ctx.leafFrame.insert(tuple, targetTupleIndex);
ctx.splitKey.reset();
-
- usefulCompression++;
} else {
+ restartOp = performLeafSplit(pageId, tuple, ctx);
+ }
+ break;
+ }
+ }
+ node.releaseWriteLatch();
+ bufferCache.unpin(node);
+ return restartOp;
+ }
+
+ private boolean performLeafSplit(int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
+ // Lock is released in unsetSmPages(), after sm has fully completed.
+ if (!treeLatch.writeLock().tryLock()) {
+ return true;
+ }
+ int rightSiblingPageId = ctx.leafFrame.getNextLeaf();
+ ICachedPage rightSibling = null;
+ if (rightSiblingPageId > 0) {
+ rightSibling = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightSiblingPageId),
+ false);
+ }
+ try {
+ int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId),
+ true);
+ rightNode.acquireWriteLatch();
+ try {
+ IBTreeLeafFrame rightFrame = ctx.createLeafFrame();
+ rightFrame.setPage(rightNode);
+ rightFrame.initBuffer((byte) 0);
+ rightFrame.setMultiComparator(cmp);
+ ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey);
- uselessCompressionTime += (end - start);
- uselessCompression++;
+ ctx.smPages.add(pageId);
+ ctx.smPages.add(rightPageId);
+ ctx.leafFrame.setSmFlag(true);
+ rightFrame.setSmFlag(true);
- // perform split
- splitsByLevel[0]++; // debug
- int rightSiblingPageId = ctx.leafFrame.getNextLeaf();
- ICachedPage rightSibling = null;
- if (rightSiblingPageId > 0) {
- rightSibling = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightSiblingPageId),
- false);
- pins++;
- }
+ rightFrame.setNextLeaf(ctx.leafFrame.getNextLeaf());
+ rightFrame.setPrevLeaf(pageId);
+ ctx.leafFrame.setNextLeaf(rightPageId);
- treeLatch.writeLock().lock(); // lock is released in
- // unsetSmPages(), after sm has
- // fully completed
- treeLatchesAcquired++;
+ // TODO: we just use increasing numbers as pageLsn,
+ // we
+ // should tie this together with the LogManager and
+ // TransactionManager
+ rightFrame.setPageLsn(rightFrame.getPageLsn() + 1);
+ ctx.leafFrame.setPageLsn(ctx.leafFrame.getPageLsn() + 1);
+
+ ctx.splitKey.setPages(pageId, rightPageId);
+
+ if (rightSibling != null) {
+ rightSibling.acquireWriteLatch();
try {
-
- int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId),
- true);
- pins++;
- rightNode.acquireWriteLatch();
- writeLatchesAcquired++;
- try {
- IBTreeLeafFrame rightFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- rightFrame.setPage(rightNode);
- rightFrame.initBuffer((byte) 0);
- rightFrame.setPageTupleFieldCount(cmp.getFieldCount());
-
- int ret = ctx.leafFrame.split(rightFrame, tuple, cmp, ctx.splitKey);
-
- ctx.smPages.add(pageId);
- ctx.smPages.add(rightPageId);
- ctx.leafFrame.setSmFlag(true);
- rightFrame.setSmFlag(true);
-
- rightFrame.setNextLeaf(ctx.leafFrame.getNextLeaf());
- rightFrame.setPrevLeaf(pageId);
- ctx.leafFrame.setNextLeaf(rightPageId);
-
- // TODO: we just use increasing numbers as pageLsn,
- // we
- // should tie this together with the LogManager and
- // TransactionManager
- rightFrame.setPageLsn(rightFrame.getPageLsn() + 1);
- ctx.leafFrame.setPageLsn(ctx.leafFrame.getPageLsn() + 1);
-
- if (ret != 0) {
- ctx.splitKey.reset();
- } else {
- // System.out.print("LEAF SPLITKEY: ");
- // cmp.printKey(splitKey.getData(), 0);
- // System.out.println("");
-
- ctx.splitKey.setPages(pageId, rightPageId);
- }
- if (rightSibling != null) {
- rightSibling.acquireWriteLatch();
- writeLatchesAcquired++;
- try {
- rightFrame.setPage(rightSibling); // reuse
- // rightFrame
- // for
- // modification
- rightFrame.setPrevLeaf(rightPageId);
- } finally {
- rightSibling.releaseWriteLatch();
- writeLatchesReleased++;
- }
- }
- } finally {
- rightNode.releaseWriteLatch();
- writeLatchesReleased++;
- bufferCache.unpin(rightNode);
- unpins++;
- }
- } catch (Exception e) {
- treeLatch.writeLock().unlock();
- treeLatchesReleased++;
- throw e;
+ // Reuse rightFrame for modification.
+ rightFrame.setPage(rightSibling);
+ rightFrame.setPrevLeaf(rightPageId);
} finally {
- if (rightSibling != null) {
- bufferCache.unpin(rightSibling);
- unpins++;
- }
+ rightSibling.releaseWriteLatch();
}
}
+ } finally {
+ rightNode.releaseWriteLatch();
+ bufferCache.unpin(rightNode);
}
- break;
-
+ } catch (Exception e) {
+ treeLatch.writeLock().unlock();
+ throw e;
+ } finally {
+ if (rightSibling != null) {
+ bufferCache.unpin(rightSibling);
+ }
}
-
+ return false;
+ }
+
+ private boolean updateLeaf(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
+ ctx.leafFrame.setPage(node);
+ int oldTupleIndex = ctx.leafFrame.findUpdateTupleIndex(tuple);
+ FrameOpSpaceStatus spaceStatus = ctx.leafFrame.hasSpaceUpdate(tuple, oldTupleIndex);
+ boolean restartOp = false;
+ switch (spaceStatus) {
+ case SUFFICIENT_INPLACE_SPACE: {
+ ctx.leafFrame.update(tuple, oldTupleIndex, true);
+ ctx.splitKey.reset();
+ break;
+ }
+ case SUFFICIENT_CONTIGUOUS_SPACE: {
+ ctx.leafFrame.update(tuple, oldTupleIndex, false);
+ ctx.splitKey.reset();
+ break;
+ }
+ case SUFFICIENT_SPACE: {
+ // Delete the old tuple, compact the frame, and insert the new tuple.
+ ctx.leafFrame.delete(tuple, oldTupleIndex);
+ ctx.leafFrame.compact();
+ int targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple);
+ ctx.leafFrame.insert(tuple, targetTupleIndex);
+ ctx.splitKey.reset();
+ break;
+ }
+ case INSUFFICIENT_SPACE: {
+ // Delete the old tuple, and try compressing the page to make space available.
+ ctx.leafFrame.delete(tuple, oldTupleIndex);
+ ctx.leafFrame.compress();
+ // We need to insert the new tuple, so check if there is space.
+ spaceStatus = ctx.leafFrame.hasSpaceInsert(tuple);
+ if (spaceStatus == FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE) {
+ int targetTupleIndex = ctx.leafFrame.findInsertTupleIndex(tuple);
+ ctx.leafFrame.insert(tuple, targetTupleIndex);
+ ctx.splitKey.reset();
+ } else {
+ restartOp = performLeafSplit(pageId, tuple, ctx);
+ }
+ break;
+ }
+ }
node.releaseWriteLatch();
- writeLatchesReleased++;
bufferCache.unpin(node);
- unpins++;
+ return restartOp;
}
private void insertInterior(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx)
throws Exception {
ctx.interiorFrame.setPage(node);
- ctx.interiorFrame.setPageTupleFieldCount(cmp.getKeyFieldCount());
-
- int targetTupleIndex = ctx.interiorFrame.findTupleIndex(tuple, cmp);
- FrameOpSpaceStatus spaceStatus = ctx.interiorFrame.hasSpaceInsert(tuple, cmp);
+ int targetTupleIndex = ctx.interiorFrame.findInsertTupleIndex(tuple);
+ FrameOpSpaceStatus spaceStatus = ctx.interiorFrame.hasSpaceInsert(tuple);
switch (spaceStatus) {
case INSUFFICIENT_SPACE: {
- splitsByLevel[ctx.interiorFrame.getLevel()]++; // debug
int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
- pins++;
rightNode.acquireWriteLatch();
- writeLatchesAcquired++;
try {
- ITreeIndexFrame rightFrame = interiorFrameFactory.createFrame();
+ IBTreeFrame rightFrame = ctx.createInteriorFrame();
rightFrame.setPage(rightNode);
rightFrame.initBuffer((byte) ctx.interiorFrame.getLevel());
- rightFrame.setPageTupleFieldCount(cmp.getKeyFieldCount());
+ rightFrame.setMultiComparator(cmp);
// instead of creating a new split key, use the existing
// splitKey
- int ret = ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), cmp, ctx.splitKey);
-
+ ctx.interiorFrame.split(rightFrame, ctx.splitKey.getTuple(), ctx.splitKey);
ctx.smPages.add(pageId);
ctx.smPages.add(rightPageId);
ctx.interiorFrame.setSmFlag(true);
rightFrame.setSmFlag(true);
-
// TODO: we just use increasing numbers as pageLsn, we
// should tie this together with the LogManager and
// TransactionManager
rightFrame.setPageLsn(rightFrame.getPageLsn() + 1);
ctx.interiorFrame.setPageLsn(ctx.interiorFrame.getPageLsn() + 1);
- if (ret != 0) {
- ctx.splitKey.reset();
- } else {
- // System.out.print("INTERIOR SPLITKEY: ");
- // cmp.printKey(splitKey.getData(), 0);
- // System.out.println("");
-
- ctx.splitKey.setPages(pageId, rightPageId);
- }
+ ctx.splitKey.setPages(pageId, rightPageId);
} finally {
rightNode.releaseWriteLatch();
- writeLatchesReleased++;
bufferCache.unpin(rightNode);
- unpins++;
}
- }
break;
+ }
case SUFFICIENT_CONTIGUOUS_SPACE: {
- // System.out.println("INSERT INTERIOR: " + pageId);
- ctx.interiorFrame.insert(tuple, cmp, targetTupleIndex);
+ ctx.interiorFrame.insert(tuple, targetTupleIndex);
ctx.splitKey.reset();
- }
break;
+ }
case SUFFICIENT_SPACE: {
- boolean slotsChanged = ctx.interiorFrame.compact(cmp);
- if (slotsChanged)
- targetTupleIndex = ctx.interiorFrame.findTupleIndex(tuple, cmp);
- ctx.interiorFrame.insert(tuple, cmp, targetTupleIndex);
+ boolean slotsChanged = ctx.interiorFrame.compact();
+ if (slotsChanged) {
+ targetTupleIndex = ctx.interiorFrame.findInsertTupleIndex(tuple);
+ }
+ ctx.interiorFrame.insert(tuple, targetTupleIndex);
ctx.splitKey.reset();
- }
break;
-
+ }
}
}
- @Override
- public void delete(ITupleReference tuple, IndexOpContext ictx) throws Exception {
- BTreeOpContext ctx = (BTreeOpContext) ictx;
- ctx.reset();
- ctx.pred.setLowKeyComparator(cmp);
- ctx.pred.setHighKeyComparator(cmp);
- ctx.pred.setLowKey(tuple, true);
- ctx.pred.setHighKey(tuple, true);
- ctx.splitKey.reset();
- ctx.splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
-
- boolean repeatOp = true;
- // we use this loop to deal with possibly multiple operation restarts
- // due to ongoing structure modifications during the descent
- while (repeatOp && ctx.opRestarts < MAX_RESTARTS) {
- performOp(rootPage, null, ctx);
-
- // if we reach this stage then we need to restart from the (possibly
- // new) root
- if (!ctx.pageLsns.isEmpty() && ctx.pageLsns.getLast() == RESTART_OP) {
- ctx.pageLsns.removeLast(); // pop the restart op indicator
- continue;
- }
-
- // tree is empty, reset level to zero
- if (ctx.splitKey.getBuffer() != null) {
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
- pins++;
- rootNode.acquireWriteLatch();
- writeLatchesAcquired++;
- try {
- ctx.leafFrame.setPage(rootNode);
- ctx.leafFrame.initBuffer((byte) 0);
- currentLevel = 0; // debug
- } finally {
- rootNode.releaseWriteLatch();
- writeLatchesReleased++;
- bufferCache.unpin(rootNode);
- unpins++;
- }
- }
-
- unsetSmPages(ctx);
-
- addFreePages(ctx);
-
- repeatOp = false;
- }
- }
-
- // TODO: to avoid latch deadlock, must modify cursor to detect empty leaves
- private void deleteLeaf(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
+ private boolean deleteLeaf(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx) throws Exception {
ctx.leafFrame.setPage(node);
-
- // will this leaf become empty?
- if (ctx.leafFrame.getTupleCount() == 1) {
- IBTreeLeafFrame siblingFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
-
- ICachedPage leftNode = null;
- ICachedPage rightNode = null;
- int nextLeaf = ctx.leafFrame.getNextLeaf();
- int prevLeaf = ctx.leafFrame.getPrevLeaf();
-
- if (prevLeaf > 0)
- leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, prevLeaf), false);
-
- try {
-
- if (nextLeaf > 0)
- rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeaf), false);
-
- try {
- treeLatch.writeLock().lock();
- treeLatchesAcquired++;
-
- try {
- ctx.leafFrame.delete(tuple, cmp, true);
- // to propagate the deletion we only need to make the
- // splitKey != null
- // we can reuse data to identify which key to delete in
- // the parent
- ctx.splitKey.initData(1);
- } catch (Exception e) {
- // don't propagate deletion upwards if deletion at this
- // level fails
- ctx.splitKey.reset();
- throw e;
- }
-
- // TODO: tie together with loggins
- ctx.leafFrame.setPageLsn(ctx.leafFrame.getPageLsn() + 1);
- ctx.leafFrame.setLevel(freePageManager.getFreePageLevelIndicator());
-
- ctx.smPages.add(pageId);
- ctx.leafFrame.setSmFlag(true);
-
- node.releaseWriteLatch();
- writeLatchesReleased++;
- bufferCache.unpin(node);
- unpins++;
-
- if (leftNode != null) {
- leftNode.acquireWriteLatch();
- try {
- siblingFrame.setPage(leftNode);
- siblingFrame.setNextLeaf(nextLeaf);
- siblingFrame.setPageLsn(siblingFrame.getPageLsn() + 1); // TODO:
- // tie
- // together
- // with
- // logging
- } finally {
- leftNode.releaseWriteLatch();
- }
- }
-
- if (rightNode != null) {
- rightNode.acquireWriteLatch();
- try {
- siblingFrame.setPage(rightNode);
- siblingFrame.setPrevLeaf(prevLeaf);
- siblingFrame.setPageLsn(siblingFrame.getPageLsn() + 1); // TODO:
- // tie
- // together
- // with
- // logging
- } finally {
- rightNode.releaseWriteLatch();
- }
- }
-
- // register pageId as a free
- ctx.freePages.add(pageId);
-
- } catch (Exception e) {
- treeLatch.writeLock().unlock();
- treeLatchesReleased++;
- throw e;
- } finally {
- if (rightNode != null) {
- bufferCache.unpin(rightNode);
- }
- }
- } finally {
- if (leftNode != null) {
- bufferCache.unpin(leftNode);
- }
- }
- } else { // leaf will not become empty
- ctx.leafFrame.delete(tuple, cmp, true);
+ int tupleIndex = ctx.leafFrame.findDeleteTupleIndex(tuple);
+
+ // Will this leaf become empty?
+ if (ctx.leafFrame.getTupleCount() > 1) {
+ // Leaf will not become empty.
+ ctx.leafFrame.delete(tuple, tupleIndex);
node.releaseWriteLatch();
- writeLatchesReleased++;
bufferCache.unpin(node);
- unpins++;
+ return false;
}
+
+ // Leaf will become empty.
+ IBTreeLeafFrame siblingFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
+ siblingFrame.setMultiComparator(cmp);
+ ICachedPage leftNode = null;
+ ICachedPage rightNode = null;
+ int nextLeaf = ctx.leafFrame.getNextLeaf();
+ int prevLeaf = ctx.leafFrame.getPrevLeaf();
+ // Try to get the tree latch, if it's already taken, then restart this operation
+ // to avoid latch deadlock.
+ if (!treeLatch.writeLock().tryLock()) {
+ return true;
+ }
+ try {
+ if (prevLeaf > 0) {
+ leftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, prevLeaf), false);
+ }
+ try {
+ if (nextLeaf > 0) {
+ rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeaf), false);
+ }
+ try {
+ try {
+ ctx.leafFrame.delete(tuple, tupleIndex);
+ // To propagate the deletion we only need to make the
+ // splitKey != null.
+ // Reuse data to identify which key to delete in the parent.
+ ctx.splitKey.initData(1);
+ } catch (Exception e) {
+ // Don't propagate deletion.
+ ctx.splitKey.reset();
+ throw e;
+ }
+
+ // TODO: Tie together with logging.
+ ctx.leafFrame.setPageLsn(ctx.leafFrame.getPageLsn() + 1);
+ ctx.leafFrame.setLevel(freePageManager.getFreePageLevelIndicator());
+
+ ctx.smPages.add(pageId);
+ ctx.leafFrame.setSmFlag(true);
+
+ node.releaseWriteLatch();
+ bufferCache.unpin(node);
+
+ if (leftNode != null) {
+ leftNode.acquireWriteLatch();
+ try {
+ siblingFrame.setPage(leftNode);
+ siblingFrame.setNextLeaf(nextLeaf);
+ // TODO: Tie together with logging.
+ siblingFrame.setPageLsn(siblingFrame.getPageLsn() + 1);
+ } finally {
+ leftNode.releaseWriteLatch();
+ }
+ }
+
+ if (rightNode != null) {
+ rightNode.acquireWriteLatch();
+ try {
+ siblingFrame.setPage(rightNode);
+ siblingFrame.setPrevLeaf(prevLeaf);
+ // TODO: Tie together with logging.
+ siblingFrame.setPageLsn(siblingFrame.getPageLsn() + 1);
+ } finally {
+ rightNode.releaseWriteLatch();
+ }
+ }
+ // Register pageId as a free.
+ ctx.freePages.add(pageId);
+ } finally {
+ if (rightNode != null) {
+ bufferCache.unpin(rightNode);
+ }
+ }
+ } finally {
+ if (leftNode != null) {
+ bufferCache.unpin(leftNode);
+ }
+ }
+ } catch (Exception e) {
+ treeLatch.writeLock().unlock();
+ throw e;
+ }
+ return false;
}
private void deleteInterior(ICachedPage node, int pageId, ITupleReference tuple, BTreeOpContext ctx)
throws Exception {
ctx.interiorFrame.setPage(node);
+ int tupleIndex = ctx.interiorFrame.findDeleteTupleIndex(tuple);
+
// this means there is only a child pointer but no key, this case
// propagates the split
if (ctx.interiorFrame.getTupleCount() == 0) {
@@ -806,63 +628,52 @@
ctx.freePages.add(pageId);
} else {
- ctx.interiorFrame.delete(tuple, cmp, false);
- ctx.interiorFrame.setPageLsn(ctx.interiorFrame.getPageLsn() + 1); // TODO:
- // tie
- // together
- // with
- // logging
- ctx.splitKey.reset(); // don't propagate deletion
+ ctx.interiorFrame.delete(tuple, tupleIndex);
+ // TODO: Tie together with logging.
+ ctx.interiorFrame.setPageLsn(ctx.interiorFrame.getPageLsn() + 1);
+ // Don't propagate deletion.
+ ctx.splitKey.reset();
}
}
- private final void acquireLatch(ICachedPage node, IndexOp op, boolean isLeaf) {
- if (isLeaf && (op.equals(IndexOp.INSERT) || op.equals(IndexOp.DELETE))) {
- node.acquireWriteLatch();
- writeLatchesAcquired++;
- } else {
+ private final void acquireLatch(ICachedPage node, BTreeOpContext ctx, boolean isLeaf) {
+ if (!isLeaf || (ctx.op == IndexOp.SEARCH && !ctx.cursor.exclusiveLatchNodes())) {
node.acquireReadLatch();
- readLatchesAcquired++;
+ } else {
+ node.acquireWriteLatch();
}
}
- private final void releaseLatch(ICachedPage node, IndexOp op, boolean isLeaf) {
- if (isLeaf && (op.equals(IndexOp.INSERT) || op.equals(IndexOp.DELETE))) {
- node.releaseWriteLatch();
- writeLatchesReleased++;
- } else {
+ private final void releaseLatch(ICachedPage node, BTreeOpContext ctx, boolean isLeaf) {
+ if (!isLeaf || (ctx.op == IndexOp.SEARCH && !ctx.cursor.exclusiveLatchNodes())) {
node.releaseReadLatch();
- readLatchesReleased++;
+ } else {
+ node.releaseWriteLatch();
}
}
private boolean isConsistent(int pageId, BTreeOpContext ctx) throws Exception {
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pins++;
node.acquireReadLatch();
- readLatchesAcquired++;
ctx.interiorFrame.setPage(node);
boolean isConsistent = false;
try {
isConsistent = ctx.pageLsns.getLast() == ctx.interiorFrame.getPageLsn();
} finally {
node.releaseReadLatch();
- readLatchesReleased++;
bufferCache.unpin(node);
- unpins++;
}
return isConsistent;
}
- private void performOp(int pageId, ICachedPage parent, BTreeOpContext ctx) throws Exception {
+ private void performOp(int pageId, ICachedPage parent, BTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pins++;
-
ctx.interiorFrame.setPage(node);
+
// this check performs an unprotected read in the page
// the following could happen: TODO fill out
boolean unsafeIsLeaf = ctx.interiorFrame.isLeaf();
- acquireLatch(node, ctx.op, unsafeIsLeaf);
+ acquireLatch(node, ctx, unsafeIsLeaf);
boolean smFlag = ctx.interiorFrame.getSmFlag();
// re-check leafness after latching
boolean isLeaf = ctx.interiorFrame.isLeaf();
@@ -870,120 +681,85 @@
// remember trail of pageLsns, to unwind recursion in case of an ongoing
// structure modification
ctx.pageLsns.add(ctx.interiorFrame.getPageLsn());
-
try {
-
// latch coupling, note: parent should never be write latched,
// otherwise something is wrong.
if (parent != null) {
parent.releaseReadLatch();
- readLatchesReleased++;
bufferCache.unpin(parent);
- unpins++;
}
-
if (!isLeaf || smFlag) {
if (!smFlag) {
- // we use this loop to deal with possibly multiple operation
+ // We use this loop to deal with possibly multiple operation
// restarts due to ongoing structure modifications during
- // the descent
+ // the descent.
boolean repeatOp = true;
while (repeatOp && ctx.opRestarts < MAX_RESTARTS) {
- int childPageId = ctx.interiorFrame.getChildPageId(ctx.pred, cmp);
+ int childPageId = ctx.interiorFrame.getChildPageId(ctx.pred);
performOp(childPageId, node, ctx);
if (!ctx.pageLsns.isEmpty() && ctx.pageLsns.getLast() == RESTART_OP) {
- ctx.pageLsns.removeLast(); // pop the restart op
- // indicator
+ // Pop the restart op indicator.
+ ctx.pageLsns.removeLast();
if (isConsistent(pageId, ctx)) {
- node = null; // to avoid unpinning and
- // unlatching node again in
- // recursive call
- continue; // descend the tree again
+ // Don't unpin and unlatch node again in recursive call.
+ node = null;
+ // Descend the tree again.
+ continue;
} else {
- ctx.pageLsns.removeLast(); // pop pageLsn of
- // this page
- // (version seen by this op
- // during descent)
- ctx.pageLsns.add(RESTART_OP); // this node is
- // not
- // consistent,
- // set the
- // restart
- // indicator for
- // upper level
+ // Pop pageLsn of this page (version seen by this op during descent).
+ ctx.pageLsns.removeLast();
+ // This node is not consistent set the restart indicator for upper level.
+ ctx.pageLsns.add(RESTART_OP);
break;
}
}
-
+
switch (ctx.op) {
-
- case INSERT: {
- if (ctx.splitKey.getBuffer() != null) {
- node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pins++;
- node.acquireWriteLatch();
- writeLatchesAcquired++;
- try {
- insertInterior(node, pageId, ctx.splitKey.getTuple(), ctx);
- } finally {
- node.releaseWriteLatch();
- writeLatchesReleased++;
- bufferCache.unpin(node);
- unpins++;
- }
- } else {
- unsetSmPages(ctx);
- }
- }
- break;
-
+ case INSERT:
+ case UPDATE:
case DELETE: {
+ // Is there a propagated split key?
if (ctx.splitKey.getBuffer() != null) {
node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pins++;
node.acquireWriteLatch();
- writeLatchesAcquired++;
try {
- deleteInterior(node, pageId, ctx.pred.getLowKey(), ctx);
+ if (ctx.op == IndexOp.DELETE) {
+ deleteInterior(node, pageId, ctx.pred.getLowKey(), ctx);
+ } else {
+ // Insert or update op. Both can cause split keys to propagate upwards.
+ insertInterior(node, pageId, ctx.splitKey.getTuple(), ctx);
+ }
} finally {
node.releaseWriteLatch();
- writeLatchesReleased++;
bufferCache.unpin(node);
- unpins++;
}
} else {
unsetSmPages(ctx);
}
- }
break;
-
- case SEARCH: {
- // do nothing
}
+ default: {
+ // Do nothing for Search and DiskOrderScan.
break;
-
+ }
}
-
- repeatOp = false; // operation completed
-
+ // Operation completed.
+ repeatOp = false;
} // end while
} else { // smFlag
ctx.opRestarts++;
- System.out.println("ONGOING SM ON PAGE " + pageId + " AT LEVEL " + ctx.interiorFrame.getLevel()
- + ", RESTARTS: " + ctx.opRestarts);
- releaseLatch(node, ctx.op, unsafeIsLeaf);
+ releaseLatch(node, ctx, unsafeIsLeaf);
bufferCache.unpin(node);
- unpins++;
// TODO: this should be an instant duration lock, how to do
// this in java?
// instead we just immediately release the lock. this is
// inefficient but still correct and will not cause
// latch-deadlock
- treeLatch.readLock().lock();
- treeLatch.readLock().unlock();
-
+ treeLatch.writeLock().lock();
+ treeLatch.writeLock().unlock();
+
// unwind recursion and restart operation, find lowest page
// with a pageLsn as seen by this operation during descent
ctx.pageLsns.removeLast(); // pop current page lsn
@@ -992,53 +768,55 @@
ctx.pageLsns.add(RESTART_OP);
}
} else { // isLeaf and !smFlag
- switch (ctx.op) {
+ // We may have to restart an op to avoid latch deadlock.
+ boolean restartOp = false;
+ switch (ctx.op) {
case INSERT: {
- insertLeaf(node, pageId, ctx.pred.getLowKey(), ctx);
- }
+ restartOp = insertLeaf(node, pageId, ctx.pred.getLowKey(), ctx);
break;
-
+ }
+ case UPDATE: {
+ restartOp = updateLeaf(node, pageId, ctx.pred.getLowKey(), ctx);
+ break;
+ }
case DELETE: {
- deleteLeaf(node, pageId, ctx.pred.getLowKey(), ctx);
- }
+ restartOp = deleteLeaf(node, pageId, ctx.pred.getLowKey(), ctx);
break;
-
+ }
case SEARCH: {
ctx.cursorInitialState.setPage(node);
ctx.cursor.open(ctx.cursorInitialState, ctx.pred);
- }
break;
+ }
}
+ if (restartOp) {
+ ctx.pageLsns.removeLast();
+ ctx.pageLsns.add(RESTART_OP);
+ }
}
} catch (TreeIndexException e) {
- // System.out.println("BTREE EXCEPTION");
- // System.out.println(e.getMessage());
- // e.printStackTrace();
- if (!e.getHandled()) {
- releaseLatch(node, ctx.op, unsafeIsLeaf);
+ if (!ctx.exceptionHandled) {
+ releaseLatch(node, ctx, unsafeIsLeaf);
bufferCache.unpin(node);
- unpins++;
- e.setHandled(true);
+ ctx.exceptionHandled = true;
}
throw e;
- } catch (Exception e) { // this could be caused, e.g. by a
- // failure to pin a new node during a split
- System.out.println("ASTERIX EXCEPTION");
- e.printStackTrace();
- releaseLatch(node, ctx.op, unsafeIsLeaf);
+ } catch (PageAllocationException e) {
+ if (!ctx.exceptionHandled) {
+ releaseLatch(node, ctx, unsafeIsLeaf);
+ bufferCache.unpin(node);
+ ctx.exceptionHandled = true;
+ }
+ throw e;
+ } catch (Exception e) {
+ e.printStackTrace();
+ releaseLatch(node, ctx, unsafeIsLeaf);
bufferCache.unpin(node);
- unpins++;
- BTreeException propException = new BTreeException(e);
- propException.setHandled(true); // propagate a BTreeException,
- // indicating that the parent node
- // must not be unlatched and
- // unpinned
- throw propException;
+ BTreeException wrappedException = new BTreeException(e);
+ throw wrappedException;
}
}
- private boolean bulkNewPage = false;
-
public final class BulkLoadContext implements IIndexBulkLoadContext {
public final int slotSize;
public final int leafMaxBytes;
@@ -1053,15 +831,18 @@
private final ITreeIndexTupleWriter tupleWriter;
public BulkLoadContext(float fillFactor, IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+ ITreeIndexMetaDataFrame metaFrame, MultiComparator cmp) throws HyracksDataException, PageAllocationException {
+ leafFrame.setMultiComparator(cmp);
+ interiorFrame.setMultiComparator(cmp);
+
splitKey = new BTreeSplitKey(leafFrame.getTupleWriter().createTupleReference());
tupleWriter = leafFrame.getTupleWriter();
NodeFrontier leafFrontier = new NodeFrontier(leafFrame.createTupleReference());
leafFrontier.pageId = freePageManager.getFreePage(metaFrame);
leafFrontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId),
- bulkNewPage);
+ true);
leafFrontier.page.acquireWriteLatch();
interiorFrame.setPage(leafFrontier.page);
@@ -1081,10 +862,10 @@
nodeFrontiers.add(leafFrontier);
}
- private void addLevel() throws HyracksDataException {
+ private void addLevel() throws HyracksDataException, PageAllocationException {
NodeFrontier frontier = new NodeFrontier(tupleWriter.createTupleReference());
frontier.pageId = freePageManager.getFreePage(metaFrame);
- frontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, frontier.pageId), bulkNewPage);
+ frontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, frontier.pageId), true);
frontier.page.acquireWriteLatch();
frontier.lastTuple.setFieldCount(cmp.getKeyFieldCount());
interiorFrame.setPage(frontier.page);
@@ -1093,7 +874,7 @@
}
}
- private void propagateBulk(BulkLoadContext ctx, int level) throws HyracksDataException {
+ private void propagateBulk(BulkLoadContext ctx, int level) throws HyracksDataException, PageAllocationException {
if (ctx.splitKey.getBuffer() == null)
return;
@@ -1112,8 +893,7 @@
BTreeSplitKey copyKey = ctx.splitKey.duplicate(ctx.leafFrame.getTupleWriter().createTupleReference());
tuple = copyKey.getTuple();
- frontier.lastTuple.resetByTupleOffset(frontier.page.getBuffer(),
- ctx.interiorFrame.getTupleOffset(ctx.interiorFrame.getTupleCount() - 1));
+ frontier.lastTuple.resetByTupleIndex(ctx.interiorFrame, ctx.interiorFrame.getTupleCount() - 1);
int splitKeySize = ctx.tupleWriter.bytesRequired(frontier.lastTuple, 0, cmp.getKeyFieldCount());
ctx.splitKey.initData(splitKeySize);
ctx.tupleWriter
@@ -1121,7 +901,7 @@
ctx.splitKey.getTuple().resetByTupleOffset(ctx.splitKey.getBuffer(), 0);
ctx.splitKey.setLeftPage(frontier.pageId);
- ctx.interiorFrame.deleteGreatest(cmp);
+ ctx.interiorFrame.deleteGreatest();
frontier.page.releaseWriteLatch();
bufferCache.unpin(frontier.page);
@@ -1130,38 +910,30 @@
ctx.splitKey.setRightPage(frontier.pageId);
propagateBulk(ctx, level + 1);
- frontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, frontier.pageId), bulkNewPage);
+ frontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, frontier.pageId), true);
frontier.page.acquireWriteLatch();
ctx.interiorFrame.setPage(frontier.page);
ctx.interiorFrame.initBuffer((byte) level);
}
- ctx.interiorFrame.insertSorted(tuple, cmp);
-
- // debug print
- // ISerializerDeserializer[] btreeSerde = {
- // UTF8StringSerializerDeserializer.INSTANCE,
- // IntegerSerializerDeserializer.INSTANCE };
- // String s = ctx.interiorFrame.printKeys(cmp, btreeSerde);
- // System.out.println(s);
+ ctx.interiorFrame.insertSorted(tuple);
}
// assumes btree has been created and opened
@Override
- public IIndexBulkLoadContext beginBulkLoad(float fillFactor, ITreeIndexFrame leafFrame,
- ITreeIndexFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
-
- if (loaded)
- throw new HyracksDataException("Trying to bulk-load BTree but BTree has already been loaded.");
-
- BulkLoadContext ctx = new BulkLoadContext(fillFactor, (IBTreeLeafFrame) leafFrame,
- (IBTreeInteriorFrame) interiorFrame, metaFrame);
- ctx.nodeFrontiers.get(0).lastTuple.setFieldCount(cmp.getFieldCount());
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws TreeIndexException, HyracksDataException, PageAllocationException {
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame)leafFrameFactory.createFrame();
+ if (!isEmptyTree(leafFrame)) {
+ throw new BTreeException("Trying to Bulk-load a non-empty BTree.");
+ }
+
+ BulkLoadContext ctx = new BulkLoadContext(fillFactor, leafFrame,
+ (IBTreeInteriorFrame)interiorFrameFactory.createFrame(), freePageManager.getMetaDataFrameFactory().createFrame(), cmp);
ctx.splitKey.getTuple().setFieldCount(cmp.getKeyFieldCount());
return ctx;
}
@Override
- public void bulkLoadAddTuple(IIndexBulkLoadContext ictx, ITupleReference tuple) throws HyracksDataException {
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException, PageAllocationException {
BulkLoadContext ctx = (BulkLoadContext) ictx;
NodeFrontier leafFrontier = ctx.nodeFrontiers.get(0);
IBTreeLeafFrame leafFrame = ctx.leafFrame;
@@ -1171,7 +943,7 @@
// try to free space by compression
if (spaceUsed + spaceNeeded > ctx.leafMaxBytes) {
- leafFrame.compress(cmp);
+ leafFrame.compress();
spaceUsed = leafFrame.getBuffer().capacity() - leafFrame.getTotalFreeSpace();
}
@@ -1194,7 +966,7 @@
propagateBulk(ctx, 1);
leafFrontier.page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, leafFrontier.pageId),
- bulkNewPage);
+ true);
leafFrontier.page.acquireWriteLatch();
leafFrame.setPage(leafFrontier.page);
leafFrame.initBuffer((byte) 0);
@@ -1202,21 +974,14 @@
}
leafFrame.setPage(leafFrontier.page);
- leafFrame.insertSorted(tuple, cmp);
-
- // debug print
- // ISerializerDeserializer[] btreeSerde = {
- // UTF8StringSerializerDeserializer.INSTANCE,
- // IntegerSerializerDeserializer.INSTANCE };
- // String s = leafFrame.printKeys(cmp, btreeSerde);
- // System.out.println(s);
+ leafFrame.insertSorted(tuple);
}
@Override
public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
// copy root
BulkLoadContext ctx = (BulkLoadContext) ictx;
- ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), bulkNewPage);
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
rootNode.acquireWriteLatch();
NodeFrontier lastNodeFrontier = ctx.nodeFrontiers.get(ctx.nodeFrontiers.size() - 1);
IBTreeInteriorFrame interiorFrame = ctx.interiorFrame;
@@ -1241,18 +1006,13 @@
bufferCache.unpin(ctx.nodeFrontiers.get(i).page);
}
}
- // debug
- currentLevel = (byte) ctx.nodeFrontiers.size();
-
- loaded = true;
}
- @Override
- public BTreeOpContext createOpContext(IndexOp op, ITreeIndexFrame leafFrame, ITreeIndexFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame) {
- return new BTreeOpContext(op, (IBTreeLeafFrame) leafFrame, (IBTreeInteriorFrame) interiorFrame, metaFrame, 6);
+ private BTreeOpContext createOpContext() {
+ return new BTreeOpContext(leafFrameFactory, interiorFrameFactory, freePageManager.getMetaDataFrameFactory()
+ .createFrame(), cmp);
}
-
+
public ITreeIndexFrameFactory getInteriorFrameFactory() {
return interiorFrameFactory;
}
@@ -1271,20 +1031,141 @@
public int getRootPageId() {
return rootPage;
- }
-
- @Override
- public void update(ITupleReference tuple, IndexOpContext ictx) throws Exception {
- throw new Exception("BTree Update not implemented.");
- }
+ }
@Override
public int getFieldCount() {
- return cmp.getFieldCount();
+ return fieldCount;
}
@Override
public IndexType getIndexType() {
return IndexType.BTREE;
}
-}
\ No newline at end of file
+
+ public byte getTreeHeight(IBTreeLeafFrame leafFrame) throws HyracksDataException {
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
+ rootNode.acquireReadLatch();
+ try {
+ leafFrame.setPage(rootNode);
+ return leafFrame.getLevel();
+ } finally {
+ rootNode.releaseReadLatch();
+ bufferCache.unpin(rootNode);
+ }
+ }
+
+ public boolean isEmptyTree(IBTreeLeafFrame leafFrame) throws HyracksDataException {
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), false);
+ rootNode.acquireReadLatch();
+ try {
+ leafFrame.setPage(rootNode);
+ if (leafFrame.getLevel() == 0 && leafFrame.getTupleCount() == 0) {
+ return true;
+ } else {
+ return false;
+ }
+ } finally {
+ rootNode.releaseReadLatch();
+ bufferCache.unpin(rootNode);
+ }
+ }
+
+ @SuppressWarnings("rawtypes")
+ public String printTree(IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame, ISerializerDeserializer[] keySerdes)
+ throws Exception {
+ byte treeHeight = getTreeHeight(leafFrame);
+ StringBuilder strBuilder = new StringBuilder();
+ printTree(rootPage, null, false, leafFrame, interiorFrame, treeHeight, keySerdes, strBuilder);
+ return strBuilder.toString();
+ }
+
+ @SuppressWarnings("rawtypes")
+ public void printTree(int pageId, ICachedPage parent, boolean unpin, IBTreeLeafFrame leafFrame,
+ IBTreeInteriorFrame interiorFrame, byte treeHeight, ISerializerDeserializer[] keySerdes, StringBuilder strBuilder) throws Exception {
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ node.acquireReadLatch();
+ try {
+ if (parent != null && unpin == true) {
+ parent.releaseReadLatch();
+ bufferCache.unpin(parent);
+ }
+ interiorFrame.setPage(node);
+ int level = interiorFrame.getLevel();
+ strBuilder.append(String.format("%1d ", level));
+ strBuilder.append(String.format("%3d ", pageId) + ": ");
+ for (int i = 0; i < treeHeight - level; i++) {
+ strBuilder.append(" ");
+ }
+
+ String keyString;
+ if (interiorFrame.isLeaf()) {
+ leafFrame.setPage(node);
+ keyString = TreeIndexUtils.printFrameTuples(leafFrame, keySerdes);
+ } else {
+ keyString = TreeIndexUtils.printFrameTuples(interiorFrame, keySerdes);
+ }
+
+ strBuilder.append(keyString + "\n");
+ if (!interiorFrame.isLeaf()) {
+ ArrayList<Integer> children = ((BTreeNSMInteriorFrame) (interiorFrame)).getChildren(cmp);
+ for (int i = 0; i < children.size(); i++) {
+ printTree(children.get(i), node, i == children.size() - 1, leafFrame, interiorFrame, treeHeight, keySerdes, strBuilder);
+ }
+ } else {
+ node.releaseReadLatch();
+ bufferCache.unpin(node);
+ }
+ } catch (Exception e) {
+ node.releaseReadLatch();
+ bufferCache.unpin(node);
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public ITreeIndexAccessor createAccessor() {
+ return new BTreeAccessor(this);
+ }
+
+ private class BTreeAccessor implements ITreeIndexAccessor {
+ private BTree btree;
+ private BTreeOpContext ctx;
+
+ public BTreeAccessor(BTree btree) {
+ this.btree = btree;
+ this.ctx = btree.createOpContext();
+ }
+
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ ctx.reset(IndexOp.INSERT);
+ btree.insert(tuple, ctx);
+ }
+
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ ctx.reset(IndexOp.UPDATE);
+ btree.update(tuple, ctx);
+ }
+
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ ctx.reset(IndexOp.DELETE);
+ btree.delete(tuple, ctx);
+ }
+
+ @Override
+ public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
+ TreeIndexException, PageAllocationException {
+ ctx.reset(IndexOp.SEARCH);
+ btree.search(cursor, searchPred, ctx);
+ }
+
+ @Override
+ public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
+ ctx.reset(IndexOp.DISKORDERSCAN);
+ btree.diskOrderScan(cursor, ctx);
+ }
+ }
+}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeException.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeException.java
deleted file mode 100644
index 8019bcb..0000000
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeException.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree.impls;
-
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-
-public class BTreeException extends TreeIndexException {
-
- private static final long serialVersionUID = 1L;
- private boolean handled = false;
-
- public BTreeException(Exception e) {
- super(e);
- }
-
- public BTreeException(String message) {
- super(message);
- }
-
- public void setHandled(boolean handled) {
- this.handled = handled;
- }
-
- public boolean getHandled() {
- return handled;
- }
-}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java
index 6d65b14..07c645c 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeOpContext.java
@@ -17,45 +17,47 @@
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IntArrayList;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.LongArrayList;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-public final class BTreeOpContext implements IndexOpContext {
- public final IndexOp op;
- public final IBTreeLeafFrame leafFrame;
- public final IBTreeInteriorFrame interiorFrame;
- public final ITreeIndexMetaDataFrame metaFrame;
+public class BTreeOpContext implements IIndexOpContext {
+ private final int INIT_ARRAYLIST_SIZE = 6;
+ protected ITreeIndexFrameFactory leafFrameFactory;
+ protected ITreeIndexFrameFactory interiorFrameFactory;
+ public IBTreeLeafFrame leafFrame;
+ public IBTreeInteriorFrame interiorFrame;
+ public ITreeIndexMetaDataFrame metaFrame;
+ public IndexOp op;
public ITreeIndexCursor cursor;
public BTreeCursorInitialState cursorInitialState;
public RangePredicate pred;
- public final BTreeSplitKey splitKey;
+ public BTreeSplitKey splitKey;
+ public LongArrayList pageLsns;
+ public IntArrayList smPages;
+ public IntArrayList freePages;
public int opRestarts = 0;
- public final IntArrayList pageLsns; // used like a stack
- public final IntArrayList smPages;
- public final IntArrayList freePages;
-
- public BTreeOpContext(IndexOp op, IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame, int treeHeightHint) {
- this.op = op;
- this.leafFrame = leafFrame;
- this.interiorFrame = interiorFrame;
- this.metaFrame = metaFrame;
-
- pageLsns = new IntArrayList(treeHeightHint, treeHeightHint);
- if (op != IndexOp.SEARCH && op != IndexOp.DISKORDERSCAN) {
- smPages = new IntArrayList(treeHeightHint, treeHeightHint);
- freePages = new IntArrayList(treeHeightHint, treeHeightHint);
- pred = new RangePredicate(true, null, null, true, true, null, null);
- splitKey = new BTreeSplitKey(leafFrame.getTupleWriter().createTupleReference());
- } else {
- smPages = null;
- freePages = null;
- splitKey = null;
- cursorInitialState = new BTreeCursorInitialState(null);
+ public boolean exceptionHandled;
+
+ public BTreeOpContext(ITreeIndexFrameFactory leafFrameFactory, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexMetaDataFrame metaFrame, MultiComparator cmp) {
+ this.leafFrameFactory = leafFrameFactory;
+ this.leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
+ if (leafFrame != null) {
+ leafFrame.setMultiComparator(cmp);
}
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
+ if (interiorFrame != null) {
+ interiorFrame.setMultiComparator(cmp);
+ }
+ this.metaFrame = metaFrame;
+ this.pageLsns = new LongArrayList(INIT_ARRAYLIST_SIZE, INIT_ARRAYLIST_SIZE);
}
public void reset() {
@@ -66,5 +68,39 @@
if (smPages != null)
smPages.clear();
opRestarts = 0;
+ exceptionHandled = false;
+ }
+
+ @Override
+ public void reset(IndexOp newOp) {
+ if (newOp == IndexOp.SEARCH || newOp == IndexOp.DISKORDERSCAN) {
+ if (cursorInitialState == null) {
+ cursorInitialState = new BTreeCursorInitialState(null);
+ }
+ } else {
+ // Insert, update or delete operation.
+ if (smPages == null) {
+ smPages = new IntArrayList(INIT_ARRAYLIST_SIZE, INIT_ARRAYLIST_SIZE);
+ }
+ if (freePages == null) {
+ freePages = new IntArrayList(INIT_ARRAYLIST_SIZE, INIT_ARRAYLIST_SIZE);
+ }
+ if (pred == null) {
+ pred = new RangePredicate(true, null, null, true, true, null, null);
+ }
+ if (splitKey == null) {
+ splitKey = new BTreeSplitKey(leafFrame.getTupleWriter().createTupleReference());
+ }
+ }
+ op = newOp;
+ exceptionHandled = false;
+ }
+
+ public IBTreeLeafFrame createLeafFrame() {
+ return (IBTreeLeafFrame) leafFrameFactory.createFrame();
+ }
+
+ public IBTreeInteriorFrame createInteriorFrame() {
+ return (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
index 4c7503d..8122314 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTreeRangeSearchCursor.java
@@ -33,7 +33,6 @@
private int fileId = -1;
private ICachedPage page = null;
- private IBTreeLeafFrame frame = null;
private IBufferCache bufferCache = null;
private int tupleIndex = 0;
@@ -46,7 +45,9 @@
private FindTupleNoExactMatchPolicy lowKeyFtp;
private FindTupleNoExactMatchPolicy highKeyFtp;
- private ITreeIndexTupleReference frameTuple;
+ private final IBTreeLeafFrame frame;
+ private final ITreeIndexTupleReference frameTuple;
+ private final boolean exclusiveLatchNodes;
private RangePredicate pred;
private MultiComparator lowKeyCmp;
@@ -54,20 +55,25 @@
private ITupleReference lowKey;
private ITupleReference highKey;
- public BTreeRangeSearchCursor(IBTreeLeafFrame frame) {
+ public BTreeRangeSearchCursor(IBTreeLeafFrame frame, boolean exclusiveLatchNodes) {
this.frame = frame;
this.frameTuple = frame.createTupleReference();
+ this.exclusiveLatchNodes = exclusiveLatchNodes;
}
@Override
public void close() throws Exception {
if (page != null) {
- page.releaseReadLatch();
+ if (exclusiveLatchNodes) {
+ page.releaseWriteLatch();
+ } else {
+ page.releaseReadLatch();
+ }
bufferCache.unpin(page);
}
tupleIndex = 0;
- page = null;
- pred = null;
+ page = null;
+ pred = null;
}
public ITupleReference getTuple() {
@@ -81,18 +87,20 @@
private void fetchNextLeafPage(int nextLeafPage) throws HyracksDataException {
ICachedPage nextLeaf = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, nextLeafPage), false);
- nextLeaf.acquireReadLatch();
-
- page.releaseReadLatch();
+ if (exclusiveLatchNodes) {
+ nextLeaf.acquireWriteLatch();
+ page.releaseWriteLatch();
+ } else {
+ nextLeaf.acquireReadLatch();
+ page.releaseReadLatch();
+ }
bufferCache.unpin(page);
-
page = nextLeaf;
- frame.setPage(page);
+ frame.setPage(page);
}
@Override
public boolean hasNext() throws Exception {
-
if (pred.isForward()) {
if (tupleIndex >= frame.getTupleCount()) {
int nextLeafPage = frame.getNextLeaf();
@@ -141,43 +149,46 @@
tupleIndex += tupleIndexInc;
}
- private int getLowKeyIndex() {
- int index;
- if (lowKey == null)
- index = 0;
- else {
- index = frame.findTupleIndex(lowKey, frameTuple, lowKeyCmp, lowKeyFtm, lowKeyFtp);
- if (pred.lowKeyInclusive) {
- index++;
- } else {
- if (index < 0)
- index = frame.getTupleCount();
+ private int getLowKeyIndex() throws HyracksDataException {
+ if (lowKey == null) {
+ return 0;
+ }
+ int index = frame.findTupleIndex(lowKey, frameTuple, lowKeyCmp, lowKeyFtm, lowKeyFtp);
+ if (pred.lowKeyInclusive) {
+ index++;
+ } else {
+ if (index < 0) {
+ index = frame.getTupleCount();
}
}
return index;
}
- private int getHighKeyIndex() {
- int index;
- if (highKey == null)
- index = frame.getTupleCount() - 1;
- else {
- index = frame.findTupleIndex(highKey, frameTuple, highKeyCmp, highKeyFtm, highKeyFtp);
- if (pred.highKeyInclusive) {
- if (index < 0)
- index = frame.getTupleCount() - 1;
- else
- index--;
+ private int getHighKeyIndex() throws HyracksDataException {
+ if (highKey == null) {
+ return frame.getTupleCount() - 1;
+ }
+ int index = frame.findTupleIndex(highKey, frameTuple, highKeyCmp, highKeyFtm, highKeyFtp);
+ if (pred.highKeyInclusive) {
+ if (index < 0) {
+ index = frame.getTupleCount() - 1;
+ }
+ else {
+ index--;
}
}
return index;
}
@Override
- public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws Exception {
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
// in case open is called multiple times without closing
if (page != null) {
- page.releaseReadLatch();
+ if (exclusiveLatchNodes) {
+ page.releaseWriteLatch();
+ } else {
+ page.releaseReadLatch();
+ }
bufferCache.unpin(page);
}
@@ -191,23 +202,19 @@
lowKey = pred.getLowKey();
highKey = pred.getHighKey();
- // field count must be identical for lowKeyCmp and highKeyCmp (key count
- // may be different)
- frameTuple.setFieldCount(lowKeyCmp.getFieldCount());
-
// init
- lowKeyFtm = FindTupleMode.FTM_EXCLUSIVE;
+ lowKeyFtm = FindTupleMode.EXCLUSIVE;
if (pred.lowKeyInclusive) {
- lowKeyFtp = FindTupleNoExactMatchPolicy.FTP_LOWER_KEY;
+ lowKeyFtp = FindTupleNoExactMatchPolicy.LOWER_KEY;
} else {
- lowKeyFtp = FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY;
+ lowKeyFtp = FindTupleNoExactMatchPolicy.HIGHER_KEY;
}
- highKeyFtm = FindTupleMode.FTM_EXCLUSIVE;
+ highKeyFtm = FindTupleMode.EXCLUSIVE;
if (pred.highKeyInclusive) {
- highKeyFtp = FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY;
+ highKeyFtp = FindTupleNoExactMatchPolicy.HIGHER_KEY;
} else {
- highKeyFtp = FindTupleNoExactMatchPolicy.FTP_LOWER_KEY;
+ highKeyFtp = FindTupleNoExactMatchPolicy.LOWER_KEY;
}
if (pred.isForward()) {
@@ -224,10 +231,10 @@
@Override
public void reset() {
try {
- close();
- } catch (Exception e) {
- e.printStackTrace();
- }
+ close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
}
@Override
@@ -239,4 +246,9 @@
public void setFileId(int fileId) {
this.fileId = fileId;
}
+
+ @Override
+ public boolean exclusiveLatchNodes() {
+ return exclusiveLatchNodes;
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixPrefixTupleReference.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixPrefixTupleReference.java
index 621281a..80ac173 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixPrefixTupleReference.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixPrefixTupleReference.java
@@ -15,14 +15,14 @@
package edu.uci.ics.hyracks.storage.am.btree.impls;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleReference;
public class FieldPrefixPrefixTupleReference extends TypeAwareTupleReference {
- public FieldPrefixPrefixTupleReference(ITypeTrait[] typeTraits) {
+ public FieldPrefixPrefixTupleReference(ITypeTraits[] typeTraits) {
super(typeTraits);
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixSlotManager.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixSlotManager.java
index 6f98c8c..6525bb9 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixSlotManager.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixSlotManager.java
@@ -20,6 +20,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IPrefixSlotManager;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
@@ -30,11 +31,13 @@
private static final int slotSize = 4;
public static final int TUPLE_UNCOMPRESSED = 0xFF;
public static final int MAX_PREFIX_SLOTS = 0xFE;
- public static final int GREATEST_SLOT = 0x00FFFFFF;
+ public static final int GREATEST_KEY_INDICATOR = 0x00FFFFFF;
+ public static final int ERROR_INDICATOR = 0x00FFFFFE;
private ByteBuffer buf;
private BTreeFieldPrefixNSMLeafFrame frame;
-
+ private MultiComparator cmp;
+
public int decodeFirstSlotField(int slot) {
return (slot & 0xFF000000) >>> 24;
}
@@ -48,7 +51,7 @@
}
// returns prefix slot number, or TUPLE_UNCOMPRESSED of no match was found
- public int findPrefix(ITupleReference tuple, ITreeIndexTupleReference framePrefixTuple, MultiComparator multiCmp) {
+ public int findPrefix(ITupleReference tuple, ITreeIndexTupleReference framePrefixTuple) {
int prefixMid;
int prefixBegin = 0;
int prefixEnd = frame.getPrefixTupleCount() - 1;
@@ -57,10 +60,10 @@
while (prefixBegin <= prefixEnd) {
prefixMid = (prefixBegin + prefixEnd) / 2;
framePrefixTuple.resetByTupleIndex(frame, prefixMid);
- int cmp = multiCmp.fieldRangeCompare(tuple, framePrefixTuple, 0, framePrefixTuple.getFieldCount());
- if (cmp < 0)
+ int cmpVal = cmp.fieldRangeCompare(tuple, framePrefixTuple, 0, framePrefixTuple.getFieldCount());
+ if (cmpVal < 0)
prefixEnd = prefixMid - 1;
- else if (cmp > 0)
+ else if (cmpVal > 0)
prefixBegin = prefixMid + 1;
else
return prefixMid;
@@ -75,9 +78,7 @@
ITreeIndexTupleReference framePrefixTuple, MultiComparator multiCmp, FindTupleMode mode,
FindTupleNoExactMatchPolicy matchPolicy) {
if (frame.getTupleCount() <= 0)
- encodeSlotFields(TUPLE_UNCOMPRESSED, GREATEST_SLOT);
-
- frameTuple.setFieldCount(multiCmp.getFieldCount());
+ encodeSlotFields(TUPLE_UNCOMPRESSED, GREATEST_KEY_INDICATOR);
int prefixMid;
int prefixBegin = 0;
@@ -101,8 +102,8 @@
prefixBegin = prefixMid + 1;
tuplePrefixSlotNumUbound = prefixMid + 1;
} else {
- if (mode == FindTupleMode.FTM_EXCLUSIVE) {
- if (matchPolicy == FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY)
+ if (mode == FindTupleMode.EXCLUSIVE) {
+ if (matchPolicy == FindTupleNoExactMatchPolicy.HIGHER_KEY)
prefixBegin = prefixMid + 1;
else
prefixEnd = prefixMid - 1;
@@ -153,13 +154,17 @@
else if (cmp > 0)
tupleBegin = tupleMid + 1;
else {
- if (mode == FindTupleMode.FTM_EXCLUSIVE) {
- if (matchPolicy == FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY)
+ if (mode == FindTupleMode.EXCLUSIVE) {
+ if (matchPolicy == FindTupleNoExactMatchPolicy.HIGHER_KEY)
tupleBegin = tupleMid + 1;
else
tupleEnd = tupleMid - 1;
} else {
- return encodeSlotFields(prefixMatch, tupleMid);
+ if (mode == FindTupleMode.EXCLUSIVE_ERROR_IF_EXISTS) {
+ return encodeSlotFields(prefixMatch, ERROR_INDICATOR);
+ } else {
+ return encodeSlotFields(prefixMatch, tupleMid);
+ }
}
}
}
@@ -167,27 +172,27 @@
// System.out.println("RECS: " + recBegin + " " + recMid + " " +
// recEnd);
- if (mode == FindTupleMode.FTM_EXACT)
- return encodeSlotFields(prefixMatch, GREATEST_SLOT);
+ if (mode == FindTupleMode.EXACT)
+ return encodeSlotFields(prefixMatch, ERROR_INDICATOR);
// do final comparison to determine whether the search key is greater
// than all keys or in between some existing keys
- if (matchPolicy == FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY) {
+ if (matchPolicy == FindTupleNoExactMatchPolicy.HIGHER_KEY) {
if (tupleBegin > frame.getTupleCount() - 1)
- return encodeSlotFields(prefixMatch, GREATEST_SLOT);
+ return encodeSlotFields(prefixMatch, GREATEST_KEY_INDICATOR);
frameTuple.resetByTupleIndex(frame, tupleBegin);
if (multiCmp.compare(searchKey, frameTuple) < 0)
return encodeSlotFields(prefixMatch, tupleBegin);
else
- return encodeSlotFields(prefixMatch, GREATEST_SLOT);
+ return encodeSlotFields(prefixMatch, GREATEST_KEY_INDICATOR);
} else {
if (tupleEnd < 0)
- return encodeSlotFields(prefixMatch, GREATEST_SLOT);
+ return encodeSlotFields(prefixMatch, GREATEST_KEY_INDICATOR);
frameTuple.resetByTupleIndex(frame, tupleEnd);
if (multiCmp.compare(searchKey, frameTuple) > 0)
return encodeSlotFields(prefixMatch, tupleEnd);
else
- return encodeSlotFields(prefixMatch, GREATEST_SLOT);
+ return encodeSlotFields(prefixMatch, GREATEST_KEY_INDICATOR);
}
}
@@ -217,7 +222,10 @@
public int insertSlot(int slot, int tupleOff) {
int slotNum = decodeSecondSlotField(slot);
- if (slotNum == GREATEST_SLOT) {
+ if (slotNum == ERROR_INDICATOR) {
+ System.out.println("WOW BIG PROBLEM!");
+ }
+ if (slotNum == GREATEST_KEY_INDICATOR) {
int slotOff = getTupleSlotEndOff() - slotSize;
int newSlot = encodeSlotFields(decodeFirstSlotField(slot), tupleOff);
setSlot(slotOff, newSlot);
@@ -235,11 +243,6 @@
}
}
- public void setFrame(BTreeFieldPrefixNSMLeafFrame frame) {
- this.frame = frame;
- this.buf = frame.getBuffer();
- }
-
public int getPrefixSlotOff(int tupleIndex) {
return getPrefixSlotStartOff() - tupleIndex * slotSize;
}
@@ -251,4 +254,51 @@
public void setPrefixSlot(int tupleIndex, int slot) {
buf.putInt(getPrefixSlotOff(tupleIndex), slot);
}
+
+ @Override
+ public int getGreatestKeyIndicator() {
+ return GREATEST_KEY_INDICATOR;
+ }
+
+ @Override
+ public int getErrorIndicator() {
+ return ERROR_INDICATOR;
+ }
+
+ @Override
+ public void setFrame(ITreeIndexFrame frame) {
+ this.frame = (BTreeFieldPrefixNSMLeafFrame)frame;
+ this.buf = frame.getBuffer();
+ }
+
+ @Override
+ public int findTupleIndex(ITupleReference searchKey,
+ ITreeIndexTupleReference frameTuple, MultiComparator multiCmp,
+ FindTupleMode mode, FindTupleNoExactMatchPolicy matchPolicy) {
+ throw new UnsupportedOperationException("Not implemented.");
+ }
+
+ @Override
+ public int getSlotStartOff() {
+ throw new UnsupportedOperationException("Not implemented.");
+ }
+
+ @Override
+ public int getSlotEndOff() {
+ throw new UnsupportedOperationException("Not implemented.");
+ }
+
+ @Override
+ public int getTupleOff(int slotOff) {
+ throw new UnsupportedOperationException("Not implemented.");
+ }
+
+ @Override
+ public int getSlotOff(int tupleIndex) {
+ throw new UnsupportedOperationException("Not implemented.");
+ }
+
+ public void setMultiComparator(MultiComparator cmp) {
+ this.cmp = cmp;
+ }
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java
index 9a655fc..9644a3e 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java
@@ -17,6 +17,7 @@
public FieldPrefixTupleReference(ITreeIndexTupleReference helperTuple) {
this.helperTuple = helperTuple;
+ this.fieldCount = helperTuple.getFieldCount();
}
@Override
@@ -46,7 +47,7 @@
@Override
public void setFieldCount(int fieldStartIndex, int fieldCount) {
- // not implemented
+ throw new UnsupportedOperationException("Not supported.");
}
@Override
@@ -88,6 +89,28 @@
// unsupported operation
@Override
public void resetByTupleOffset(ByteBuffer buf, int tupleStartOffset) {
- frame = null;
+ throw new UnsupportedOperationException("Resetting this type of frame by offset is not supported.");
+ }
+
+ @Override
+ public int getTupleSize() {
+ return getSuffixTupleSize() + getPrefixTupleSize();
+ }
+
+ public int getSuffixTupleSize() {
+ helperTuple.setFieldCount(numPrefixFields, fieldCount - numPrefixFields);
+ helperTuple.resetByTupleOffset(frame.getBuffer(), suffixTupleStartOff);
+ return helperTuple.getTupleSize();
+ }
+
+ public int getPrefixTupleSize() {
+ if (numPrefixFields == 0) return 0;
+ helperTuple.setFieldCount(numPrefixFields);
+ helperTuple.resetByTupleOffset(frame.getBuffer(), prefixTupleStartOff);
+ return helperTuple.getTupleSize();
+ }
+
+ public int getNumPrefixFields() {
+ return numPrefixFields;
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
new file mode 100644
index 0000000..f6310af
--- /dev/null
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeUtils.java
@@ -0,0 +1,61 @@
+package edu.uci.ics.hyracks.storage.am.btree.util;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class BTreeUtils {
+ public static BTree createBTree(IBufferCache bufferCache, int btreeFileId, ITypeTraits[] typeTraits, IBinaryComparator[] cmps, BTreeLeafFrameType leafType) throws BTreeException {
+ MultiComparator cmp = new MultiComparator(cmps);
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ ITreeIndexFrameFactory leafFrameFactory = getLeafFrameFactory(tupleWriterFactory, leafType);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ BTree btree = new BTree(bufferCache, typeTraits.length, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ return btree;
+ }
+
+ public static MultiComparator getSearchMultiComparator(MultiComparator btreeCmp, ITupleReference searchKey) {
+ if (searchKey == null) {
+ return btreeCmp;
+ }
+ if (btreeCmp.getKeyFieldCount() == searchKey.getFieldCount()) {
+ return btreeCmp;
+ }
+ IBinaryComparator[] cmps = new IBinaryComparator[searchKey.getFieldCount()];
+ for (int i = 0; i < searchKey.getFieldCount(); i++) {
+ cmps[i] = btreeCmp.getComparators()[i];
+ }
+ return new MultiComparator(cmps);
+ }
+
+ public static ITreeIndexFrameFactory getLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory, BTreeLeafFrameType leafType) throws BTreeException {
+ switch(leafType) {
+ case REGULAR_NSM: {
+ return new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ }
+ case FIELD_PREFIX_COMPRESSED_NSM: {
+ return new BTreeFieldPrefixNSMLeafFrameFactory(tupleWriterFactory);
+ }
+ default: {
+ throw new BTreeException("Unknown BTreeLeafFrameType: " + leafType.toString());
+ }
+ }
+ }
+}
diff --git a/hyracks-storage-am-common/.classpath b/hyracks-storage-am-common/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-storage-am-common/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-storage-am-common/.project b/hyracks-storage-am-common/.project
deleted file mode 100644
index ec47f6b..0000000
--- a/hyracks-storage-am-common/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-storage-am-common</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-storage-am-common/.settings/org.eclipse.jdt.core.prefs b/hyracks-storage-am-common/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index b37b3bb..0000000
--- a/hyracks-storage-am-common/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Thu Jul 07 12:23:56 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-storage-am-common/.settings/org.maven.ide.eclipse.prefs b/hyracks-storage-am-common/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 34ccda1..0000000
--- a/hyracks-storage-am-common/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Mon Aug 15 10:50:16 PDT 2011
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-storage-am-common/pom.xml b/hyracks-storage-am-common/pom.xml
index 783faed..9f0e53a 100644
--- a/hyracks-storage-am-common/pom.xml
+++ b/hyracks-storage-am-common/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,28 +27,28 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ICursorInitialState.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ICursorInitialState.java
index 7c80be1..60e8ba9 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ICursorInitialState.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ICursorInitialState.java
@@ -19,6 +19,5 @@
public interface ICursorInitialState {
public ICachedPage getPage();
-
public void setPage(ICachedPage page);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
index 045ff9d..a7901c8 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IFreePageManager.java
@@ -4,7 +4,7 @@
public interface IFreePageManager {
public int getFreePage(ITreeIndexMetaDataFrame metaFrame)
- throws HyracksDataException;
+ throws HyracksDataException, PageAllocationException;
public void addFreePage(ITreeIndexMetaDataFrame metaFrame, int freePage)
throws HyracksDataException;
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexOpContext.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexOpContext.java
new file mode 100644
index 0000000..7153f78
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/IIndexOpContext.java
@@ -0,0 +1,8 @@
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+
+public interface IIndexOpContext {
+ void reset();
+ void reset(IndexOp newOp);
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ISlotManager.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ISlotManager.java
index a6102ab..2619493 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ISlotManager.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ISlotManager.java
@@ -21,12 +21,16 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
public interface ISlotManager {
- public void setFrame(ITreeIndexFrame frame);
-
public int findTupleIndex(ITupleReference searchKey,
ITreeIndexTupleReference frameTuple, MultiComparator multiCmp,
FindTupleMode mode, FindTupleNoExactMatchPolicy matchPolicy);
+ public int getGreatestKeyIndicator();
+
+ public int getErrorIndicator();
+
+ public void setFrame(ITreeIndexFrame frame);
+
public int insertSlot(int tupleIndex, int tupleOff);
public int getSlotStartOff();
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java
index d075285..46d22d1 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndex.java
@@ -1,61 +1,113 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.common.api;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOpContext;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
-public interface ITreeIndex {
- // init:
+/**
+ * Interface describing the operations of tree-based index structures. Indexes
+ * implementing this interface can easily reuse the tree index operators for
+ * dataflow. We assume that indexes store tuples with a fixed number of fields.
+ * Users must perform operations on an ITreeIndex via an ITreeIndexAccessor.
+ */
+public interface ITreeIndex extends IIndex {
- public void create(int indexFileId, ITreeIndexFrame leafFrame,
- ITreeIndexMetaDataFrame metaFrame) throws Exception;
+ /**
+ * Creates an index accessor for performing operations on this index.
+ * (insert/delete/update/search/diskorderscan). An ITreeIndexAccessor is not
+ * thread safe, but different ITreeIndexAccessors can concurrently operate
+ * on the same ITreeIndex
+ *
+ * @returns ITreeIndexAccessor A tree index accessor for this tree.
+ */
+ public ITreeIndexAccessor createAccessor();
- public void open(int indexFileId);
+ /**
+ * Prepares the index for bulk loading, returning a bulk load context. The
+ * index must be empty for bulk loading to be possible.
+ *
+ * @param fillFactor
+ * Desired fill factor in [0, 1.0].
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws TreeIndexException
+ * If the tree is not empty.
+ * @throws PageAllocationException
+ * @returns A new context for bulk loading, required for appending tuples.
+ */
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor)
+ throws TreeIndexException, HyracksDataException,
+ PageAllocationException;
- // operations:
+ /**
+ * Append a tuple to the index in the context of a bulk load.
+ *
+ * @param tuple
+ * Tuple to be inserted.
+ * @param ictx
+ * Existing bulk load context.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws PageAllocationException
+ */
+ public void bulkLoadAddTuple(ITupleReference tuple,
+ IIndexBulkLoadContext ictx) throws HyracksDataException,
+ PageAllocationException;
- public void insert(ITupleReference tuple, IndexOpContext ictx)
- throws Exception;
-
- public void update(ITupleReference tuple, IndexOpContext ictx)
- throws Exception;
-
- public void delete(ITupleReference tuple, IndexOpContext ictx)
- throws Exception;
-
- public IndexOpContext createOpContext(IndexOp op,
- ITreeIndexFrame leafFrame, ITreeIndexFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame);
-
- // bulk loading:
-
- public IIndexBulkLoadContext beginBulkLoad(float fillFactor,
- ITreeIndexFrame leafFrame, ITreeIndexFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException;
-
- public void bulkLoadAddTuple(IIndexBulkLoadContext ictx,
- ITupleReference tuple) throws HyracksDataException;
-
+ /**
+ * Finalize the bulk loading operation in the given context.
+ *
+ * @param ictx
+ * Existing bulk load context to be finalized.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws PageAllocationException
+ */
public void endBulkLoad(IIndexBulkLoadContext ictx)
- throws HyracksDataException;
+ throws HyracksDataException, PageAllocationException;
- // search:
- public void diskOrderScan(ITreeIndexCursor icursor,
- ITreeIndexFrame leafFrame, ITreeIndexMetaDataFrame metaFrame,
- IndexOpContext ictx) throws HyracksDataException;
-
- // utility:
-
- public IFreePageManager getFreePageManager();
-
- public int getRootPageId();
-
+ /**
+ * @return The index's leaf frame factory.
+ */
public ITreeIndexFrameFactory getLeafFrameFactory();
+ /**
+ * @return The index's interior frame factory.
+ */
public ITreeIndexFrameFactory getInteriorFrameFactory();
+ /**
+ * @return The index's free page manager.
+ */
+ public IFreePageManager getFreePageManager();
+
+ /**
+ * @return The number of fields tuples of this index have.
+ */
public int getFieldCount();
+ /**
+ * @return The current root page id of this index.
+ */
+ public int getRootPageId();
+
+ /**
+ * @return An enum of the concrete type of this index.
+ */
public IndexType getIndexType();
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java
new file mode 100644
index 0000000..1e679b2
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexAccessor.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Client handle for performing operations
+ * (insert/delete/update/search/diskorderscan) on an ITreeIndex. An
+ * ITreeIndexAccessor is not thread safe, but different ITreeIndexAccessors can
+ * concurrently operate on the same ITreeIndex (i.e., the ITreeIndex must allow
+ * concurrent operations).
+ */
+public interface ITreeIndexAccessor {
+ /**
+ * Inserts the given tuple.
+ *
+ * @param tuple
+ * Tuple to be inserted.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws TreeIndexException
+ * If an index-specific constraint is violated, e.g., the key
+ * already exists.
+ * @throws PageAllocationException
+ */
+ public void insert(ITupleReference tuple) throws HyracksDataException,
+ TreeIndexException, PageAllocationException;
+
+ /**
+ * Updates the tuple in the index matching the given tuple with the new
+ * contents in the given tuple.
+ *
+ * @param tuple
+ * Tuple whose match in the index is to be update with the given
+ * tuples contents.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws TreeIndexException
+ * If there is no matching tuple in the index.
+ * @throws PageAllocationException
+ */
+ public void update(ITupleReference tuple) throws HyracksDataException,
+ TreeIndexException, PageAllocationException;
+
+ /**
+ * Deletes the tuple in the index matching the given tuple.
+ *
+ * @param tuple
+ * Tuple to be deleted.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws TreeIndexException
+ * If there is no matching tuple in the index.
+ * @throws PageAllocationException
+ */
+ public void delete(ITupleReference tuple) throws HyracksDataException,
+ TreeIndexException, PageAllocationException;
+
+ /**
+ * Open the given cursor for an index search using the given predicate as
+ * search condition.
+ *
+ * @param icursor
+ * Cursor over the index entries satisfying searchPred.
+ * @param searchPred
+ * Search condition.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ * @throws TreeIndexException
+ * @throws PageAllocationException
+ */
+ public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred)
+ throws HyracksDataException, TreeIndexException, PageAllocationException;
+
+ /**
+ * Open the given cursor for a disk-order scan, positioning the cursor to
+ * the first leaf tuple.
+ *
+ * @param icursor
+ * Cursor to be opened for disk-order scanning.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ */
+ public void diskOrderScan(ITreeIndexCursor cursor)
+ throws HyracksDataException;
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
index 56dfd47..d3ce386 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexCursor.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.storage.am.common.api;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
@@ -27,7 +28,7 @@
public void next() throws Exception;
public void open(ICursorInitialState initialState,
- ISearchPredicate searchPred) throws Exception;
+ ISearchPredicate searchPred) throws HyracksDataException;
public ICachedPage getPage();
@@ -38,4 +39,7 @@
public void setFileId(int fileId);
public ITupleReference getTuple();
+
+ // For allowing updates.
+ public boolean exclusiveLatchNodes();
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrame.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrame.java
index db458ec..c33a8d8 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrame.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrame.java
@@ -17,94 +17,76 @@
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
public interface ITreeIndexFrame {
- public void setPage(ICachedPage page);
-
- public ICachedPage getPage();
-
- public ByteBuffer getBuffer();
-
- public int findTupleIndex(ITupleReference tuple, MultiComparator cmp)
- throws Exception;
-
- public void insert(ITupleReference tuple, MultiComparator cmp,
- int tupleIndex) throws Exception;
-
- public void update(int rid, ITupleReference tuple) throws Exception;
-
- public void delete(ITupleReference tuple, MultiComparator cmp,
- boolean exactDelete) throws Exception;
-
- // returns true if slots were modified, false otherwise
- public boolean compact(MultiComparator cmp);
-
- public boolean compress(MultiComparator cmp) throws HyracksDataException;
public void initBuffer(byte level);
+
+ public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple);
+
+ public void insert(ITupleReference tuple, int tupleIndex);
+
+ public FrameOpSpaceStatus hasSpaceUpdate(ITupleReference newTuple, int oldTupleIndex);
+
+ public void update(ITupleReference newTuple, int oldTupleIndex, boolean inPlace);
+
+ public void delete(ITupleReference tuple, int tupleIndex);
- public int getTupleCount();
+ // returns true if slots were modified, false otherwise
+ public boolean compact();
- // assumption: page must be write-latched at this point
- public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple,
- MultiComparator cmp);
+ // returns true if compressed.
+ public boolean compress() throws HyracksDataException;
- public FrameOpSpaceStatus hasSpaceUpdate(int rid, ITupleReference tuple,
- MultiComparator cmp);
+ public int getTupleCount();
- public int getTupleOffset(int slotNum);
+ public int getTupleOffset(int slotNum);
- public int getTotalFreeSpace();
+ public int getTotalFreeSpace();
- public void setPageLsn(int pageLsn);
+ public void setPageLsn(long pageLsn);
- public int getPageLsn();
+ public long getPageLsn();
- // for debugging
- public void printHeader();
+ public void setPage(ICachedPage page);
- public String printKeys(MultiComparator cmp,
- ISerializerDeserializer[] fields) throws HyracksDataException;
+ public ICachedPage getPage();
- // TODO; what if tuples more than half-page size?
- public int split(ITreeIndexFrame rightFrame, ITupleReference tuple,
- MultiComparator cmp, ISplitKey splitKey) throws Exception;
+ public ByteBuffer getBuffer();
+
+ // for debugging
+ public String printHeader();
- public ISlotManager getSlotManager();
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException;
- // ATTENTION: in b-tree operations it may not always be possible to
- // determine whether an ICachedPage is a leaf or interior node
- // a compatible interior and leaf implementation MUST return identical
- // values when given the same ByteBuffer for the functions below
- public boolean isLeaf();
+ public ISlotManager getSlotManager();
- public boolean isInterior();
+ // ATTENTION: in b-tree operations it may not always be possible to
+ // determine whether an ICachedPage is a leaf or interior node
+ // a compatible interior and leaf implementation MUST return identical
+ // values when given the same ByteBuffer for the functions below
+ public boolean isLeaf();
- public byte getLevel();
+ public boolean isInterior();
- public void setLevel(byte level);
+ public byte getLevel();
- public boolean getSmFlag(); // structure modification flag
+ public void setLevel(byte level);
- public void setSmFlag(boolean smFlag);
+ public int getSlotSize();
- public int getSlotSize();
+ // for debugging
+ public int getFreeSpaceOff();
- // TODO: should be removed after new tuple format
- public void setPageTupleFieldCount(int fieldCount);
+ public void setFreeSpaceOff(int freeSpace);
- // for debugging
- public int getFreeSpaceOff();
+ public ITreeIndexTupleWriter getTupleWriter();
- public void setFreeSpaceOff(int freeSpace);
-
- public ITreeIndexTupleWriter getTupleWriter();
-
- public int getPageHeaderSize();
+ public int getPageHeaderSize();
+
+ public ITreeIndexTupleReference createTupleReference();
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrameCompressor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrameCompressor.java
new file mode 100644
index 0000000..75ee598
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrameCompressor.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public interface ITreeIndexFrameCompressor {
+ public boolean compress(ITreeIndexFrame frame, MultiComparator cmp) throws Exception;
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
index 9ec69d9..32c77be 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrameFactory.java
@@ -3,5 +3,6 @@
import java.io.Serializable;
public interface ITreeIndexFrameFactory extends Serializable {
- public ITreeIndexFrame createFrame();
+ public ITreeIndexFrame createFrame();
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory();
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java
index 4d81e5e6..17519ae 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexMetaDataFrame.java
@@ -18,7 +18,7 @@
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
public interface ITreeIndexMetaDataFrame {
- public void initBuffer(int level);
+ public void initBuffer(byte level);
public void setPage(ICachedPage page);
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleReference.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleReference.java
index 8b845ac..b989dd9 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleReference.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleReference.java
@@ -20,11 +20,13 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
public interface ITreeIndexTupleReference extends ITupleReference {
- public void setFieldCount(int fieldCount);
+ public void setFieldCount(int fieldCount);
- public void setFieldCount(int fieldStartIndex, int fieldCount);
+ public void setFieldCount(int fieldStartIndex, int fieldCount);
- public void resetByTupleOffset(ByteBuffer buf, int tupleStartOffset);
+ public void resetByTupleOffset(ByteBuffer buf, int tupleStartOffset);
- public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex);
+ public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex);
+
+ public int getTupleSize();
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java
index 6cd12fb..f0bb7aa 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexTupleWriter.java
@@ -20,20 +20,21 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
public interface ITreeIndexTupleWriter {
- public int writeTuple(ITupleReference tuple, ByteBuffer targetBuf,
- int targetOff);
+ public int writeTuple(ITupleReference tuple, ByteBuffer targetBuf, int targetOff);
+
+ public int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff);
- public int bytesRequired(ITupleReference tuple);
+ public int bytesRequired(ITupleReference tuple);
- public int writeTupleFields(ITupleReference tuple, int startField,
- int numFields, ByteBuffer targetBuf, int targetOff);
+ // TODO: change to byte[] as well.
+ public int writeTupleFields(ITupleReference tuple, int startField, int numFields, ByteBuffer targetBuf,
+ int targetOff);
- public int bytesRequired(ITupleReference tuple, int startField,
- int numFields);
+ public int bytesRequired(ITupleReference tuple, int startField, int numFields);
- // return a tuplereference instance that can read the tuple written by this
- // writer
- // the main idea is that the format of the written tuple may not be the same
- // as the format written by this writer
- public ITreeIndexTupleReference createTupleReference();
+ // return a tuplereference instance that can read the tuple written by this
+ // writer
+ // the main idea is that the format of the written tuple may not be the same
+ // as the format written by this writer
+ public ITreeIndexTupleReference createTupleReference();
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleUpdater.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleUpdater.java
new file mode 100644
index 0000000..e201cc3
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleUpdater.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+
+/**
+ * Interface for updating a tuple. Warning: By convention, this interface
+ * assumes that the modifications do not change the size of the tuple, and that
+ * it does not change keys (e.g., BTree keys). This interface is used to
+ * implement update scans.
+ *
+ */
+public interface ITupleUpdater {
+ public void updateTuple(ITupleReference tuple);
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleUpdaterFactory.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleUpdaterFactory.java
new file mode 100644
index 0000000..ee20b6c
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITupleUpdaterFactory.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+import java.io.Serializable;
+
+public interface ITupleUpdaterFactory extends Serializable {
+ public ITupleUpdater createTupleUpdater();
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java
new file mode 100644
index 0000000..e6eec66
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/PageAllocationException.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.api;
+
+public class PageAllocationException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ public PageAllocationException(Throwable cause) {
+ super(cause);
+ }
+
+ public PageAllocationException(String message) {
+ super(message);
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/TreeIndexException.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/TreeIndexException.java
index 48acb06..4cf596f 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/TreeIndexException.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/TreeIndexException.java
@@ -20,7 +20,7 @@
private static final long serialVersionUID = 1L;
private boolean handled = false;
- public TreeIndexException(Exception e) {
+ public TreeIndexException(Exception e) {
super(e);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/data/PointablePrimitiveValueProviderFactory.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/data/PointablePrimitiveValueProviderFactory.java
new file mode 100644
index 0000000..c159ba5
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/data/PointablePrimitiveValueProviderFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.common.data;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.api.INumeric;
+import edu.uci.ics.hyracks.data.std.api.IPointable;
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+
+public class PointablePrimitiveValueProviderFactory implements IPrimitiveValueProviderFactory {
+ private static final long serialVersionUID = 1L;
+
+ private final IPointableFactory pf;
+
+ public PointablePrimitiveValueProviderFactory(IPointableFactory pf) {
+ this.pf = pf;
+ }
+
+ @Override
+ public IPrimitiveValueProvider createPrimitiveValueProvider() {
+ final IPointable p = pf.createPointable();
+ ITypeTraits traits = pf.getTypeTraits();
+ assert traits.isFixedLength();
+ final int length = traits.getFixedLength();
+ return new IPrimitiveValueProvider() {
+ @Override
+ public double getValue(byte[] bytes, int offset) {
+ p.set(bytes, offset, length);
+ return ((INumeric) p).doubleValue();
+ }
+ };
+ }
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/AbstractTreeIndexOperatorDescriptor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/AbstractTreeIndexOperatorDescriptor.java
index d7c7ab9..ff4d078 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/AbstractTreeIndexOperatorDescriptor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/AbstractTreeIndexOperatorDescriptor.java
@@ -16,63 +16,58 @@
package edu.uci.ics.hyracks.storage.am.common.dataflow;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public abstract class AbstractTreeIndexOperatorDescriptor extends
AbstractSingleActivityOperatorDescriptor implements
- ITreeIndexOperatorDescriptorHelper {
+ ITreeIndexOperatorDescriptor {
private static final long serialVersionUID = 1L;
protected final IFileSplitProvider fileSplitProvider;
protected final IBinaryComparatorFactory[] comparatorFactories;
- protected final IPrimitiveValueProviderFactory[] valueProviderFactories;
protected final ITreeIndexFrameFactory interiorFrameFactory;
protected final ITreeIndexFrameFactory leafFrameFactory;
protected final IStorageManagerInterface storageManager;
- protected final IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider;
+ protected final IIndexRegistryProvider<IIndex> indexRegistryProvider;
- protected final ITypeTrait[] typeTraits;
-
- protected final ITreeIndexOpHelperFactory opHelperFactory;
+ protected final ITypeTraits[] typeTraits;
+ protected final IIndexDataflowHelperFactory dataflowHelperFactory;
public AbstractTreeIndexOperatorDescriptor(JobSpecification spec,
int inputArity, int outputArity, RecordDescriptor recDesc,
IStorageManagerInterface storageManager,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
+ IIndexRegistryProvider<IIndex> indexRegistryProvider,
IFileSplitProvider fileSplitProvider,
ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] typeTraits,
+ ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] comparatorFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories,
- ITreeIndexOpHelperFactory opHelperFactory) {
+ IIndexDataflowHelperFactory dataflowHelperFactory) {
super(spec, inputArity, outputArity);
this.fileSplitProvider = fileSplitProvider;
this.storageManager = storageManager;
- this.treeIndexRegistryProvider = treeIndexRegistryProvider;
+ this.indexRegistryProvider = indexRegistryProvider;
this.interiorFrameFactory = interiorFrameFactory;
this.leafFrameFactory = leafFrameFactory;
this.typeTraits = typeTraits;
this.comparatorFactories = comparatorFactories;
- this.valueProviderFactories = valueProviderFactories;
- this.opHelperFactory = opHelperFactory;
- if (outputArity > 0)
+ this.dataflowHelperFactory = dataflowHelperFactory;
+ if (outputArity > 0) {
recordDescriptors[0] = recDesc;
+ }
}
@Override
- public IFileSplitProvider getTreeIndexFileSplitProvider() {
+ public IFileSplitProvider getFileSplitProvider() {
return fileSplitProvider;
}
@@ -82,16 +77,11 @@
}
@Override
- public ITypeTrait[] getTreeIndexTypeTraits() {
+ public ITypeTraits[] getTreeIndexTypeTraits() {
return typeTraits;
}
@Override
- public IPrimitiveValueProviderFactory[] getTreeIndexValueProviderFactories() {
- return valueProviderFactories;
- }
-
- @Override
public ITreeIndexFrameFactory getTreeIndexInteriorFactory() {
return interiorFrameFactory;
}
@@ -107,8 +97,8 @@
}
@Override
- public IIndexRegistryProvider<ITreeIndex> getTreeIndexRegistryProvider() {
- return treeIndexRegistryProvider;
+ public IIndexRegistryProvider<IIndex> getIndexRegistryProvider() {
+ return indexRegistryProvider;
}
@Override
@@ -117,7 +107,7 @@
}
@Override
- public ITreeIndexOpHelperFactory getTreeIndexOpHelperFactory() {
- return opHelperFactory;
+ public IIndexDataflowHelperFactory getIndexDataflowHelperFactory() {
+ return dataflowHelperFactory;
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
new file mode 100644
index 0000000..38b275b
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndex.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Interface describing the operations common to all indexes.
+ */
+public interface IIndex {
+ /**
+ * Initializes the persistent state of an index, e.g., the root page,
+ * and metadata pages.
+ *
+ * @param indexFileId
+ * The file id to use for this index.
+ * @throws HyracksDataException
+ * If the BufferCache throws while un/pinning or un/latching.
+ */
+ public void create(int indexFileId) throws HyracksDataException;
+
+ /**
+ * Opens the index backed by the given file id.
+ *
+ * @param indexFileId
+ * The file id backing this index.
+ */
+ public void open(int indexFileId);
+
+ /**
+ * Closes the index.
+ */
+ public void close();
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java
new file mode 100644
index 0000000..a42cf5f
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexDataflowHelperFactory.java
@@ -0,0 +1,25 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import java.io.Serializable;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+
+public interface IIndexDataflowHelperFactory extends Serializable {
+ public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc,
+ final IHyracksTaskContext ctx, int partition, boolean createIfNotExists);
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexOperatorDescriptor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexOperatorDescriptor.java
new file mode 100644
index 0000000..edf2bba
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IIndexOperatorDescriptor.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import edu.uci.ics.hyracks.api.dataflow.IActivity;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+
+public interface IIndexOperatorDescriptor extends IActivity {
+ public IFileSplitProvider getFileSplitProvider();
+
+ public IStorageManagerInterface getStorageManager();
+
+ public IIndexRegistryProvider<IIndex> getIndexRegistryProvider();
+
+ public RecordDescriptor getRecordDescriptor();
+
+ public IIndexDataflowHelperFactory getIndexDataflowHelperFactory();
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOpHelperFactory.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOpHelperFactory.java
deleted file mode 100644
index 1f734f4..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOpHelperFactory.java
+++ /dev/null
@@ -1,12 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.common.dataflow;
-
-import java.io.Serializable;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-
-public interface ITreeIndexOpHelperFactory extends Serializable {
- public TreeIndexOpHelper createTreeIndexOpHelper(
- ITreeIndexOperatorDescriptorHelper opDesc,
- final IHyracksTaskContext ctx, int partition,
- IndexHelperOpenMode mode);
-}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptor.java
new file mode 100644
index 0000000..e415cbb
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptor.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+
+public interface ITreeIndexOperatorDescriptor extends IIndexOperatorDescriptor {
+ public IBinaryComparatorFactory[] getTreeIndexComparatorFactories();
+
+ public ITreeIndexFrameFactory getTreeIndexInteriorFactory();
+
+ public ITreeIndexFrameFactory getTreeIndexLeafFactory();
+
+ public ITypeTraits[] getTreeIndexTypeTraits();
+
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptorHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptorHelper.java
deleted file mode 100644
index 79ecb37..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/ITreeIndexOperatorDescriptorHelper.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.common.dataflow;
-
-import edu.uci.ics.hyracks.api.dataflow.IActivity;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-
-public interface ITreeIndexOperatorDescriptorHelper extends IActivity {
- public IFileSplitProvider getTreeIndexFileSplitProvider();
-
- public IBinaryComparatorFactory[] getTreeIndexComparatorFactories();
-
- public IPrimitiveValueProviderFactory[] getTreeIndexValueProviderFactories();
-
- public ITypeTrait[] getTreeIndexTypeTraits();
-
- public ITreeIndexFrameFactory getTreeIndexInteriorFactory();
-
- public ITreeIndexFrameFactory getTreeIndexLeafFactory();
-
- public IStorageManagerInterface getStorageManager();
-
- public IIndexRegistryProvider<ITreeIndex> getTreeIndexRegistryProvider();
-
- public RecordDescriptor getRecordDescriptor();
-
- public ITreeIndexOpHelperFactory getTreeIndexOpHelperFactory();
-}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
new file mode 100644
index 0000000..f0adb0e
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexDataflowHelper.java
@@ -0,0 +1,114 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+
+public abstract class IndexDataflowHelper {
+ protected IIndex index;
+ protected int indexFileId = -1;
+ protected int partition;
+
+ protected final IIndexOperatorDescriptor opDesc;
+ protected final IHyracksTaskContext ctx;
+ protected final boolean createIfNotExists;
+
+ public IndexDataflowHelper(IIndexOperatorDescriptor opDesc, final IHyracksTaskContext ctx,
+ int partition, boolean createIfNotExists) {
+ this.opDesc = opDesc;
+ this.ctx = ctx;
+ this.partition = partition;
+ this.createIfNotExists = createIfNotExists;
+ }
+
+ public void init() throws HyracksDataException {
+ IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
+ IFileMapProvider fileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
+
+ FileReference f = getFilereference();
+ int fileId = -1;
+ synchronized (fileMapProvider) {
+ boolean fileIsMapped = fileMapProvider.isMapped(f);
+ if (!fileIsMapped) {
+ bufferCache.createFile(f);
+ }
+ fileId = fileMapProvider.lookupFileId(f);
+ try {
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // Revert state of buffer cache since file failed to open.
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId);
+ }
+ throw e;
+ }
+ }
+
+ // Only set indexFileId member when openFile() succeeds,
+ // otherwise deinit() will try to close the file that failed to open
+ indexFileId = fileId;
+ IndexRegistry<IIndex> indexRegistry = opDesc.getIndexRegistryProvider().getRegistry(ctx);
+ // Create new index instance and register it.
+ synchronized (indexRegistry) {
+ // Check if the index has already been registered.
+ index = indexRegistry.get(indexFileId);
+ if (index != null) {
+ return;
+ }
+ index = createIndexInstance();
+ if (createIfNotExists) {
+ index.create(indexFileId);
+ }
+ index.open(indexFileId);
+ indexRegistry.register(indexFileId, index);
+ }
+ }
+
+ public abstract IIndex createIndexInstance() throws HyracksDataException;
+
+ public FileReference getFilereference() {
+ IFileSplitProvider fileSplitProvider = opDesc.getFileSplitProvider();
+ return fileSplitProvider.getFileSplits()[partition].getLocalFile();
+ }
+
+ public void deinit() throws HyracksDataException {
+ if (indexFileId != -1) {
+ IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
+ bufferCache.closeFile(indexFileId);
+ }
+ }
+
+ public IIndex getIndex() {
+ return index;
+ }
+
+ public IHyracksTaskContext getHyracksTaskContext() {
+ return ctx;
+ }
+
+ public IIndexOperatorDescriptor getOperatorDescriptor() {
+ return opDesc;
+ }
+
+ public int getIndexFileId() {
+ return indexFileId;
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexHelperOpenMode.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexHelperOpenMode.java
deleted file mode 100644
index 0b27a0e..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexHelperOpenMode.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.common.dataflow;
-
-public enum IndexHelperOpenMode {
- OPEN, CREATE, ENLIST
-}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexRegistry.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexRegistry.java
index df372f4c..5f3b0b3 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexRegistry.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/IndexRegistry.java
@@ -16,26 +16,15 @@
package edu.uci.ics.hyracks.storage.am.common.dataflow;
import java.util.HashMap;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReentrantLock;
public class IndexRegistry<IndexType> {
private HashMap<Integer, IndexType> map = new HashMap<Integer, IndexType>();
- private Lock registryLock = new ReentrantLock();
public IndexType get(int fileId) {
return map.get(fileId);
}
- public void lock() {
- registryLock.lock();
- }
-
- public void unlock() {
- registryLock.unlock();
- }
-
public void register(int fileId, IndexType index) {
map.put(fileId, index);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
index 5076c56..af30105 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorDescriptor.java
@@ -19,47 +19,34 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-public class TreeIndexBulkLoadOperatorDescriptor extends
- AbstractTreeIndexOperatorDescriptor {
+public class TreeIndexBulkLoadOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1L;
- private final int[] fieldPermutation;
- private final float fillFactor;
+ private final int[] fieldPermutation;
+ private final float fillFactor;
- public TreeIndexBulkLoadOperatorDescriptor(JobSpecification spec,
- IStorageManagerInterface storageManager,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
- IFileSplitProvider fileSplitProvider,
- ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] typeTraits,
- IBinaryComparatorFactory[] comparatorFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories,
- int[] fieldPermutation, float fillFactor,
- ITreeIndexOpHelperFactory opHelperFactory) {
- super(spec, 1, 0, null, storageManager, treeIndexRegistryProvider,
- fileSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, comparatorFactories, valueProviderFactories,
- opHelperFactory);
- this.fieldPermutation = fieldPermutation;
- this.fillFactor = fillFactor;
- }
+ public TreeIndexBulkLoadOperatorDescriptor(JobSpecification spec, IStorageManagerInterface storageManager,
+ IIndexRegistryProvider<IIndex> indexRegistryProvider, IFileSplitProvider fileSplitProvider,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] comparatorFactories, int[] fieldPermutation,
+ float fillFactor, IIndexDataflowHelperFactory dataflowHelperFactory) {
+ super(spec, 1, 0, null, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
+ leafFrameFactory, typeTraits, comparatorFactories, dataflowHelperFactory);
+ this.fieldPermutation = fieldPermutation;
+ this.fillFactor = fillFactor;
+ }
- @Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IOperatorEnvironment env,
- IRecordDescriptorProvider recordDescProvider, int partition,
- int nPartitions) {
- return new TreeIndexBulkLoadOperatorNodePushable(this, ctx, partition,
- fieldPermutation, fillFactor, recordDescProvider);
- }
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new TreeIndexBulkLoadOperatorNodePushable(this, ctx, partition, fieldPermutation, fillFactor,
+ recordDescProvider);
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
index 42dfaab..bc2d67e 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexBulkLoadOperatorNodePushable.java
@@ -23,76 +23,73 @@
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
-public class TreeIndexBulkLoadOperatorNodePushable extends
- AbstractUnaryInputSinkOperatorNodePushable {
- private float fillFactor;
- private final TreeIndexOpHelper treeIndexOpHelper;
- private FrameTupleAccessor accessor;
- private IIndexBulkLoadContext bulkLoadCtx;
+public class TreeIndexBulkLoadOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
+ private float fillFactor;
+ private final TreeIndexDataflowHelper treeIndexHelper;
+ private FrameTupleAccessor accessor;
+ private IIndexBulkLoadContext bulkLoadCtx;
+ private ITreeIndex treeIndex;
- private IRecordDescriptorProvider recordDescProvider;
+ private IRecordDescriptorProvider recordDescProvider;
- private PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
+ private PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
- public TreeIndexBulkLoadOperatorNodePushable(
- AbstractTreeIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition, int[] fieldPermutation,
- float fillFactor, IRecordDescriptorProvider recordDescProvider) {
- treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory()
- .createTreeIndexOpHelper(opDesc, ctx, partition,
- IndexHelperOpenMode.CREATE);
- this.fillFactor = fillFactor;
- this.recordDescProvider = recordDescProvider;
- tuple.setFieldPermutation(fieldPermutation);
- }
+ public TreeIndexBulkLoadOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+ int partition, int[] fieldPermutation, float fillFactor, IRecordDescriptorProvider recordDescProvider) {
+ treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
+ opDesc, ctx, partition, true);
+ this.fillFactor = fillFactor;
+ this.recordDescProvider = recordDescProvider;
+ tuple.setFieldPermutation(fieldPermutation);
+ }
- @Override
- public void open() throws HyracksDataException {
- AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexOpHelper
- .getOperatorDescriptor();
- RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(
- opDesc.getOperatorId(), 0);
- accessor = new FrameTupleAccessor(treeIndexOpHelper
- .getHyracksTaskContext().getFrameSize(), recDesc);
- ITreeIndexMetaDataFrame metaFrame = new LIFOMetaDataFrame();
- try {
- treeIndexOpHelper.init();
- treeIndexOpHelper.getTreeIndex().open(
- treeIndexOpHelper.getIndexFileId());
- bulkLoadCtx = treeIndexOpHelper.getTreeIndex().beginBulkLoad(
- fillFactor, treeIndexOpHelper.getLeafFrame(),
- treeIndexOpHelper.getInteriorFrame(), metaFrame);
- } catch (Exception e) {
- // cleanup in case of failure
- treeIndexOpHelper.deinit();
- throw new HyracksDataException(e);
- }
- }
+ @Override
+ public void open() throws HyracksDataException {
+ AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexHelper
+ .getOperatorDescriptor();
+ RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
+ accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), recDesc);
+ try {
+ treeIndexHelper.init();
+ treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
+ treeIndex.open(treeIndexHelper.getIndexFileId());
+ bulkLoadCtx = treeIndex.beginBulkLoad(fillFactor);
+ } catch (Exception e) {
+ // cleanup in case of failure
+ treeIndexHelper.deinit();
+ throw new HyracksDataException(e);
+ }
+ }
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- accessor.reset(buffer);
- int tupleCount = accessor.getTupleCount();
- for (int i = 0; i < tupleCount; i++) {
- tuple.reset(accessor, i);
- treeIndexOpHelper.getTreeIndex().bulkLoadAddTuple(bulkLoadCtx,
- tuple);
- }
- }
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ for (int i = 0; i < tupleCount; i++) {
+ tuple.reset(accessor, i);
+ try {
+ treeIndex.bulkLoadAddTuple(tuple, bulkLoadCtx);
+ } catch (PageAllocationException e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ }
- @Override
- public void close() throws HyracksDataException {
- try {
- treeIndexOpHelper.getTreeIndex().endBulkLoad(bulkLoadCtx);
- } finally {
- treeIndexOpHelper.deinit();
- }
- }
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ treeIndex.endBulkLoad(bulkLoadCtx);
+ } catch (PageAllocationException e) {
+ throw new HyracksDataException(e);
+ } finally {
+ treeIndexHelper.deinit();
+ }
+ }
- @Override
- public void fail() throws HyracksDataException {
- }
+ @Override
+ public void fail() throws HyracksDataException {
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
new file mode 100644
index 0000000..17c1827
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDataflowHelper.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
+
+public abstract class TreeIndexDataflowHelper extends IndexDataflowHelper {
+ protected ITreeIndexOperatorDescriptor treeOpDesc;
+ public TreeIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
+ boolean createIfNotExists) {
+ super(opDesc, ctx, partition, createIfNotExists);
+ this.treeOpDesc = (ITreeIndexOperatorDescriptor) opDesc;
+ }
+
+ public abstract ITreeIndex createIndexInstance() throws HyracksDataException;
+
+ public ITreeIndexCursor createDiskOrderScanCursor(ITreeIndexFrame leafFrame) throws HyracksDataException {
+ return new TreeDiskOrderScanCursor(leafFrame);
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
index 5fcb49a..8d56d37 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorDescriptor.java
@@ -18,12 +18,10 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -34,21 +32,20 @@
public TreeIndexDiskOrderScanOperatorDescriptor(JobSpecification spec,
RecordDescriptor recDesc, IStorageManagerInterface storageManager,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
+ IIndexRegistryProvider<IIndex> indexRegistryProvider,
IFileSplitProvider fileSplitProvider,
ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] typeTraits,
- ITreeIndexOpHelperFactory opHelperFactory) {
- super(spec, 0, 1, recDesc, storageManager, treeIndexRegistryProvider,
+ ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
+ IIndexDataflowHelperFactory dataflowHelperFactory) {
+ super(spec, 0, 1, recDesc, storageManager, indexRegistryProvider,
fileSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, null, null, opHelperFactory);
+ typeTraits, null, dataflowHelperFactory);
}
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IOperatorEnvironment env,
- IRecordDescriptorProvider recordDescProvider, int partition,
- int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider,
+ int partition, int nPartitions) {
return new TreeIndexDiskOrderScanOperatorNodePushable(this, ctx,
partition);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
index 0b22148..d9b4d87 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDiskOrderScanOperatorNodePushable.java
@@ -24,98 +24,78 @@
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOpContext;
-public class TreeIndexDiskOrderScanOperatorNodePushable extends
- AbstractUnaryOutputSourceOperatorNodePushable {
- private final TreeIndexOpHelper treeIndexOpHelper;
+public class TreeIndexDiskOrderScanOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
+ private final TreeIndexDataflowHelper treeIndexHelper;
+ private final ITreeIndexOperatorDescriptor opDesc;
+ private ITreeIndex treeIndex;
- public TreeIndexDiskOrderScanOperatorNodePushable(
- AbstractTreeIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition) {
- treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory()
- .createTreeIndexOpHelper(opDesc, ctx, partition,
- IndexHelperOpenMode.OPEN);
- }
+ public TreeIndexDiskOrderScanOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc,
+ IHyracksTaskContext ctx, int partition) {
+ treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
+ opDesc, ctx, partition, false);
+ this.opDesc = opDesc;
+ }
- @Override
- public void initialize() throws HyracksDataException {
+ @Override
+ public void initialize() throws HyracksDataException {
+ ITreeIndexFrame cursorFrame = opDesc.getTreeIndexLeafFactory().createFrame();
+ ITreeIndexCursor cursor = treeIndexHelper.createDiskOrderScanCursor(cursorFrame);
+ try {
+ treeIndexHelper.init();
+ treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
+ ITreeIndexAccessor indexAccessor = treeIndex.createAccessor();
+ writer.open();
+ try {
+ indexAccessor.diskOrderScan(cursor);
+ int fieldCount = treeIndex.getFieldCount();
+ ByteBuffer frame = treeIndexHelper.getHyracksTaskContext().allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(treeIndexHelper.getHyracksTaskContext()
+ .getFrameSize());
+ appender.reset(frame, true);
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
- ITreeIndexFrame cursorFrame = treeIndexOpHelper.getOperatorDescriptor()
- .getTreeIndexLeafFactory().createFrame();
- ITreeIndexCursor cursor = treeIndexOpHelper
- .createDiskOrderScanCursor(cursorFrame);
- ITreeIndexMetaDataFrame metaFrame = new LIFOMetaDataFrame();
+ while (cursor.hasNext()) {
+ tb.reset();
+ cursor.next();
- IndexOpContext diskOrderScanOpCtx = treeIndexOpHelper
- .getTreeIndex()
- .createOpContext(IndexOp.DISKORDERSCAN, cursorFrame, null, null);
- try {
+ ITupleReference frameTuple = cursor.getTuple();
+ for (int i = 0; i < frameTuple.getFieldCount(); i++) {
+ dos.write(frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
- treeIndexOpHelper.init();
- writer.open();
- try {
- treeIndexOpHelper.getTreeIndex().diskOrderScan(cursor,
- cursorFrame, metaFrame, diskOrderScanOpCtx);
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ FrameUtils.flushFrame(frame, writer);
+ appender.reset(frame, true);
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ if (appender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(frame, writer);
+ }
+ } catch (Exception e) {
+ writer.fail();
+ throw new HyracksDataException(e);
+ } finally {
+ cursor.close();
+ writer.close();
+ }
+ } catch (Exception e) {
+ deinitialize();
+ throw new HyracksDataException(e);
+ }
+ }
- int fieldCount = treeIndexOpHelper.getTreeIndex()
- .getFieldCount();
- ByteBuffer frame = treeIndexOpHelper.getHyracksTaskContext()
- .allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(
- treeIndexOpHelper.getHyracksTaskContext()
- .getFrameSize());
- appender.reset(frame, true);
- ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
- DataOutput dos = tb.getDataOutput();
-
- while (cursor.hasNext()) {
- tb.reset();
- cursor.next();
-
- ITupleReference frameTuple = cursor.getTuple();
- for (int i = 0; i < frameTuple.getFieldCount(); i++) {
- dos.write(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i),
- frameTuple.getFieldLength(i));
- tb.addFieldEndOffset();
- }
-
- if (!appender.append(tb.getFieldEndOffsets(),
- tb.getByteArray(), 0, tb.getSize())) {
- FrameUtils.flushFrame(frame, writer);
- appender.reset(frame, true);
- if (!appender.append(tb.getFieldEndOffsets(),
- tb.getByteArray(), 0, tb.getSize())) {
- throw new IllegalStateException();
- }
- }
- }
-
- if (appender.getTupleCount() > 0) {
- FrameUtils.flushFrame(frame, writer);
- }
- } catch (Exception e) {
- writer.fail();
- throw new HyracksDataException(e);
- } finally {
- cursor.close();
- writer.close();
- }
-
- } catch (Exception e) {
- deinitialize();
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public void deinitialize() throws HyracksDataException {
- treeIndexOpHelper.deinit();
- }
+ @Override
+ public void deinitialize() throws HyracksDataException {
+ treeIndexHelper.deinit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorDescriptor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorDescriptor.java
index 0053877..6ce18db 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorDescriptor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorDescriptor.java
@@ -18,11 +18,9 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public class TreeIndexDropOperatorDescriptor extends
@@ -31,12 +29,12 @@
private static final long serialVersionUID = 1L;
private IStorageManagerInterface storageManager;
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider;
+ private IIndexRegistryProvider<IIndex> treeIndexRegistryProvider;
private IFileSplitProvider fileSplitProvider;
public TreeIndexDropOperatorDescriptor(JobSpecification spec,
IStorageManagerInterface storageManager,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
+ IIndexRegistryProvider<IIndex> treeIndexRegistryProvider,
IFileSplitProvider fileSplitProvider) {
super(spec, 0, 0);
this.storageManager = storageManager;
@@ -46,9 +44,8 @@
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IOperatorEnvironment env,
- IRecordDescriptorProvider recordDescProvider, int partition,
- int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider,
+ int partition, int nPartitions) {
return new TreeIndexDropOperatorNodePushable(ctx, storageManager,
treeIndexRegistryProvider, fileSplitProvider, partition);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java
index 6614fba..ec44982 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexDropOperatorNodePushable.java
@@ -25,92 +25,79 @@
import edu.uci.ics.hyracks.api.io.FileReference;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-public class TreeIndexDropOperatorNodePushable extends
- AbstractOperatorNodePushable {
- private static final Logger LOGGER = Logger
- .getLogger(TreeIndexDropOperatorNodePushable.class.getName());
+public class TreeIndexDropOperatorNodePushable extends AbstractOperatorNodePushable {
+ private static final Logger LOGGER = Logger.getLogger(TreeIndexDropOperatorNodePushable.class.getName());
- private final IHyracksTaskContext ctx;
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider;
- private IStorageManagerInterface storageManager;
- private IFileSplitProvider fileSplitProvider;
- private int partition;
+ private final IHyracksTaskContext ctx;
+ private IIndexRegistryProvider<IIndex> treeIndexRegistryProvider;
+ private IStorageManagerInterface storageManager;
+ private IFileSplitProvider fileSplitProvider;
+ private int partition;
- public TreeIndexDropOperatorNodePushable(IHyracksTaskContext ctx,
- IStorageManagerInterface storageManager,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
- IFileSplitProvider fileSplitProvider, int partition) {
- this.ctx = ctx;
- this.storageManager = storageManager;
- this.treeIndexRegistryProvider = treeIndexRegistryProvider;
- this.fileSplitProvider = fileSplitProvider;
- this.partition = partition;
- }
+ public TreeIndexDropOperatorNodePushable(IHyracksTaskContext ctx, IStorageManagerInterface storageManager,
+ IIndexRegistryProvider<IIndex> treeIndexRegistryProvider, IFileSplitProvider fileSplitProvider,
+ int partition) {
+ this.ctx = ctx;
+ this.storageManager = storageManager;
+ this.treeIndexRegistryProvider = treeIndexRegistryProvider;
+ this.fileSplitProvider = fileSplitProvider;
+ this.partition = partition;
+ }
- @Override
- public void deinitialize() throws HyracksDataException {
- }
+ @Override
+ public void deinitialize() throws HyracksDataException {
+ }
- @Override
- public int getInputArity() {
- return 0;
- }
+ @Override
+ public int getInputArity() {
+ return 0;
+ }
- @Override
- public IFrameWriter getInputFrameWriter(int index) {
- return null;
- }
+ @Override
+ public IFrameWriter getInputFrameWriter(int index) {
+ return null;
+ }
- @Override
- public void initialize() throws HyracksDataException {
- try {
+ @Override
+ public void initialize() throws HyracksDataException {
+ try {
+ IndexRegistry<IIndex> treeIndexRegistry = treeIndexRegistryProvider.getRegistry(ctx);
+ IBufferCache bufferCache = storageManager.getBufferCache(ctx);
+ IFileMapProvider fileMapProvider = storageManager.getFileMapProvider(ctx);
+
+ FileReference f = fileSplitProvider.getFileSplits()[partition].getLocalFile();
+ int indexFileId = -1;
+ synchronized (fileMapProvider) {
+ boolean fileIsMapped = fileMapProvider.isMapped(f);
+ if (!fileIsMapped) {
+ throw new HyracksDataException("Cannot drop Tree with name " + f.toString()
+ + ". No file mapping exists.");
+ }
+ indexFileId = fileMapProvider.lookupFileId(f);
+ }
+ // Unregister tree instance.
+ synchronized (treeIndexRegistry) {
+ treeIndexRegistry.unregister(indexFileId);
+ }
- IndexRegistry<ITreeIndex> treeIndexRegistry = treeIndexRegistryProvider
- .getRegistry(ctx);
- IBufferCache bufferCache = storageManager.getBufferCache(ctx);
- IFileMapProvider fileMapProvider = storageManager
- .getFileMapProvider(ctx);
+ // remove name to id mapping
+ bufferCache.deleteFile(indexFileId);
+ }
+ // TODO: for the time being we don't throw,
+ // with proper exception handling (no hanging job problem) we should
+ // throw
+ catch (Exception e) {
+ if (LOGGER.isLoggable(Level.WARNING)) {
+ LOGGER.warning("Tree Drop Operator Failed Due To Exception: " + e.getMessage());
+ }
+ }
+ }
- FileReference f = fileSplitProvider.getFileSplits()[partition]
- .getLocalFile();
-
- boolean fileIsMapped = fileMapProvider.isMapped(f);
- if (!fileIsMapped) {
- throw new HyracksDataException("Cannot drop Tree with name "
- + f.toString() + ". No file mapping exists.");
- }
-
- int indexFileId = fileMapProvider.lookupFileId(f);
-
- // unregister tree instance
- treeIndexRegistry.lock();
- try {
- treeIndexRegistry.unregister(indexFileId);
- } finally {
- treeIndexRegistry.unlock();
- }
-
- // remove name to id mapping
- bufferCache.deleteFile(indexFileId);
- }
- // TODO: for the time being we don't throw,
- // with proper exception handling (no hanging job problem) we should
- // throw
- catch (Exception e) {
- if (LOGGER.isLoggable(Level.WARNING)) {
- LOGGER.warning("Tree Drop Operator Failed Due To Exception: "
- + e.getMessage());
- }
- }
- }
-
- @Override
- public void setOutputFrameWriter(int index, IFrameWriter writer,
- RecordDescriptor recordDesc) {
- }
+ @Override
+ public void setOutputFrameWriter(int index, IFrameWriter writer, RecordDescriptor recordDesc) {
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexFileEnlistmentOperatorDescriptor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexFileEnlistmentOperatorDescriptor.java
deleted file mode 100644
index 60fa8cc..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexFileEnlistmentOperatorDescriptor.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.common.dataflow;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
-import edu.uci.ics.hyracks.api.job.JobSpecification;
-import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-
-// re-create in-memory state for a tree index that has already been built (i.e., the file exists):
-// 1. register files in file manager (FileManager)
-// 2. create file mappings (FileMappingProvider)
-// 3. register tree index instance (IndexRegistry)
-
-public class TreeIndexFileEnlistmentOperatorDescriptor extends
- AbstractTreeIndexOperatorDescriptor {
-
- private static final long serialVersionUID = 1L;
-
- public TreeIndexFileEnlistmentOperatorDescriptor(JobSpecification spec,
- RecordDescriptor recDesc, IStorageManagerInterface storageManager,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
- IFileSplitProvider fileSplitProvider,
- ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] typeTraits,
- IBinaryComparatorFactory[] comparatorFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories,
- ITreeIndexOpHelperFactory opHelperFactory) {
- super(spec, 0, 0, recDesc, storageManager, treeIndexRegistryProvider,
- fileSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, comparatorFactories, valueProviderFactories,
- opHelperFactory);
- }
-
- @Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IOperatorEnvironment env,
- IRecordDescriptorProvider recordDescProvider, int partition,
- int partitions) throws HyracksDataException {
- return new TreeIndexFileEnlistmentOperatorNodePushable(this, ctx,
- partition);
- }
-
-}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexFileEnlistmentOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexFileEnlistmentOperatorNodePushable.java
deleted file mode 100644
index 124e2cc..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexFileEnlistmentOperatorNodePushable.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.common.dataflow;
-
-import edu.uci.ics.hyracks.api.comm.IFrameWriter;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
-
-public class TreeIndexFileEnlistmentOperatorNodePushable extends
- AbstractOperatorNodePushable {
-
- private final TreeIndexOpHelper treeIndexOpHelper;
-
- public TreeIndexFileEnlistmentOperatorNodePushable(
- AbstractTreeIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition) {
- treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory()
- .createTreeIndexOpHelper(opDesc, ctx, partition,
- IndexHelperOpenMode.ENLIST);
- }
-
- @Override
- public void deinitialize() throws HyracksDataException {
- }
-
- @Override
- public int getInputArity() {
- return 0;
- }
-
- @Override
- public IFrameWriter getInputFrameWriter(int index) {
- return null;
- }
-
- @Override
- public void initialize() throws HyracksDataException {
- try {
- treeIndexOpHelper.init();
- } finally {
- treeIndexOpHelper.deinit();
- }
- }
-
- @Override
- public void setOutputFrameWriter(int index, IFrameWriter writer,
- RecordDescriptor recordDesc) {
- }
-}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
index b41bd98..e08f5d6 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorDescriptor.java
@@ -19,13 +19,10 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -41,27 +38,25 @@
public TreeIndexInsertUpdateDeleteOperatorDescriptor(JobSpecification spec,
RecordDescriptor recDesc, IStorageManagerInterface storageManager,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
+ IIndexRegistryProvider<IIndex> indexRegistryProvider,
IFileSplitProvider fileSplitProvider,
ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] typeTraits,
+ ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
IBinaryComparatorFactory[] comparatorFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories,
int[] fieldPermutation, IndexOp op,
- ITreeIndexOpHelperFactory opHelperFactory) {
- super(spec, 1, 1, recDesc, storageManager, treeIndexRegistryProvider,
+ IIndexDataflowHelperFactory dataflowHelperFactory) {
+ super(spec, 1, 1, recDesc, storageManager, indexRegistryProvider,
fileSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, comparatorFactories, valueProviderFactories,
- opHelperFactory);
+ typeTraits, comparatorFactories,
+ dataflowHelperFactory);
this.fieldPermutation = fieldPermutation;
this.op = op;
}
@Override
public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IOperatorEnvironment env,
- IRecordDescriptorProvider recordDescProvider, int partition,
- int nPartitions) {
+ IRecordDescriptorProvider recordDescProvider,
+ int partition, int nPartitions) {
return new TreeIndexInsertUpdateDeleteOperatorNodePushable(this, ctx,
partition, fieldPermutation, recordDescProvider, op);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorNodePushable.java
index cc49998..bf78249 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorNodePushable.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexInsertUpdateDeleteOperatorNodePushable.java
@@ -24,108 +24,96 @@
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOpContext;
-public class TreeIndexInsertUpdateDeleteOperatorNodePushable extends
- AbstractUnaryInputUnaryOutputOperatorNodePushable {
- private final TreeIndexOpHelper treeIndexOpHelper;
- private FrameTupleAccessor accessor;
- private final IRecordDescriptorProvider recordDescProvider;
- private final IndexOp op;
- private final PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
- private ByteBuffer writeBuffer;
- private IndexOpContext opCtx;
+public class TreeIndexInsertUpdateDeleteOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
+ private final TreeIndexDataflowHelper treeIndexHelper;
+ private FrameTupleAccessor accessor;
+ private final IRecordDescriptorProvider recordDescProvider;
+ private final IndexOp op;
+ private final PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
+ private ByteBuffer writeBuffer;
+ private ITreeIndexAccessor indexAccessor;
- public TreeIndexInsertUpdateDeleteOperatorNodePushable(
- AbstractTreeIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition, int[] fieldPermutation,
- IRecordDescriptorProvider recordDescProvider, IndexOp op) {
- treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory()
- .createTreeIndexOpHelper(opDesc, ctx, partition,
- IndexHelperOpenMode.OPEN);
- this.recordDescProvider = recordDescProvider;
- this.op = op;
- tuple.setFieldPermutation(fieldPermutation);
- }
+ public TreeIndexInsertUpdateDeleteOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc,
+ IHyracksTaskContext ctx, int partition, int[] fieldPermutation,
+ IRecordDescriptorProvider recordDescProvider, IndexOp op) {
+ // Only create the if insert operation is an insert.
+ boolean createIfNotExists = (op == IndexOp.INSERT);
+ treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
+ opDesc, ctx, partition, createIfNotExists);
+ this.recordDescProvider = recordDescProvider;
+ this.op = op;
+ tuple.setFieldPermutation(fieldPermutation);
+ }
- @Override
- public void open() throws HyracksDataException {
- AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexOpHelper
- .getOperatorDescriptor();
- RecordDescriptor inputRecDesc = recordDescProvider
- .getInputRecordDescriptor(opDesc.getOperatorId(), 0);
- accessor = new FrameTupleAccessor(treeIndexOpHelper
- .getHyracksTaskContext().getFrameSize(), inputRecDesc);
- writeBuffer = treeIndexOpHelper.getHyracksTaskContext().allocateFrame();
- writer.open();
- try {
- treeIndexOpHelper.init();
- treeIndexOpHelper.getTreeIndex().open(
- treeIndexOpHelper.getIndexFileId());
- opCtx = treeIndexOpHelper.getTreeIndex().createOpContext(op,
- treeIndexOpHelper.getLeafFrame(),
- treeIndexOpHelper.getInteriorFrame(),
- new LIFOMetaDataFrame());
- } catch (Exception e) {
- // cleanup in case of failure
- treeIndexOpHelper.deinit();
- throw new HyracksDataException(e);
- }
- }
+ @Override
+ public void open() throws HyracksDataException {
+ AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexHelper
+ .getOperatorDescriptor();
+ RecordDescriptor inputRecDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
+ accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), inputRecDesc);
+ writeBuffer = treeIndexHelper.getHyracksTaskContext().allocateFrame();
+ writer.open();
+ try {
+ treeIndexHelper.init();
+ ITreeIndex treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
+ indexAccessor = treeIndex.createAccessor();
+ } catch (Exception e) {
+ // cleanup in case of failure
+ treeIndexHelper.deinit();
+ throw new HyracksDataException(e);
+ }
+ }
- @Override
- public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
- final ITreeIndex treeIndex = treeIndexOpHelper.getTreeIndex();
- accessor.reset(buffer);
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ for (int i = 0; i < tupleCount; i++) {
+ tuple.reset(accessor, i);
+ try {
+ switch (op) {
+ case INSERT: {
+ indexAccessor.insert(tuple);
+ break;
+ }
+ case UPDATE: {
+ indexAccessor.update(tuple);
+ break;
+ }
+ case DELETE: {
+ indexAccessor.delete(tuple);
+ break;
+ }
+ default: {
+ throw new HyracksDataException("Unsupported operation " + op
+ + " in tree index InsertUpdateDelete operator");
+ }
+ }
+ } catch (HyracksDataException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+ // Pass a copy of the frame to next op.
+ System.arraycopy(buffer.array(), 0, writeBuffer.array(), 0, buffer.capacity());
+ FrameUtils.flushFrame(writeBuffer, writer);
+ }
- int tupleCount = accessor.getTupleCount();
- for (int i = 0; i < tupleCount; i++) {
- tuple.reset(accessor, i);
- try {
- switch (op) {
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ writer.close();
+ } finally {
+ treeIndexHelper.deinit();
+ }
+ }
- case INSERT: {
- treeIndex.insert(tuple, opCtx);
- }
- break;
-
- case DELETE: {
- treeIndex.delete(tuple, opCtx);
- }
- break;
-
- default: {
- throw new HyracksDataException("Unsupported operation "
- + op + " in tree index InsertUpdateDelete operator");
- }
-
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- throw new HyracksDataException(e);
- }
- }
-
- // pass a copy of the frame to next op
- System.arraycopy(buffer.array(), 0, writeBuffer.array(), 0,
- buffer.capacity());
- FrameUtils.flushFrame(writeBuffer, writer);
- }
-
- @Override
- public void close() throws HyracksDataException {
- try {
- writer.close();
- } finally {
- treeIndexOpHelper.deinit();
- }
- }
-
- @Override
- public void fail() throws HyracksDataException {
- writer.fail();
- }
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexOpHelper.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexOpHelper.java
deleted file mode 100644
index f9fd77d..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexOpHelper.java
+++ /dev/null
@@ -1,199 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.storage.am.common.dataflow;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-
-public abstract class TreeIndexOpHelper {
-
- protected ITreeIndexFrame interiorFrame;
- protected ITreeIndexFrame leafFrame;
- protected MultiComparator cmp;
-
- protected ITreeIndex treeIndex;
- protected int indexFileId = -1;
- protected int partition;
-
- protected ITreeIndexOperatorDescriptorHelper opDesc;
- protected IHyracksTaskContext ctx;
-
- protected IndexHelperOpenMode mode;
-
- public TreeIndexOpHelper(ITreeIndexOperatorDescriptorHelper opDesc,
- final IHyracksTaskContext ctx, int partition,
- IndexHelperOpenMode mode) {
- this.opDesc = opDesc;
- this.ctx = ctx;
- this.mode = mode;
- this.partition = partition;
- }
-
- public void init() throws HyracksDataException {
- IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(
- ctx);
- IFileMapProvider fileMapProvider = opDesc.getStorageManager()
- .getFileMapProvider(ctx);
- IFileSplitProvider fileSplitProvider = opDesc
- .getTreeIndexFileSplitProvider();
-
- FileReference f = fileSplitProvider.getFileSplits()[partition]
- .getLocalFile();
- boolean fileIsMapped = fileMapProvider.isMapped(f);
-
- switch (mode) {
-
- case OPEN: {
- if (!fileIsMapped) {
- throw new HyracksDataException(
- "Trying to open tree index from unmapped file "
- + f.toString());
- }
- }
- break;
-
- case CREATE:
- case ENLIST: {
- if (!fileIsMapped) {
- bufferCache.createFile(f);
- }
- }
- break;
-
- }
-
- int fileId = fileMapProvider.lookupFileId(f);
- try {
- bufferCache.openFile(fileId);
- } catch (HyracksDataException e) {
- // revert state of buffer cache since file failed to open
- if (!fileIsMapped) {
- bufferCache.deleteFile(fileId);
- }
- throw e;
- }
-
- // only set indexFileId member when openFile() succeeds,
- // otherwise deinit() will try to close the file that failed to open
- indexFileId = fileId;
-
- interiorFrame = opDesc.getTreeIndexInteriorFactory().createFrame();
- leafFrame = opDesc.getTreeIndexLeafFactory().createFrame();
-
- IndexRegistry<ITreeIndex> treeIndexRegistry = opDesc
- .getTreeIndexRegistryProvider().getRegistry(ctx);
- treeIndex = treeIndexRegistry.get(indexFileId);
- if (treeIndex == null) {
-
- // create new tree and register it
- treeIndexRegistry.lock();
- try {
- // check if tree has already been registered by another thread
- treeIndex = treeIndexRegistry.get(indexFileId);
- if (treeIndex == null) {
- // this thread should create and register the tree
-
- IBinaryComparator[] comparators = new IBinaryComparator[opDesc
- .getTreeIndexComparatorFactories().length];
- for (int i = 0; i < opDesc
- .getTreeIndexComparatorFactories().length; i++) {
- comparators[i] = opDesc
- .getTreeIndexComparatorFactories()[i]
- .createBinaryComparator();
- }
-
- cmp = createMultiComparator(comparators);
-
- treeIndex = createTreeIndex();
- if (mode == IndexHelperOpenMode.CREATE) {
- ITreeIndexMetaDataFrame metaFrame = treeIndex
- .getFreePageManager().getMetaDataFrameFactory()
- .createFrame();
- try {
- treeIndex.create(indexFileId, leafFrame, metaFrame);
- } catch (Exception e) {
- throw new HyracksDataException(e);
- }
- }
- treeIndex.open(indexFileId);
- treeIndexRegistry.register(indexFileId, treeIndex);
- }
- } finally {
- treeIndexRegistry.unlock();
- }
- }
- }
-
- // MUST be overridden
- public ITreeIndex createTreeIndex() throws HyracksDataException {
- throw new HyracksDataException(
- "createTreeIndex Operation not implemented.");
- }
-
- // MUST be overridden
- public MultiComparator createMultiComparator(IBinaryComparator[] comparators)
- throws HyracksDataException {
- throw new HyracksDataException(
- "createComparator Operation not implemented.");
- }
-
- public ITreeIndexCursor createDiskOrderScanCursor(ITreeIndexFrame leafFrame)
- throws HyracksDataException {
- return new TreeDiskOrderScanCursor(leafFrame);
- }
-
- public void deinit() throws HyracksDataException {
- if (indexFileId != -1) {
- IBufferCache bufferCache = opDesc.getStorageManager()
- .getBufferCache(ctx);
- bufferCache.closeFile(indexFileId);
- }
- }
-
- public ITreeIndex getTreeIndex() {
- return treeIndex;
- }
-
- public IHyracksTaskContext getHyracksTaskContext() {
- return ctx;
- }
-
- public ITreeIndexOperatorDescriptorHelper getOperatorDescriptor() {
- return opDesc;
- }
-
- public ITreeIndexFrame getLeafFrame() {
- return leafFrame;
- }
-
- public ITreeIndexFrame getInteriorFrame() {
- return interiorFrame;
- }
-
- public int getIndexFileId() {
- return indexFileId;
- }
-}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
index d1a1614..4d718a8 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorDescriptor.java
@@ -1,40 +1,50 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package edu.uci.ics.hyracks.storage.am.common.dataflow;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-public class TreeIndexStatsOperatorDescriptor extends
- AbstractTreeIndexOperatorDescriptor {
+public class TreeIndexStatsOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1L;
+ private static final RecordDescriptor recDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE });
- public TreeIndexStatsOperatorDescriptor(JobSpecification spec,
- IStorageManagerInterface storageManager,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
- IFileSplitProvider fileSplitProvider,
- ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] typeTraits,
- IBinaryComparatorFactory[] comparatorFactories,
- ITreeIndexOpHelperFactory opHelperFactory) {
- super(spec, 0, 0, null, storageManager, treeIndexRegistryProvider,
- fileSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, comparatorFactories, null, opHelperFactory);
- }
+ public TreeIndexStatsOperatorDescriptor(JobSpecification spec, IStorageManagerInterface storageManager,
+ IIndexRegistryProvider<IIndex> indexRegistryProvider, IFileSplitProvider fileSplitProvider,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
+ ITypeTraits[] typeTraits, IBinaryComparatorFactory[] comparatorFactories,
+ IIndexDataflowHelperFactory dataflowHelperFactory) {
+ super(spec, 0, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
+ leafFrameFactory, typeTraits, comparatorFactories, dataflowHelperFactory);
+ }
- @Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
- IOperatorEnvironment env,
- IRecordDescriptorProvider recordDescProvider, int partition,
- int nPartitions) {
- return new TreeIndexStatsOperatorNodePushable(this, ctx, partition);
- }
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new TreeIndexStatsOperatorNodePushable(this, ctx, partition);
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
index 4b53453..ff007af 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/dataflow/TreeIndexStatsOperatorNodePushable.java
@@ -14,75 +14,76 @@
*/
package edu.uci.ics.hyracks.storage.am.common.dataflow;
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+
import edu.uci.ics.hyracks.api.comm.IFrameWriter;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.dataflow.std.base.AbstractOperatorNodePushable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryOutputSourceOperatorNodePushable;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.utility.TreeIndexStats;
-import edu.uci.ics.hyracks.storage.am.common.utility.TreeIndexStatsGatherer;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStats;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-public class TreeIndexStatsOperatorNodePushable extends
- AbstractOperatorNodePushable {
- private final TreeIndexOpHelper treeIndexOpHelper;
- private final IHyracksTaskContext ctx;
- private TreeIndexStatsGatherer statsGatherer;
+public class TreeIndexStatsOperatorNodePushable extends AbstractUnaryOutputSourceOperatorNodePushable {
+ private final TreeIndexDataflowHelper treeIndexHelper;
+ private final IHyracksTaskContext ctx;
+ private TreeIndexStatsGatherer statsGatherer;
- public TreeIndexStatsOperatorNodePushable(
- AbstractTreeIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition) {
- treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory()
- .createTreeIndexOpHelper(opDesc, ctx, partition,
- IndexHelperOpenMode.CREATE);
- this.ctx = ctx;
- }
+ public TreeIndexStatsOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+ int partition) {
+ treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
+ opDesc, ctx, partition, false);
+ this.ctx = ctx;
+ }
- @Override
- public void deinitialize() throws HyracksDataException {
- }
+ @Override
+ public void deinitialize() throws HyracksDataException {
+ }
- @Override
- public int getInputArity() {
- return 0;
- }
+ @Override
+ public IFrameWriter getInputFrameWriter(int index) {
+ return null;
+ }
- @Override
- public IFrameWriter getInputFrameWriter(int index) {
- return null;
- }
-
- @Override
- public void initialize() throws HyracksDataException {
- try {
- treeIndexOpHelper.init();
- treeIndexOpHelper.getTreeIndex().open(
- treeIndexOpHelper.getIndexFileId());
-
- ITreeIndex treeIndex = treeIndexOpHelper.getTreeIndex();
- IBufferCache bufferCache = treeIndexOpHelper
- .getOperatorDescriptor().getStorageManager()
- .getBufferCache(ctx);
-
- statsGatherer = new TreeIndexStatsGatherer(bufferCache,
- treeIndex.getFreePageManager(),
- treeIndexOpHelper.getIndexFileId(),
- treeIndex.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(treeIndex
- .getLeafFrameFactory().createFrame(), treeIndex
- .getInteriorFrameFactory().createFrame(), treeIndex
- .getFreePageManager().getMetaDataFrameFactory()
- .createFrame());
- System.err.println(stats.toString());
- } catch (Exception e) {
- treeIndexOpHelper.deinit();
- throw new HyracksDataException(e);
- }
- }
-
- @Override
- public void setOutputFrameWriter(int index, IFrameWriter writer,
- RecordDescriptor recordDesc) {
- }
+ @Override
+ public void initialize() throws HyracksDataException {
+ try {
+ writer.open();
+ treeIndexHelper.init();
+ ITreeIndex treeIndex = (ITreeIndex) treeIndexHelper.getIndex();
+ IBufferCache bufferCache = treeIndexHelper.getOperatorDescriptor().getStorageManager().getBufferCache(ctx);
+ statsGatherer = new TreeIndexStatsGatherer(bufferCache, treeIndex.getFreePageManager(),
+ treeIndexHelper.getIndexFileId(), treeIndex.getRootPageId());
+ TreeIndexStats stats = statsGatherer.gatherStats(treeIndex.getLeafFrameFactory().createFrame(), treeIndex
+ .getInteriorFrameFactory().createFrame(), treeIndex.getFreePageManager().getMetaDataFrameFactory()
+ .createFrame());
+ // Write the stats output as a single string field.
+ ByteBuffer frame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ appender.reset(frame, true);
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(1);
+ DataOutput dos = tb.getDataOutput();
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(stats.toString(), dos);
+ tb.addFieldEndOffset();
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ FrameUtils.flushFrame(frame, writer);
+ } catch (Exception e) {
+ try {
+ treeIndexHelper.deinit();
+ } finally {
+ writer.fail();
+ }
+ } finally {
+ writer.close();
+ }
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/AbstractSlotManager.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/AbstractSlotManager.java
index bce2e19..a1a38ab 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/AbstractSlotManager.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/AbstractSlotManager.java
@@ -19,7 +19,10 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
public abstract class AbstractSlotManager implements ISlotManager {
-
+
+ protected final int GREATEST_KEY_INDICATOR = -1;
+ protected final int ERROR_INDICATOR = -2;
+
protected static final int slotSize = 4;
protected ITreeIndexFrame frame;
@@ -58,4 +61,14 @@
public int getSlotOff(int tupleIndex) {
return getSlotStartOff() - tupleIndex * slotSize;
}
+
+ @Override
+ public int getGreatestKeyIndicator() {
+ return GREATEST_KEY_INDICATOR;
+ }
+
+ @Override
+ public int getErrorIndicator() {
+ return ERROR_INDICATOR;
+ }
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/FrameOpSpaceStatus.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/FrameOpSpaceStatus.java
index 7114875..da9c815 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/FrameOpSpaceStatus.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/FrameOpSpaceStatus.java
@@ -16,5 +16,5 @@
package edu.uci.ics.hyracks.storage.am.common.frames;
public enum FrameOpSpaceStatus {
- INSUFFICIENT_SPACE, SUFFICIENT_CONTIGUOUS_SPACE, SUFFICIENT_SPACE
+ INSUFFICIENT_SPACE, SUFFICIENT_CONTIGUOUS_SPACE, SUFFICIENT_SPACE, SUFFICIENT_INPLACE_SPACE
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
index b0ec13d..c87b84a 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/LIFOMetaDataFrame.java
@@ -27,12 +27,11 @@
public class LIFOMetaDataFrame implements ITreeIndexMetaDataFrame {
- protected static final int tupleCountOff = 0;
- protected static final int freeSpaceOff = tupleCountOff + 4;
- protected static final int maxPageOff = freeSpaceOff + 4;
- protected static final int dummyFieldOff = maxPageOff + 4;
- protected static final byte levelOff = dummyFieldOff + 4;
- protected static final byte nextPageOff = levelOff + 1;
+ protected static final int tupleCountOff = 0; //0
+ protected static final int freeSpaceOff = tupleCountOff + 4; //4
+ protected static final int maxPageOff = freeSpaceOff + 4; //8
+ protected static final int levelOff = maxPageOff + 12; //20
+ protected static final int nextPageOff = levelOff + 1; // 21
protected ICachedPage page = null;
protected ByteBuffer buf = null;
@@ -96,10 +95,11 @@
}
@Override
- public void initBuffer(int level) {
- buf.putInt(freeSpaceOff, nextPageOff + 4);
+ public void initBuffer(byte level) {
buf.putInt(tupleCountOff, 0);
- buf.putInt(levelOff, level);
+ buf.putInt(freeSpaceOff, nextPageOff + 4);
+ //buf.putInt(maxPageOff, -1);
+ buf.put(levelOff, level);
buf.putInt(nextPageOff, -1);
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
index e5c37ff..e2e28fd 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/frames/TreeIndexNSMFrame.java
@@ -15,332 +15,281 @@
package edu.uci.ics.hyracks.storage.am.common.frames;
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ISlotManager;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.SlotOffTupleOff;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
public abstract class TreeIndexNSMFrame implements ITreeIndexFrame {
- protected static final int pageLsnOff = 0; // 0
- protected static final int tupleCountOff = pageLsnOff + 4; // 4
- protected static final int freeSpaceOff = tupleCountOff + 4; // 8
- protected static final int totalFreeSpaceOff = freeSpaceOff + 4; // 16
- protected static final byte levelOff = totalFreeSpaceOff + 4;
- protected static final byte smFlagOff = levelOff + 1;
+ protected static final int pageLsnOff = 0; // 0
+ protected static final int tupleCountOff = pageLsnOff + 8; // 8
+ protected static final int freeSpaceOff = tupleCountOff + 4; // 12
+ protected static final int totalFreeSpaceOff = freeSpaceOff + 4; // 16
+ protected static final int levelOff = totalFreeSpaceOff + 4; // 20
+ protected static final int smFlagOff = levelOff + 1; // 21
- protected ICachedPage page = null;
- protected ByteBuffer buf = null;
- protected ISlotManager slotManager;
+ protected ICachedPage page = null;
+ protected ByteBuffer buf = null;
+ protected ISlotManager slotManager;
- protected ITreeIndexTupleWriter tupleWriter;
- protected ITreeIndexTupleReference frameTuple;
+ protected ITreeIndexTupleWriter tupleWriter;
+ protected ITreeIndexTupleReference frameTuple;
- public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter,
- ISlotManager slotManager) {
- this.tupleWriter = tupleWriter;
- this.frameTuple = tupleWriter.createTupleReference();
- this.slotManager = slotManager;
- }
+ public TreeIndexNSMFrame(ITreeIndexTupleWriter tupleWriter, ISlotManager slotManager) {
+ this.tupleWriter = tupleWriter;
+ this.frameTuple = tupleWriter.createTupleReference();
+ this.slotManager = slotManager;
+ this.slotManager.setFrame(this);
+ }
- @Override
- public void initBuffer(byte level) {
- buf.putInt(pageLsnOff, 0); // TODO: might to set to a different lsn
- // during creation
- buf.putInt(tupleCountOff, 0);
- resetSpaceParams();
- buf.put(levelOff, level);
- buf.put(smFlagOff, (byte) 0);
- }
+ @Override
+ public void initBuffer(byte level) {
+ buf.putLong(pageLsnOff, 0); // TODO: might to set to a different lsn
+ // during creation
+ buf.putInt(tupleCountOff, 0);
+ resetSpaceParams();
+ buf.put(levelOff, level);
+ buf.put(smFlagOff, (byte) 0);
+ }
- @Override
- public boolean isLeaf() {
- return buf.get(levelOff) == 0;
- }
+ @Override
+ public boolean isLeaf() {
+ return buf.get(levelOff) == 0;
+ }
- @Override
- public boolean isInterior() {
- return buf.get(levelOff) > 0;
- }
+ @Override
+ public boolean isInterior() {
+ return buf.get(levelOff) > 0;
+ }
- @Override
- public byte getLevel() {
- return buf.get(levelOff);
- }
+ @Override
+ public byte getLevel() {
+ return buf.get(levelOff);
+ }
- @Override
- public void setLevel(byte level) {
- buf.put(levelOff, level);
- }
+ @Override
+ public void setLevel(byte level) {
+ buf.put(levelOff, level);
+ }
- @Override
- public boolean getSmFlag() {
- return buf.get(smFlagOff) != 0;
- }
+ @Override
+ public int getFreeSpaceOff() {
+ return buf.getInt(freeSpaceOff);
+ }
- @Override
- public void setSmFlag(boolean smFlag) {
- if (smFlag)
- buf.put(smFlagOff, (byte) 1);
- else
- buf.put(smFlagOff, (byte) 0);
- }
+ @Override
+ public void setFreeSpaceOff(int freeSpace) {
+ buf.putInt(freeSpaceOff, freeSpace);
+ }
- @Override
- public int getFreeSpaceOff() {
- return buf.getInt(freeSpaceOff);
- }
+ @Override
+ public void setPage(ICachedPage page) {
+ this.page = page;
+ this.buf = page.getBuffer();
+ }
- @Override
- public void setFreeSpaceOff(int freeSpace) {
- buf.putInt(freeSpaceOff, freeSpace);
- }
+ @Override
+ public ByteBuffer getBuffer() {
+ return page.getBuffer();
+ }
- @Override
- public void setPage(ICachedPage page) {
- this.page = page;
- this.buf = page.getBuffer();
- slotManager.setFrame(this);
- }
+ @Override
+ public ICachedPage getPage() {
+ return page;
+ }
- @Override
- public ByteBuffer getBuffer() {
- return page.getBuffer();
- }
+ @Override
+ public boolean compact() {
+ resetSpaceParams();
+ int tupleCount = buf.getInt(tupleCountOff);
+ int freeSpace = buf.getInt(freeSpaceOff);
+ // Sort the slots by the tuple offset they point to.
+ ArrayList<SlotOffTupleOff> sortedTupleOffs = new ArrayList<SlotOffTupleOff>();
+ sortedTupleOffs.ensureCapacity(tupleCount);
+ for (int i = 0; i < tupleCount; i++) {
+ int slotOff = slotManager.getSlotOff(i);
+ int tupleOff = slotManager.getTupleOff(slotOff);
+ sortedTupleOffs.add(new SlotOffTupleOff(i, slotOff, tupleOff));
+ }
+ Collections.sort(sortedTupleOffs);
+ // Iterate over the sorted slots, and move their corresponding tuples to
+ // the left, reclaiming free space.
+ for (int i = 0; i < sortedTupleOffs.size(); i++) {
+ int tupleOff = sortedTupleOffs.get(i).tupleOff;
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ int tupleEndOff = frameTuple.getFieldStart(frameTuple.getFieldCount() - 1)
+ + frameTuple.getFieldLength(frameTuple.getFieldCount() - 1);
+ int tupleLength = tupleEndOff - tupleOff;
+ System.arraycopy(buf.array(), tupleOff, buf.array(), freeSpace, tupleLength);
+ slotManager.setSlot(sortedTupleOffs.get(i).slotOff, freeSpace);
+ freeSpace += tupleLength;
+ }
+ // Update contiguous free space pointer and total free space indicator.
+ buf.putInt(freeSpaceOff, freeSpace);
+ buf.putInt(totalFreeSpaceOff, buf.capacity() - freeSpace - tupleCount * slotManager.getSlotSize());
+ return false;
+ }
- @Override
- public ICachedPage getPage() {
- return page;
- }
+ @Override
+ public void delete(ITupleReference tuple, int tupleIndex) {
+ int slotOff = slotManager.getSlotOff(tupleIndex);
+ int tupleOff = slotManager.getTupleOff(slotOff);
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ int tupleSize = tupleWriter.bytesRequired(frameTuple);
- @Override
- public boolean compact(MultiComparator cmp) {
- resetSpaceParams();
- frameTuple.setFieldCount(cmp.getFieldCount());
+ // perform deletion (we just do a memcpy to overwrite the slot)
+ int slotStartOff = slotManager.getSlotEndOff();
+ int length = slotOff - slotStartOff;
+ System.arraycopy(buf.array(), slotStartOff, buf.array(), slotStartOff + slotManager.getSlotSize(), length);
- int tupleCount = buf.getInt(tupleCountOff);
- int freeSpace = buf.getInt(freeSpaceOff);
+ // maintain space information
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + tupleSize + slotManager.getSlotSize());
+ }
- ArrayList<SlotOffTupleOff> sortedTupleOffs = new ArrayList<SlotOffTupleOff>();
- sortedTupleOffs.ensureCapacity(tupleCount);
- for (int i = 0; i < tupleCount; i++) {
- int slotOff = slotManager.getSlotOff(i);
- int tupleOff = slotManager.getTupleOff(slotOff);
- sortedTupleOffs.add(new SlotOffTupleOff(i, slotOff, tupleOff));
- }
- Collections.sort(sortedTupleOffs);
+ @Override
+ public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) {
+ int bytesRequired = tupleWriter.bytesRequired(tuple);
+ // Enough space in the contiguous space region?
+ if (bytesRequired + slotManager.getSlotSize() <= buf.capacity() - buf.getInt(freeSpaceOff)
+ - (buf.getInt(tupleCountOff) * slotManager.getSlotSize())) {
+ return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
+ }
+ // Enough space after compaction?
+ if (bytesRequired + slotManager.getSlotSize() <= buf.getInt(totalFreeSpaceOff)) {
+ return FrameOpSpaceStatus.SUFFICIENT_SPACE;
+ }
+ return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
+ }
- for (int i = 0; i < sortedTupleOffs.size(); i++) {
- int tupleOff = sortedTupleOffs.get(i).tupleOff;
- frameTuple.resetByTupleOffset(buf, tupleOff);
+ @Override
+ public FrameOpSpaceStatus hasSpaceUpdate(ITupleReference newTuple, int oldTupleIndex) {
+ frameTuple.resetByTupleIndex(this, oldTupleIndex);
+ int oldTupleBytes = frameTuple.getTupleSize();
+ int newTupleBytes = tupleWriter.bytesRequired(newTuple);
+ int additionalBytesRequired = newTupleBytes - oldTupleBytes;
+ // Enough space for an in-place update?
+ if (additionalBytesRequired <= 0) {
+ return FrameOpSpaceStatus.SUFFICIENT_INPLACE_SPACE;
+ }
+ // Enough space if we delete the old tuple and insert the new one without compaction?
+ if (newTupleBytes <= buf.capacity() - buf.getInt(freeSpaceOff)
+ - (buf.getInt(tupleCountOff) * slotManager.getSlotSize())) {
+ return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
+ }
+ // Enough space if we delete the old tuple and compact?
+ if (additionalBytesRequired <= buf.getInt(totalFreeSpaceOff)) {
+ return FrameOpSpaceStatus.SUFFICIENT_SPACE;
+ }
+ return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
+ }
- int tupleEndOff = frameTuple.getFieldStart(frameTuple
- .getFieldCount() - 1)
- + frameTuple.getFieldLength(frameTuple.getFieldCount() - 1);
- int tupleLength = tupleEndOff - tupleOff;
- System.arraycopy(buf.array(), tupleOff, buf.array(), freeSpace,
- tupleLength);
+ protected void resetSpaceParams() {
+ buf.putInt(freeSpaceOff, smFlagOff + 1);
+ buf.putInt(totalFreeSpaceOff, buf.capacity() - (smFlagOff + 1));
+ }
- slotManager.setSlot(sortedTupleOffs.get(i).slotOff, freeSpace);
- freeSpace += tupleLength;
- }
+ @Override
+ public void insert(ITupleReference tuple, int tupleIndex) {
+ slotManager.insertSlot(tupleIndex, buf.getInt(freeSpaceOff));
+ int bytesWritten = tupleWriter.writeTuple(tuple, buf.array(), buf.getInt(freeSpaceOff));
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
+ buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - bytesWritten - slotManager.getSlotSize());
+ }
- buf.putInt(freeSpaceOff, freeSpace);
- buf.putInt(totalFreeSpaceOff, buf.capacity() - freeSpace - tupleCount
- * slotManager.getSlotSize());
+ @Override
+ public void update(ITupleReference newTuple, int oldTupleIndex, boolean inPlace) {
+ frameTuple.resetByTupleIndex(this, oldTupleIndex);
+ int oldTupleBytes = frameTuple.getTupleSize();
+ int slotOff = slotManager.getSlotOff(oldTupleIndex);
+ int bytesWritten = 0;
+ if (inPlace) {
+ // Overwrite the old tuple in place.
+ bytesWritten = tupleWriter.writeTuple(newTuple, buf.array(), buf.getInt(slotOff));
+ } else {
+ // Insert the new tuple at the end of the free space, and change the slot value (effectively "deleting" the old tuple).
+ int newTupleOff = buf.getInt(freeSpaceOff);
+ bytesWritten = tupleWriter.writeTuple(newTuple, buf.array(), newTupleOff);
+ // Update slot value.
+ buf.putInt(slotOff, newTupleOff);
+ // Update contiguous free space pointer.
+ buf.putInt(freeSpaceOff, newTupleOff + bytesWritten);
+ }
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + oldTupleBytes - bytesWritten);
+ }
- return false;
- }
+ @Override
+ public String printHeader() {
+ StringBuilder strBuilder = new StringBuilder();
+ strBuilder.append("pageLsnOff: " + pageLsnOff + "\n");
+ strBuilder.append("tupleCountOff: " + tupleCountOff + "\n");
+ strBuilder.append("freeSpaceOff: " + freeSpaceOff + "\n");
+ strBuilder.append("totalFreeSpaceOff: " + totalFreeSpaceOff + "\n");
+ strBuilder.append("levelOff: " + levelOff + "\n");
+ strBuilder.append("smFlagOff: " + smFlagOff + "\n");
+ return strBuilder.toString();
+ }
- @Override
- public void delete(ITupleReference tuple, MultiComparator cmp,
- boolean exactDelete) throws Exception {
+ @Override
+ public int getTupleCount() {
+ return buf.getInt(tupleCountOff);
+ }
- frameTuple.setFieldCount(cmp.getFieldCount());
- int tupleIndex = slotManager.findTupleIndex(tuple, frameTuple, cmp,
- FindTupleMode.FTM_EXACT,
- FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY);
- int slotOff = slotManager.getSlotOff(tupleIndex);
- if (tupleIndex < 0) {
- throw new TreeIndexException("Key to be deleted does not exist.");
- } else {
- if (exactDelete) {
- // check the non-key columns for equality by byte-by-byte
- // comparison
- int tupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.resetByTupleOffset(buf, tupleOff);
+ public ISlotManager getSlotManager() {
+ return slotManager;
+ }
- int comparison = cmp.fieldRangeCompare(tuple, frameTuple,
- cmp.getKeyFieldCount() - 1,
- cmp.getFieldCount() - cmp.getKeyFieldCount());
- if (comparison != 0) {
- throw new TreeIndexException(
- "Cannot delete tuple. Byte-by-byte comparison failed to prove equality.");
- }
- }
+ @Override
+ public int getTupleOffset(int slotNum) {
+ return slotManager.getTupleOff(slotManager.getSlotStartOff() - slotNum * slotManager.getSlotSize());
+ }
- int tupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.resetByTupleOffset(buf, tupleOff);
- int tupleSize = tupleWriter.bytesRequired(frameTuple);
+ @Override
+ public long getPageLsn() {
+ return buf.getLong(pageLsnOff);
+ }
- // perform deletion (we just do a memcpy to overwrite the slot)
- int slotStartOff = slotManager.getSlotEndOff();
- int length = slotOff - slotStartOff;
- System.arraycopy(buf.array(), slotStartOff, buf.array(),
- slotStartOff + slotManager.getSlotSize(), length);
+ @Override
+ public void setPageLsn(long pageLsn) {
+ buf.putLong(pageLsnOff, pageLsn);
+ }
- // maintain space information
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff)
- + tupleSize + slotManager.getSlotSize());
- }
- }
+ @Override
+ public int getTotalFreeSpace() {
+ return buf.getInt(totalFreeSpaceOff);
+ }
- @Override
- public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple,
- MultiComparator cmp) {
- int bytesRequired = tupleWriter.bytesRequired(tuple);
- if (bytesRequired + slotManager.getSlotSize() <= buf.capacity()
- - buf.getInt(freeSpaceOff)
- - (buf.getInt(tupleCountOff) * slotManager.getSlotSize()))
- return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
- else if (bytesRequired + slotManager.getSlotSize() <= buf
- .getInt(totalFreeSpaceOff))
- return FrameOpSpaceStatus.SUFFICIENT_SPACE;
- else
- return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
- }
+ @Override
+ public boolean compress() {
+ return false;
+ }
- @Override
- public FrameOpSpaceStatus hasSpaceUpdate(int rid, ITupleReference tuple,
- MultiComparator cmp) {
- // TODO Auto-generated method stub
- return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
- }
+ @Override
+ public int getSlotSize() {
+ return slotManager.getSlotSize();
+ }
- protected void resetSpaceParams() {
- buf.putInt(freeSpaceOff, smFlagOff + 1);
- buf.putInt(totalFreeSpaceOff, buf.capacity() - (smFlagOff + 1));
- }
-
- @Override
- public int findTupleIndex(ITupleReference tuple, MultiComparator cmp)
- throws Exception {
- frameTuple.setFieldCount(cmp.getFieldCount());
- return slotManager.findTupleIndex(tuple, frameTuple, cmp,
- FindTupleMode.FTM_INCLUSIVE,
- FindTupleNoExactMatchPolicy.FTP_HIGHER_KEY);
- }
-
- @Override
- public void insert(ITupleReference tuple, MultiComparator cmp,
- int tupleIndex) throws Exception {
- slotManager.insertSlot(tupleIndex, buf.getInt(freeSpaceOff));
- int bytesWritten = tupleWriter.writeTuple(tuple, buf,
- buf.getInt(freeSpaceOff));
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
- buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff)
- - bytesWritten - slotManager.getSlotSize());
- }
-
- @Override
- public void update(int rid, ITupleReference tuple) throws Exception {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public void printHeader() {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public int getTupleCount() {
- return buf.getInt(tupleCountOff);
- }
-
- public ISlotManager getSlotManager() {
- return slotManager;
- }
-
- @Override
- public String printKeys(MultiComparator cmp,
- ISerializerDeserializer[] fields) throws HyracksDataException {
- StringBuilder strBuilder = new StringBuilder();
- int tupleCount = buf.getInt(tupleCountOff);
- frameTuple.setFieldCount(fields.length);
- for (int i = 0; i < tupleCount; i++) {
- frameTuple.resetByTupleIndex(this, i);
- for (int j = 0; j < cmp.getKeyFieldCount(); j++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j),
- frameTuple.getFieldLength(j));
- DataInput dataIn = new DataInputStream(inStream);
- Object o = fields[j].deserialize(dataIn);
- strBuilder.append(o.toString() + " ");
- }
- strBuilder.append(" | ");
- }
- strBuilder.append("\n");
- return strBuilder.toString();
- }
-
- @Override
- public int getTupleOffset(int slotNum) {
- return slotManager.getTupleOff(slotManager.getSlotStartOff() - slotNum
- * slotManager.getSlotSize());
- }
-
- @Override
- public int getPageLsn() {
- return buf.getInt(pageLsnOff);
- }
-
- @Override
- public void setPageLsn(int pageLsn) {
- buf.putInt(pageLsnOff, pageLsn);
- }
-
- @Override
- public int getTotalFreeSpace() {
- return buf.getInt(totalFreeSpaceOff);
- }
-
- @Override
- public boolean compress(MultiComparator cmp) {
- return false;
- }
-
- @Override
- public int getSlotSize() {
- return slotManager.getSlotSize();
- }
-
- @Override
- public void setPageTupleFieldCount(int fieldCount) {
- frameTuple.setFieldCount(fieldCount);
- }
-
- public ITreeIndexTupleWriter getTupleWriter() {
- return tupleWriter;
+ @Override
+ public ITreeIndexTupleWriter getTupleWriter() {
+ return tupleWriter;
+ }
+
+ @Override
+ public ITreeIndexTupleReference createTupleReference() {
+ return tupleWriter.createTupleReference();
+ }
+
+ public int getFreeContiguousSpace() {
+ return buf.capacity() - getFreeSpaceOff()
+ - (getTupleCount() * slotManager.getSlotSize());
}
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
index 01b91b1..1a81c3f 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/impls/TreeDiskOrderScanCursor.java
@@ -21,7 +21,6 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
@@ -30,17 +29,17 @@
private int tupleIndex = 0;
private int fileId = -1;
- int currentPageId = -1;
- int maxPageId = -1;
- private ICachedPage page = null;
- private ITreeIndexFrame frame = null;
+ private int currentPageId = -1;
+ private int maxPageId = -1;
+ private ICachedPage page = null;
private IBufferCache bufferCache = null;
-
- private ITreeIndexTupleReference frameTuple;
-
+
+ private final ITreeIndexFrame frame;
+ private final ITreeIndexTupleReference frameTuple;
+
public TreeDiskOrderScanCursor(ITreeIndexFrame frame) {
- this.frame = frame;
- this.frameTuple = frame.getTupleWriter().createTupleReference();
+ this.frame = frame;
+ this.frameTuple = frame.createTupleReference();
}
@Override
@@ -62,8 +61,7 @@
private boolean positionToNextLeaf(boolean skipCurrent)
throws HyracksDataException {
- while ((frame.getLevel() != 0 || skipCurrent)
- && (currentPageId <= maxPageId) || (frame.getTupleCount() == 0)) {
+ while ((frame.getLevel() != 0 || skipCurrent) && (currentPageId <= maxPageId)) {
currentPageId++;
ICachedPage nextPage = bufferCache.pin(
@@ -86,7 +84,10 @@
}
@Override
- public boolean hasNext() throws Exception {
+ public boolean hasNext() throws Exception {
+ if (currentPageId > maxPageId) {
+ return false;
+ }
if (tupleIndex >= frame.getTupleCount()) {
boolean nextLeafExists = positionToNextLeaf(true);
if (nextLeafExists) {
@@ -95,9 +96,8 @@
} else {
return false;
}
- }
-
- frameTuple.resetByTupleIndex(frame, tupleIndex);
+ }
+ frameTuple.resetByTupleIndex(frame, tupleIndex);
return true;
}
@@ -114,17 +114,10 @@
page.releaseReadLatch();
bufferCache.unpin(page);
}
-
page = initialState.getPage();
- tupleIndex = 0;
+ tupleIndex = 0;
frame.setPage(page);
- MultiComparator lowKeyCmp = searchPred.getLowKeyComparator();
- frameTuple.setFieldCount(lowKeyCmp.getFieldCount());
- boolean leafExists = positionToNextLeaf(false);
- if (!leafExists) {
- throw new HyracksDataException(
- "Failed to open disk-order scan cursor for tree index. Traget tree index has no leaves.");
- }
+ positionToNextLeaf(false);
}
@Override
@@ -152,4 +145,9 @@
public void setMaxPageId(int maxPageId) {
this.maxPageId = maxPageId;
}
+
+ @Override
+ public boolean exclusiveLatchNodes() {
+ return false;
+ }
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/FindTupleMode.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/FindTupleMode.java
index 11ac257..5002189 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/FindTupleMode.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/FindTupleMode.java
@@ -16,5 +16,5 @@
package edu.uci.ics.hyracks.storage.am.common.ophelpers;
public enum FindTupleMode {
- FTM_INCLUSIVE, FTM_EXCLUSIVE, FTM_EXACT
+ INCLUSIVE, EXCLUSIVE, EXCLUSIVE_ERROR_IF_EXISTS, EXACT
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/FindTupleNoExactMatchPolicy.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/FindTupleNoExactMatchPolicy.java
index b8f3c9a..8b3f7f5 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/FindTupleNoExactMatchPolicy.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/FindTupleNoExactMatchPolicy.java
@@ -16,5 +16,5 @@
package edu.uci.ics.hyracks.storage.am.common.ophelpers;
public enum FindTupleNoExactMatchPolicy {
- FTP_LOWER_KEY, FTP_HIGHER_KEY
+ LOWER_KEY, HIGHER_KEY, NONE
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOpContext.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOpContext.java
deleted file mode 100644
index 9122174..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/IndexOpContext.java
+++ /dev/null
@@ -1,5 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.common.ophelpers;
-
-public interface IndexOpContext {
- void reset();
-}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/LongArrayList.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/LongArrayList.java
new file mode 100644
index 0000000..4dd1b5f
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/LongArrayList.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.ophelpers;
+
+public class LongArrayList {
+ private long[] data;
+ private int size;
+ private int first;
+ private final int growth;
+
+ public LongArrayList(int initialCapacity, int growth) {
+ data = new long[initialCapacity];
+ size = 0;
+ first = 0;
+ this.growth = growth;
+ }
+
+ public int size() {
+ return size;
+ }
+
+ public int first() {
+ return first;
+ }
+
+ public void add(long i) {
+ if (size == data.length) {
+ long[] newData = new long[data.length + growth];
+ System.arraycopy(data, 0, newData, 0, data.length);
+ data = newData;
+ }
+
+ data[size++] = i;
+ }
+
+ public void removeLast() {
+ if (size > 0)
+ size--;
+ }
+
+ // WARNING: caller is responsible for checking size > 0
+ public long getLast() {
+ return data[size - 1];
+ }
+
+ public long get(int i) {
+ return data[i];
+ }
+
+ // WARNING: caller is responsible for checking i < size
+ public void set(int i, long value) {
+ data[i] = value;
+
+ }
+
+ public long getFirst() {
+ return data[first];
+ }
+
+ public void moveFirst() {
+ first++;
+ }
+
+ public void clear() {
+ size = 0;
+ first = 0;
+ }
+
+ public boolean isLast() {
+ return size == first;
+ }
+
+ public boolean isEmpty() {
+ return size == 0;
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
index a07ef03..df24484 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/MultiComparator.java
@@ -15,45 +15,17 @@
package edu.uci.ics.hyracks.storage.am.common.ophelpers;
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
-
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
public class MultiComparator {
- private static final long serialVersionUID = 1L;
+ private final IBinaryComparator[] cmps;
- private IBinaryComparator[] cmps = null;
- private ITypeTrait[] typeTraits;
- private IPrimitiveValueProvider[] valueProviders = null;
-
- private IBinaryComparator intCmp = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
-
- public IBinaryComparator getIntCmp() {
- return intCmp;
- }
-
- public MultiComparator(ITypeTrait[] typeTraits, IBinaryComparator[] cmps) {
- this.typeTraits = typeTraits;
+ public MultiComparator(IBinaryComparator[] cmps) {
this.cmps = cmps;
}
- public MultiComparator(ITypeTrait[] typeTraits, IBinaryComparator[] cmps,
- IPrimitiveValueProvider[] valueProviders) {
- this.typeTraits = typeTraits;
- this.cmps = cmps;
- this.valueProviders = valueProviders;
- }
-
public int compare(ITupleReference tupleA, ITupleReference tupleB) {
for (int i = 0; i < cmps.length; i++) {
int cmp = cmps[i].compare(tupleA.getFieldData(i),
@@ -83,20 +55,6 @@
return 0;
}
- public String printTuple(ITupleReference tuple,
- ISerializerDeserializer[] fields) throws HyracksDataException {
- StringBuilder strBuilder = new StringBuilder();
- for (int i = 0; i < tuple.getFieldCount(); i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- tuple.getFieldData(i), tuple.getFieldStart(i),
- tuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- Object o = fields[i].deserialize(dataIn);
- strBuilder.append(o.toString() + " ");
- }
- return strBuilder.toString();
- }
-
public IBinaryComparator[] getComparators() {
return cmps;
}
@@ -104,21 +62,4 @@
public int getKeyFieldCount() {
return cmps.length;
}
-
- public void setComparators(IBinaryComparator[] cmps) {
- this.cmps = cmps;
- }
-
- public int getFieldCount() {
- return typeTraits.length;
- }
-
- public ITypeTrait[] getTypeTraits() {
- return typeTraits;
- }
-
- public IPrimitiveValueProvider[] getValueProviders() {
- return valueProviders;
- }
-
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/SlotOffTupleOff.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/SlotOffTupleOff.java
index 35231c2..7e9042c 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/SlotOffTupleOff.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/ophelpers/SlotOffTupleOff.java
@@ -30,4 +30,9 @@
public int compareTo(SlotOffTupleOff o) {
return tupleOff - o.tupleOff;
}
+
+ @Override
+ public String toString() {
+ return tupleIndex + " " + slotOff + " " + tupleOff;
+ }
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleReference.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleReference.java
index 353bd95..a470d04 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleReference.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleReference.java
@@ -22,77 +22,78 @@
public class SimpleTupleReference implements ITreeIndexTupleReference {
- protected ByteBuffer buf;
- protected int fieldStartIndex;
- protected int fieldCount;
- protected int tupleStartOff;
- protected int nullFlagsBytes;
- protected int fieldSlotsBytes;
+ protected ByteBuffer buf;
+ protected int fieldStartIndex;
+ protected int fieldCount;
+ protected int tupleStartOff;
+ protected int nullFlagsBytes;
+ protected int fieldSlotsBytes;
+
+ @Override
+ public void resetByTupleOffset(ByteBuffer buf, int tupleStartOff) {
+ this.buf = buf;
+ this.tupleStartOff = tupleStartOff;
+ }
+
+ @Override
+ public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
+ resetByTupleOffset(frame.getBuffer(), frame.getTupleOffset(tupleIndex));
+ }
+
+ @Override
+ public void setFieldCount(int fieldCount) {
+ this.fieldCount = fieldCount;
+ nullFlagsBytes = getNullFlagsBytes();
+ fieldSlotsBytes = getFieldSlotsBytes();
+ fieldStartIndex = 0;
+ }
+
+ @Override
+ public void setFieldCount(int fieldStartIndex, int fieldCount) {
+ this.fieldCount = fieldCount;
+ this.fieldStartIndex = fieldStartIndex;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return fieldCount;
+ }
+
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ return buf.array();
+ }
+
+ @Override
+ public int getFieldLength(int fIdx) {
+ if (fIdx == 0) {
+ return buf.getShort(tupleStartOff + nullFlagsBytes);
+ } else {
+ return buf.getShort(tupleStartOff + nullFlagsBytes + fIdx * 2)
+ - buf.getShort(tupleStartOff + nullFlagsBytes + ((fIdx - 1) * 2));
+ }
+ }
+
+ @Override
+ public int getFieldStart(int fIdx) {
+ if (fIdx == 0) {
+ return tupleStartOff + nullFlagsBytes + fieldSlotsBytes;
+ } else {
+ return tupleStartOff + nullFlagsBytes + fieldSlotsBytes
+ + buf.getShort(tupleStartOff + nullFlagsBytes + ((fIdx - 1) * 2));
+ }
+ }
+
+ protected int getNullFlagsBytes() {
+ return (int) Math.ceil(fieldCount / 8.0);
+ }
+
+ protected int getFieldSlotsBytes() {
+ return fieldCount * 2;
+ }
@Override
- public void resetByTupleOffset(ByteBuffer buf, int tupleStartOff) {
- this.buf = buf;
- this.tupleStartOff = tupleStartOff;
- }
-
- @Override
- public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
- resetByTupleOffset(frame.getBuffer(), frame.getTupleOffset(tupleIndex));
- }
-
- @Override
- public void setFieldCount(int fieldCount) {
- this.fieldCount = fieldCount;
- nullFlagsBytes = getNullFlagsBytes();
- fieldSlotsBytes = getFieldSlotsBytes();
- fieldStartIndex = 0;
- }
-
- @Override
- public void setFieldCount(int fieldStartIndex, int fieldCount) {
- this.fieldCount = fieldCount;
- this.fieldStartIndex = fieldStartIndex;
- }
-
- @Override
- public int getFieldCount() {
- return fieldCount;
- }
-
- @Override
- public byte[] getFieldData(int fIdx) {
- return buf.array();
- }
-
- @Override
- public int getFieldLength(int fIdx) {
- if (fIdx == 0) {
- return buf.getShort(tupleStartOff + nullFlagsBytes);
- } else {
- return buf.getShort(tupleStartOff + nullFlagsBytes + fIdx * 2)
- - buf.getShort(tupleStartOff + nullFlagsBytes
- + ((fIdx - 1) * 2));
- }
- }
-
- @Override
- public int getFieldStart(int fIdx) {
- if (fIdx == 0) {
- return tupleStartOff + nullFlagsBytes + fieldSlotsBytes;
- } else {
- return tupleStartOff
- + nullFlagsBytes
- + fieldSlotsBytes
- + buf.getShort(tupleStartOff + nullFlagsBytes
- + ((fIdx - 1) * 2));
- }
- }
-
- protected int getNullFlagsBytes() {
- return (int) Math.ceil(fieldCount / 8.0);
- }
-
- protected int getFieldSlotsBytes() {
- return fieldCount * 2;
+ public int getTupleSize() {
+ return nullFlagsBytes + fieldSlotsBytes + buf.getShort(tupleStartOff + nullFlagsBytes + (fieldCount-1) * 2);
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
index 11f7820..831247e 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/SimpleTupleWriter.java
@@ -23,95 +23,103 @@
public class SimpleTupleWriter implements ITreeIndexTupleWriter {
- @Override
- public int bytesRequired(ITupleReference tuple) {
- int bytes = getNullFlagsBytes(tuple) + getFieldSlotsBytes(tuple);
- for (int i = 0; i < tuple.getFieldCount(); i++) {
- bytes += tuple.getFieldLength(i);
- }
- return bytes;
+ // Write short in little endian to target byte array at given offset.
+ private static void writeShortL(short s, byte[] buf, int targetOff) {
+ buf[targetOff] = (byte)(s >> 8);
+ buf[targetOff + 1] = (byte)(s >> 0);
+ }
+
+ // Write short in big endian to target byte array at given offset.
+ private static void writeShortB(short s, byte[] buf, int targetOff) {
+ buf[targetOff] = (byte) (s >> 0);
+ buf[targetOff + 1] = (byte) (s >> 8);
+ }
+
+ @Override
+ public int bytesRequired(ITupleReference tuple) {
+ int bytes = getNullFlagsBytes(tuple) + getFieldSlotsBytes(tuple);
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ bytes += tuple.getFieldLength(i);
+ }
+ return bytes;
+ }
+
+ @Override
+ public int bytesRequired(ITupleReference tuple, int startField, int numFields) {
+ int bytes = getNullFlagsBytes(tuple, startField, numFields) + getFieldSlotsBytes(tuple, startField, numFields);
+ for (int i = startField; i < startField + numFields; i++) {
+ bytes += tuple.getFieldLength(i);
+ }
+ return bytes;
+ }
+
+ @Override
+ public ITreeIndexTupleReference createTupleReference() {
+ return new SimpleTupleReference();
+ }
+
+ @Override
+ public int writeTuple(ITupleReference tuple, ByteBuffer targetBuf, int targetOff) {
+ return writeTuple(tuple, targetBuf.array(), targetOff);
+ }
+
+ @Override
+ public int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff) {
+ int runner = targetOff;
+ int nullFlagsBytes = getNullFlagsBytes(tuple);
+ int fieldSlotsBytes = getFieldSlotsBytes(tuple);
+ for (int i = 0; i < nullFlagsBytes; i++) {
+ targetBuf[runner++] = (byte) 0;
+ }
+ runner += fieldSlotsBytes;
+ int fieldEndOff = 0;
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner,
+ tuple.getFieldLength(i));
+ fieldEndOff += tuple.getFieldLength(i);
+ runner += tuple.getFieldLength(i);
+ writeShortL((short) fieldEndOff, targetBuf, targetOff + nullFlagsBytes + i * 2);
+ }
+ return runner - targetOff;
}
- @Override
- public int bytesRequired(ITupleReference tuple, int startField,
- int numFields) {
- int bytes = getNullFlagsBytes(tuple, startField, numFields)
- + getFieldSlotsBytes(tuple, startField, numFields);
- for (int i = startField; i < startField + numFields; i++) {
- bytes += tuple.getFieldLength(i);
- }
- return bytes;
- }
+ @Override
+ public int writeTupleFields(ITupleReference tuple, int startField, int numFields, ByteBuffer targetBuf,
+ int targetOff) {
+ int runner = targetOff;
+ int nullFlagsBytes = getNullFlagsBytes(tuple, startField, numFields);
+ for (int i = 0; i < nullFlagsBytes; i++) {
+ targetBuf.put(runner++, (byte) 0);
+ }
+ runner += getFieldSlotsBytes(tuple, startField, numFields);
- @Override
- public ITreeIndexTupleReference createTupleReference() {
- return new SimpleTupleReference();
- }
+ int fieldEndOff = 0;
+ int fieldCounter = 0;
+ for (int i = startField; i < startField + numFields; i++) {
+ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf.array(), runner,
+ tuple.getFieldLength(i));
+ fieldEndOff += tuple.getFieldLength(i);
+ runner += tuple.getFieldLength(i);
+ targetBuf.putShort(targetOff + nullFlagsBytes + fieldCounter * 2, (short) fieldEndOff);
+ fieldCounter++;
+ }
- @Override
- public int writeTuple(ITupleReference tuple, ByteBuffer targetBuf,
- int targetOff) {
- int runner = targetOff;
- int nullFlagsBytes = getNullFlagsBytes(tuple);
- int fieldSlotsBytes = getFieldSlotsBytes(tuple);
- for (int i = 0; i < nullFlagsBytes; i++) {
- targetBuf.put(runner++, (byte) 0);
- }
- runner += fieldSlotsBytes;
+ return runner - targetOff;
+ }
- int fieldEndOff = 0;
- for (int i = 0; i < tuple.getFieldCount(); i++) {
- System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i),
- targetBuf.array(), runner, tuple.getFieldLength(i));
- fieldEndOff += tuple.getFieldLength(i);
- runner += tuple.getFieldLength(i);
- targetBuf.putShort(targetOff + nullFlagsBytes + i * 2,
- (short) fieldEndOff);
- }
+ protected int getNullFlagsBytes(ITupleReference tuple) {
+ return (int) Math.ceil((double) tuple.getFieldCount() / 8.0);
+ }
- return runner - targetOff;
- }
+ protected int getFieldSlotsBytes(ITupleReference tuple) {
+ return tuple.getFieldCount() * 2;
+ }
- @Override
- public int writeTupleFields(ITupleReference tuple, int startField,
- int numFields, ByteBuffer targetBuf, int targetOff) {
- int runner = targetOff;
- int nullFlagsBytes = getNullFlagsBytes(tuple, startField, numFields);
- for (int i = 0; i < nullFlagsBytes; i++) {
- targetBuf.put(runner++, (byte) 0);
- }
- runner += getFieldSlotsBytes(tuple, startField, numFields);
+ protected int getNullFlagsBytes(ITupleReference tuple, int startField, int numFields) {
+ return (int) Math.ceil((double) numFields / 8.0);
+ }
- int fieldEndOff = 0;
- int fieldCounter = 0;
- for (int i = startField; i < startField + numFields; i++) {
- System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i),
- targetBuf.array(), runner, tuple.getFieldLength(i));
- fieldEndOff += tuple.getFieldLength(i);
- runner += tuple.getFieldLength(i);
- targetBuf.putShort(targetOff + nullFlagsBytes + fieldCounter * 2,
- (short) fieldEndOff);
- fieldCounter++;
- }
-
- return runner - targetOff;
- }
-
- protected int getNullFlagsBytes(ITupleReference tuple) {
- return (int) Math.ceil((double) tuple.getFieldCount() / 8.0);
- }
-
- protected int getFieldSlotsBytes(ITupleReference tuple) {
- return tuple.getFieldCount() * 2;
- }
-
- protected int getNullFlagsBytes(ITupleReference tuple, int startField,
- int numFields) {
- return (int) Math.ceil((double) numFields / 8.0);
- }
-
- protected int getFieldSlotsBytes(ITupleReference tuple, int startField,
- int numFields) {
- return numFields * 2;
- }
+ protected int getFieldSlotsBytes(ITupleReference tuple, int startField, int numFields) {
+ return numFields * 2;
+ }
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleReference.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleReference.java
index 31b32e6..4776bdd 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleReference.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleReference.java
@@ -17,104 +17,109 @@
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
public class TypeAwareTupleReference implements ITreeIndexTupleReference {
- protected ByteBuffer buf;
- protected int fieldStartIndex;
- protected int fieldCount;
- protected int tupleStartOff;
- protected int nullFlagsBytes;
- protected int dataStartOff;
+ protected ByteBuffer buf;
+ protected int fieldStartIndex;
+ protected int fieldCount;
+ protected int tupleStartOff;
+ protected int nullFlagsBytes;
+ protected int dataStartOff;
- protected ITypeTrait[] typeTraits;
- protected VarLenIntEncoderDecoder encDec = new VarLenIntEncoderDecoder();
- protected int[] decodedFieldSlots;
+ protected ITypeTraits[] typeTraits;
+ protected VarLenIntEncoderDecoder encDec = new VarLenIntEncoderDecoder();
+ protected int[] decodedFieldSlots;
- public TypeAwareTupleReference(ITypeTrait[] typeTraits) {
- this.typeTraits = typeTraits;
- this.fieldStartIndex = 0;
- }
+ public TypeAwareTupleReference(ITypeTraits[] typeTraits) {
+ this.typeTraits = typeTraits;
+ this.fieldStartIndex = 0;
+ setFieldCount(typeTraits.length);
+ }
- @Override
- public void resetByTupleOffset(ByteBuffer buf, int tupleStartOff) {
- this.buf = buf;
- this.tupleStartOff = tupleStartOff;
+ @Override
+ public void resetByTupleOffset(ByteBuffer buf, int tupleStartOff) {
+ this.buf = buf;
+ this.tupleStartOff = tupleStartOff;
- // decode field slots
- int field = 0;
- int cumul = 0;
- int end = fieldStartIndex + fieldCount;
- encDec.reset(buf.array(), tupleStartOff + nullFlagsBytes);
- for (int i = fieldStartIndex; i < end; i++) {
- int staticDataLen = typeTraits[i].getStaticallyKnownDataLength();
- if (staticDataLen == ITypeTrait.VARIABLE_LENGTH) {
- cumul += encDec.decode();
- decodedFieldSlots[field++] = cumul;
- } else {
- cumul += staticDataLen;
- decodedFieldSlots[field++] = cumul;
- }
- }
- dataStartOff = encDec.getPos();
- }
+ // decode field slots
+ int field = 0;
+ int cumul = 0;
+ int end = fieldStartIndex + fieldCount;
+ encDec.reset(buf.array(), tupleStartOff + nullFlagsBytes);
+ for (int i = fieldStartIndex; i < end; i++) {
+ if (!typeTraits[i].isFixedLength()) {
+ cumul += encDec.decode();
+ decodedFieldSlots[field++] = cumul;
+ } else {
+ cumul += typeTraits[i].getFixedLength();
+ decodedFieldSlots[field++] = cumul;
+ }
+ }
+ dataStartOff = encDec.getPos();
+ }
- @Override
- public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
- resetByTupleOffset(frame.getBuffer(), frame.getTupleOffset(tupleIndex));
- }
+ @Override
+ public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
+ resetByTupleOffset(frame.getBuffer(), frame.getTupleOffset(tupleIndex));
+ }
- @Override
- public void setFieldCount(int fieldCount) {
- this.fieldCount = fieldCount;
- if (decodedFieldSlots == null) {
- decodedFieldSlots = new int[fieldCount];
- } else {
- if (fieldCount > decodedFieldSlots.length) {
- decodedFieldSlots = new int[fieldCount];
- }
- }
- nullFlagsBytes = getNullFlagsBytes();
- this.fieldStartIndex = 0;
- }
+ @Override
+ public void setFieldCount(int fieldCount) {
+ this.fieldCount = fieldCount;
+ if (decodedFieldSlots == null) {
+ decodedFieldSlots = new int[fieldCount];
+ } else {
+ if (fieldCount > decodedFieldSlots.length) {
+ decodedFieldSlots = new int[fieldCount];
+ }
+ }
+ nullFlagsBytes = getNullFlagsBytes();
+ this.fieldStartIndex = 0;
+ }
- @Override
- public void setFieldCount(int fieldStartIndex, int fieldCount) {
- setFieldCount(fieldCount);
- this.fieldStartIndex = fieldStartIndex;
- }
+ @Override
+ public void setFieldCount(int fieldStartIndex, int fieldCount) {
+ setFieldCount(fieldCount);
+ this.fieldStartIndex = fieldStartIndex;
+ }
- @Override
- public int getFieldCount() {
- return fieldCount;
- }
+ @Override
+ public int getFieldCount() {
+ return fieldCount;
+ }
- @Override
- public byte[] getFieldData(int fIdx) {
- return buf.array();
- }
+ @Override
+ public byte[] getFieldData(int fIdx) {
+ return buf.array();
+ }
- @Override
- public int getFieldLength(int fIdx) {
- if (fIdx == 0) {
- return decodedFieldSlots[0];
- } else {
- return decodedFieldSlots[fIdx] - decodedFieldSlots[fIdx - 1];
- }
- }
+ @Override
+ public int getFieldLength(int fIdx) {
+ if (fIdx == 0) {
+ return decodedFieldSlots[0];
+ } else {
+ return decodedFieldSlots[fIdx] - decodedFieldSlots[fIdx - 1];
+ }
+ }
- @Override
- public int getFieldStart(int fIdx) {
- if (fIdx == 0) {
- return dataStartOff;
- } else {
- return dataStartOff + decodedFieldSlots[fIdx - 1];
- }
- }
+ @Override
+ public int getFieldStart(int fIdx) {
+ if (fIdx == 0) {
+ return dataStartOff;
+ } else {
+ return dataStartOff + decodedFieldSlots[fIdx - 1];
+ }
+ }
- protected int getNullFlagsBytes() {
- return (int) Math.ceil(fieldCount / 8.0);
- }
+ protected int getNullFlagsBytes() {
+ return (int) Math.ceil(fieldCount / 8.0);
+ }
+
+ @Override
+ public int getTupleSize() {
+ return dataStartOff - tupleStartOff + decodedFieldSlots[fieldCount - 1];
+ }
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
index 95468d4..fe52608 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriter.java
@@ -17,138 +17,138 @@
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
public class TypeAwareTupleWriter implements ITreeIndexTupleWriter {
- protected ITypeTrait[] typeTraits;
- protected VarLenIntEncoderDecoder encDec = new VarLenIntEncoderDecoder();
+ protected ITypeTraits[] typeTraits;
+ protected VarLenIntEncoderDecoder encDec = new VarLenIntEncoderDecoder();
- public TypeAwareTupleWriter(ITypeTrait[] typeTraits) {
- this.typeTraits = typeTraits;
- }
+ public TypeAwareTupleWriter(ITypeTraits[] typeTraits) {
+ this.typeTraits = typeTraits;
+ }
- @Override
- public int bytesRequired(ITupleReference tuple) {
- int bytes = getNullFlagsBytes(tuple) + getFieldSlotsBytes(tuple);
- for (int i = 0; i < tuple.getFieldCount(); i++) {
- bytes += tuple.getFieldLength(i);
- }
- return bytes;
- }
+ @Override
+ public int bytesRequired(ITupleReference tuple) {
+ int bytes = getNullFlagsBytes(tuple) + getFieldSlotsBytes(tuple);
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ bytes += tuple.getFieldLength(i);
+ }
+ return bytes;
+ }
- @Override
- public int bytesRequired(ITupleReference tuple, int startField,
- int numFields) {
- int bytes = getNullFlagsBytes(numFields)
- + getFieldSlotsBytes(tuple, startField, numFields);
- for (int i = startField; i < startField + numFields; i++) {
- bytes += tuple.getFieldLength(i);
- }
- return bytes;
- }
+ @Override
+ public int bytesRequired(ITupleReference tuple, int startField, int numFields) {
+ int bytes = getNullFlagsBytes(numFields) + getFieldSlotsBytes(tuple, startField, numFields);
+ for (int i = startField; i < startField + numFields; i++) {
+ bytes += tuple.getFieldLength(i);
+ }
+ return bytes;
+ }
- @Override
- public ITreeIndexTupleReference createTupleReference() {
- return new TypeAwareTupleReference(typeTraits);
- }
+ @Override
+ public ITreeIndexTupleReference createTupleReference() {
+ return new TypeAwareTupleReference(typeTraits);
+ }
- @Override
- public int writeTuple(ITupleReference tuple, ByteBuffer targetBuf,
- int targetOff) {
- int runner = targetOff;
- int nullFlagsBytes = getNullFlagsBytes(tuple);
- // write null indicator bits
- for (int i = 0; i < nullFlagsBytes; i++) {
- targetBuf.put(runner++, (byte) 0);
- }
+ @Override
+ public int writeTuple(ITupleReference tuple, ByteBuffer targetBuf, int targetOff) {
+ return writeTuple(tuple, targetBuf.array(), targetOff);
+ }
- // write field slots for variable length fields
- encDec.reset(targetBuf.array(), runner);
- for (int i = 0; i < tuple.getFieldCount(); i++) {
- if (typeTraits[i].getStaticallyKnownDataLength() == ITypeTrait.VARIABLE_LENGTH) {
- encDec.encode(tuple.getFieldLength(i));
- }
- }
- runner = encDec.getPos();
+ @Override
+ public int writeTuple(ITupleReference tuple, byte[] targetBuf, int targetOff) {
+ int runner = targetOff;
+ int nullFlagsBytes = getNullFlagsBytes(tuple);
+ // write null indicator bits
+ for (int i = 0; i < nullFlagsBytes; i++) {
+ targetBuf[runner++] = (byte) 0;
+ }
- // write data fields
- for (int i = 0; i < tuple.getFieldCount(); i++) {
- System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i),
- targetBuf.array(), runner, tuple.getFieldLength(i));
- runner += tuple.getFieldLength(i);
- }
+ // write field slots for variable length fields
+ encDec.reset(targetBuf, runner);
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ if (!typeTraits[i].isFixedLength()) {
+ encDec.encode(tuple.getFieldLength(i));
+ }
+ }
+ runner = encDec.getPos();
- return runner - targetOff;
- }
+ // write data fields
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ int s = tuple.getFieldStart(i);
+ int l = tuple.getFieldLength(i);
+ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf, runner, tuple.getFieldLength(i));
+ runner += tuple.getFieldLength(i);
+ }
- @Override
- public int writeTupleFields(ITupleReference tuple, int startField,
- int numFields, ByteBuffer targetBuf, int targetOff) {
- int runner = targetOff;
- int nullFlagsBytes = getNullFlagsBytes(numFields);
- // write null indicator bits
- for (int i = 0; i < nullFlagsBytes; i++) {
- targetBuf.put(runner++, (byte) 0);
- }
+ return runner - targetOff;
+ }
- // write field slots for variable length fields
- encDec.reset(targetBuf.array(), runner);
- for (int i = startField; i < startField + numFields; i++) {
- if (typeTraits[i].getStaticallyKnownDataLength() == ITypeTrait.VARIABLE_LENGTH) {
- encDec.encode(tuple.getFieldLength(i));
- }
- }
- runner = encDec.getPos();
+ @Override
+ public int writeTupleFields(ITupleReference tuple, int startField, int numFields, ByteBuffer targetBuf,
+ int targetOff) {
+ int runner = targetOff;
+ int nullFlagsBytes = getNullFlagsBytes(numFields);
+ // write null indicator bits
+ for (int i = 0; i < nullFlagsBytes; i++) {
+ targetBuf.put(runner++, (byte) 0);
+ }
- for (int i = startField; i < startField + numFields; i++) {
- System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i),
- targetBuf.array(), runner, tuple.getFieldLength(i));
- runner += tuple.getFieldLength(i);
- }
+ // write field slots for variable length fields
+ encDec.reset(targetBuf.array(), runner);
+ for (int i = startField; i < startField + numFields; i++) {
+ if (!typeTraits[i].isFixedLength()) {
+ encDec.encode(tuple.getFieldLength(i));
+ }
+ }
+ runner = encDec.getPos();
- return runner - targetOff;
- }
+ for (int i = startField; i < startField + numFields; i++) {
+ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), targetBuf.array(), runner,
+ tuple.getFieldLength(i));
+ runner += tuple.getFieldLength(i);
+ }
- protected int getNullFlagsBytes(ITupleReference tuple) {
- return (int) Math.ceil((double) tuple.getFieldCount() / 8.0);
- }
+ return runner - targetOff;
+ }
- protected int getFieldSlotsBytes(ITupleReference tuple) {
- int fieldSlotBytes = 0;
- for (int i = 0; i < tuple.getFieldCount(); i++) {
- if (typeTraits[i].getStaticallyKnownDataLength() == ITypeTrait.VARIABLE_LENGTH) {
- fieldSlotBytes += encDec.getBytesRequired(tuple
- .getFieldLength(i));
- }
- }
- return fieldSlotBytes;
- }
+ protected int getNullFlagsBytes(ITupleReference tuple) {
+ return (int) Math.ceil((double) tuple.getFieldCount() / 8.0);
+ }
- protected int getNullFlagsBytes(int numFields) {
- return (int) Math.ceil((double) numFields / 8.0);
- }
+ protected int getFieldSlotsBytes(ITupleReference tuple) {
+ int fieldSlotBytes = 0;
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ if (!typeTraits[i].isFixedLength()) {
+ fieldSlotBytes += encDec.getBytesRequired(tuple.getFieldLength(i));
+ }
+ }
+ return fieldSlotBytes;
+ }
- protected int getFieldSlotsBytes(ITupleReference tuple, int startField,
- int numFields) {
- int fieldSlotBytes = 0;
- for (int i = startField; i < startField + numFields; i++) {
- if (typeTraits[i].getStaticallyKnownDataLength() == ITypeTrait.VARIABLE_LENGTH) {
- fieldSlotBytes += encDec.getBytesRequired(tuple
- .getFieldLength(i));
- }
- }
- return fieldSlotBytes;
- }
+ protected int getNullFlagsBytes(int numFields) {
+ return (int) Math.ceil((double) numFields / 8.0);
+ }
- public ITypeTrait[] getTypeTraits() {
- return typeTraits;
- }
+ protected int getFieldSlotsBytes(ITupleReference tuple, int startField, int numFields) {
+ int fieldSlotBytes = 0;
+ for (int i = startField; i < startField + numFields; i++) {
+ if (!typeTraits[i].isFixedLength()) {
+ fieldSlotBytes += encDec.getBytesRequired(tuple.getFieldLength(i));
+ }
+ }
+ return fieldSlotBytes;
+ }
- public void setTypeTraits(ITypeTrait[] typeTraits) {
- this.typeTraits = typeTraits;
- }
+ public ITypeTraits[] getTypeTraits() {
+ return typeTraits;
+ }
+
+ public void setTypeTraits(ITypeTraits[] typeTraits) {
+ this.typeTraits = typeTraits;
+ }
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriterFactory.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriterFactory.java
index 28e2535..9e6ba6f 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriterFactory.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/tuples/TypeAwareTupleWriterFactory.java
@@ -15,7 +15,7 @@
package edu.uci.ics.hyracks.storage.am.common.tuples;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
@@ -23,9 +23,9 @@
ITreeIndexTupleWriterFactory {
private static final long serialVersionUID = 1L;
- private ITypeTrait[] typeTraits;
+ private ITypeTraits[] typeTraits;
- public TypeAwareTupleWriterFactory(ITypeTrait[] typeTraits) {
+ public TypeAwareTupleWriterFactory(ITypeTraits[] typeTraits) {
this.typeTraits = typeTraits;
}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/IndexUtils.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/IndexUtils.java
new file mode 100644
index 0000000..389855f
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/IndexUtils.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.common.util;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+public class IndexUtils {
+ public static MultiComparator createMultiComparator(IBinaryComparatorFactory[] cmpFactories) {
+ IBinaryComparator[] cmps = new IBinaryComparator[cmpFactories.length];
+ for (int i = 0; i < cmpFactories.length; i++) {
+ cmps[i] = cmpFactories[i].createBinaryComparator();
+ }
+ return new MultiComparator(cmps);
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
new file mode 100644
index 0000000..65ea8de
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexBufferCacheWarmup.java
@@ -0,0 +1,88 @@
+package edu.uci.ics.hyracks.storage.am.common.util;
+
+import java.util.ArrayList;
+import java.util.Random;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.IntArrayList;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class TreeIndexBufferCacheWarmup {
+ private final IBufferCache bufferCache;
+ private final IFreePageManager freePageManager;
+ private final int fileId;
+ private final ArrayList<IntArrayList> pagesByLevel = new ArrayList<IntArrayList>();
+ private final Random rnd = new Random();
+
+ public TreeIndexBufferCacheWarmup(IBufferCache bufferCache,
+ IFreePageManager freePageManager, int fileId) {
+ this.bufferCache = bufferCache;
+ this.freePageManager = freePageManager;
+ this.fileId = fileId;
+ }
+
+ public void warmup(ITreeIndexFrame frame,
+ ITreeIndexMetaDataFrame metaFrame, int[] warmupTreeLevels,
+ int[] warmupRepeats) throws HyracksDataException {
+ bufferCache.openFile(fileId);
+
+ // scan entire file to determine pages in each level
+ int maxPageId = freePageManager.getMaxPage(metaFrame);
+ for (int pageId = 0; pageId <= maxPageId; pageId++) {
+ ICachedPage page = bufferCache.pin(
+ BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ page.acquireReadLatch();
+ try {
+ frame.setPage(page);
+ byte level = frame.getLevel();
+ while (level >= pagesByLevel.size()) {
+ pagesByLevel.add(new IntArrayList(100, 100));
+ }
+ if (level >= 0) {
+ // System.out.println("ADDING: " + level + " " + pageId);
+ pagesByLevel.get(level).add(pageId);
+ }
+ } finally {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ }
+ }
+
+ // pin certain pages again to simulate frequent access
+ for (int i = 0; i < warmupTreeLevels.length; i++) {
+ if (warmupTreeLevels[i] < pagesByLevel.size()) {
+ int repeats = warmupRepeats[i];
+ IntArrayList pageIds = pagesByLevel.get(warmupTreeLevels[i]);
+ int[] remainingPageIds = new int[pageIds.size()];
+ for (int r = 0; r < repeats; r++) {
+ for (int j = 0; j < pageIds.size(); j++) {
+ remainingPageIds[j] = pageIds.get(j);
+ }
+
+ int remainingLength = pageIds.size();
+ for (int j = 0; j < pageIds.size(); j++) {
+ int index = Math.abs(rnd.nextInt()) % remainingLength;
+ int pageId = remainingPageIds[index];
+
+ // pin & latch then immediately unlatch & unpin
+ ICachedPage page = bufferCache.pin(BufferedFileHandle
+ .getDiskPageId(fileId, pageId), false);
+ page.acquireReadLatch();
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+
+ remainingPageIds[index] = remainingPageIds[remainingLength - 1];
+ remainingLength--;
+ }
+ }
+ }
+ }
+
+ bufferCache.closeFile(fileId);
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexStats.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexStats.java
new file mode 100644
index 0000000..d5d9b5d
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexStats.java
@@ -0,0 +1,147 @@
+package edu.uci.ics.hyracks.storage.am.common.util;
+
+import java.text.DecimalFormat;
+
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+
+public class TreeIndexStats {
+
+ private TreeIndexNodeTypeStats rootStats = new TreeIndexNodeTypeStats();
+ private TreeIndexNodeTypeStats interiorStats = new TreeIndexNodeTypeStats();
+ private TreeIndexNodeTypeStats leafStats = new TreeIndexNodeTypeStats();
+
+ private int freePages = 0;
+ private int metaPages = 0;
+ private int treeLevels = 0;
+
+ public void begin() {
+ rootStats.clear();
+ interiorStats.clear();
+ leafStats.clear();
+ freePages = 0;
+ metaPages = 0;
+ treeLevels = 0;
+ }
+
+ public void addRoot(ITreeIndexFrame frame) {
+ treeLevels = frame.getLevel() + 1;
+ rootStats.add(frame);
+ }
+
+ public void add(ITreeIndexFrame frame) {
+ if (frame.isLeaf()) {
+ leafStats.add(frame);
+ } else if (frame.isInterior()) {
+ interiorStats.add(frame);
+ }
+ }
+
+ public void add(ITreeIndexMetaDataFrame metaFrame,
+ IFreePageManager freePageManager) {
+ if (freePageManager.isFreePage(metaFrame)) {
+ freePages++;
+ } else if (freePageManager.isMetaPage(metaFrame)) {
+ metaPages++;
+ }
+ }
+
+ public void end() {
+ // nothing here currently
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder strBuilder = new StringBuilder();
+ DecimalFormat df = new DecimalFormat("#####.##");
+
+ strBuilder.append("TREE LEVELS: " + treeLevels + "\n");
+ strBuilder.append("FREE PAGES : " + freePages + "\n");
+ strBuilder.append("META PAGES : " + metaPages + "\n");
+ long totalPages = interiorStats.getNumPages() + leafStats.getNumPages()
+ + freePages + metaPages;
+ strBuilder.append("TOTAL PAGES : " + totalPages + "\n");
+
+ strBuilder.append("\n");
+ strBuilder.append("ROOT STATS" + "\n");
+ strBuilder
+ .append("NUM TUPLES: " + rootStats.getNumTuples() + "\n");
+ strBuilder.append("FILL FACTOR : "
+ + df.format(rootStats.getAvgFillFactor()) + "\n");
+
+ if (interiorStats.getNumPages() > 0) {
+ strBuilder.append("\n");
+ strBuilder.append("INTERIOR STATS" + "\n");
+ strBuilder.append("NUM PAGES: " + interiorStats.getNumPages()
+ + "\n");
+ strBuilder.append("NUM TUPLES: "
+ + interiorStats.getNumTuples() + "\n");
+ strBuilder.append("AVG TUPLES/PAGE: "
+ + df.format(interiorStats.getAvgNumTuples()) + "\n");
+ strBuilder.append("AVG FILL FACTOR: "
+ + df.format(interiorStats.getAvgFillFactor()) + "\n");
+ }
+
+ if (leafStats.getNumPages() > 0) {
+ strBuilder.append("\n");
+ strBuilder.append("LEAF STATS" + "\n");
+ strBuilder.append("NUM PAGES: "
+ + df.format(leafStats.getNumPages()) + "\n");
+ strBuilder.append("NUM TUPLES: "
+ + df.format(leafStats.getNumTuples()) + "\n");
+ strBuilder.append("AVG TUPLES/PAGE: "
+ + df.format(leafStats.getAvgNumTuples()) + "\n");
+ strBuilder.append("AVG FILL FACTOR: "
+ + df.format(leafStats.getAvgFillFactor()) + "\n");
+ }
+
+ return strBuilder.toString();
+ }
+
+ public class TreeIndexNodeTypeStats {
+ private long numTuples;
+ private long sumTuplesSizes;
+ private long numPages;
+ private double sumFillFactors;
+
+ public void clear() {
+ numTuples = 0;
+ sumTuplesSizes = 0;
+ numPages = 0;
+ }
+
+ public void add(ITreeIndexFrame frame) {
+ numPages++;
+ numTuples += frame.getTupleCount();
+ sumFillFactors += (double) (frame.getBuffer().capacity() - frame
+ .getTotalFreeSpace())
+ / (double) frame.getBuffer().capacity();
+ }
+
+ public long getNumTuples() {
+ return numTuples;
+ }
+
+ public long getSumTupleSizes() {
+ return sumTuplesSizes;
+ }
+
+ public long getNumPages() {
+ return numPages;
+ }
+
+ public double getAvgNumTuples() {
+ return (double) numTuples / (double) numPages;
+ }
+
+ public double getAvgTupleSize() {
+ return (double) sumTuplesSizes / (double) numTuples;
+ }
+
+ public double getAvgFillFactor() {
+ return sumFillFactors / numPages;
+ }
+ }
+
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
new file mode 100644
index 0000000..eeacccd
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexStatsGatherer.java
@@ -0,0 +1,73 @@
+package edu.uci.ics.hyracks.storage.am.common.util;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class TreeIndexStatsGatherer {
+
+ private final TreeIndexStats treeIndexStats = new TreeIndexStats();
+ private final IBufferCache bufferCache;
+ private final IFreePageManager freePageManager;
+ private final int fileId;
+ private final int rootPage;
+
+ public TreeIndexStatsGatherer(IBufferCache bufferCache,
+ IFreePageManager freePageManager, int fileId, int rootPage) {
+ this.bufferCache = bufferCache;
+ this.freePageManager = freePageManager;
+ this.fileId = fileId;
+ this.rootPage = rootPage;
+ }
+
+ public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame,
+ ITreeIndexFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame)
+ throws HyracksDataException {
+
+ bufferCache.openFile(fileId);
+
+ treeIndexStats.begin();
+
+ int maxPageId = freePageManager.getMaxPage(metaFrame);
+ for (int pageId = 0; pageId <= maxPageId; pageId++) {
+ ICachedPage page = bufferCache.pin(
+ BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ page.acquireReadLatch();
+ try {
+ metaFrame.setPage(page);
+ leafFrame.setPage(page);
+ interiorFrame.setPage(page);
+
+ if (leafFrame.isLeaf()) {
+ if (pageId == rootPage) {
+ treeIndexStats.addRoot(leafFrame);
+ } else {
+ treeIndexStats.add(leafFrame);
+ }
+ } else if (interiorFrame.isInterior()) {
+ if (pageId == rootPage) {
+ treeIndexStats.addRoot(interiorFrame);
+ } else {
+ treeIndexStats.add(interiorFrame);
+ }
+ } else {
+ treeIndexStats.add(metaFrame, freePageManager);
+ }
+
+ } finally {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ }
+ }
+
+ treeIndexStats.end();
+
+ bufferCache.closeFile(fileId);
+
+ return treeIndexStats;
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexUtils.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexUtils.java
new file mode 100644
index 0000000..a1e493d
--- /dev/null
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/util/TreeIndexUtils.java
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.common.util;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
+
+@SuppressWarnings("rawtypes")
+public class TreeIndexUtils {
+ public static String printFrameTuples(ITreeIndexFrame frame, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
+ StringBuilder strBuilder = new StringBuilder();
+ ITreeIndexTupleReference tuple = frame.createTupleReference();
+ for (int i = 0; i < frame.getTupleCount(); i++) {
+ tuple.resetByTupleIndex(frame, i);
+ String tupleString = TupleUtils.printTuple(tuple, fieldSerdes);
+ strBuilder.append(tupleString);
+ if (i != frame.getTupleCount() - 1) {
+ strBuilder.append(" | ");
+ }
+ }
+ return strBuilder.toString();
+ }
+}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/utility/TreeIndexBufferCacheWarmup.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/utility/TreeIndexBufferCacheWarmup.java
deleted file mode 100644
index 8179d58..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/utility/TreeIndexBufferCacheWarmup.java
+++ /dev/null
@@ -1,88 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.common.utility;
-
-import java.util.ArrayList;
-import java.util.Random;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IntArrayList;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
-import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
-
-public class TreeIndexBufferCacheWarmup {
- private final IBufferCache bufferCache;
- private final IFreePageManager freePageManager;
- private final int fileId;
- private final ArrayList<IntArrayList> pagesByLevel = new ArrayList<IntArrayList>();
- private final Random rnd = new Random();
-
- public TreeIndexBufferCacheWarmup(IBufferCache bufferCache,
- IFreePageManager freePageManager, int fileId) {
- this.bufferCache = bufferCache;
- this.freePageManager = freePageManager;
- this.fileId = fileId;
- }
-
- public void warmup(ITreeIndexFrame frame,
- ITreeIndexMetaDataFrame metaFrame, int[] warmupTreeLevels,
- int[] warmupRepeats) throws HyracksDataException {
- bufferCache.openFile(fileId);
-
- // scan entire file to determine pages in each level
- int maxPageId = freePageManager.getMaxPage(metaFrame);
- for (int pageId = 0; pageId <= maxPageId; pageId++) {
- ICachedPage page = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- page.acquireReadLatch();
- try {
- frame.setPage(page);
- byte level = frame.getLevel();
- while (level >= pagesByLevel.size()) {
- pagesByLevel.add(new IntArrayList(100, 100));
- }
- if (level >= 0) {
- // System.out.println("ADDING: " + level + " " + pageId);
- pagesByLevel.get(level).add(pageId);
- }
- } finally {
- page.releaseReadLatch();
- bufferCache.unpin(page);
- }
- }
-
- // pin certain pages again to simulate frequent access
- for (int i = 0; i < warmupTreeLevels.length; i++) {
- if (warmupTreeLevels[i] < pagesByLevel.size()) {
- int repeats = warmupRepeats[i];
- IntArrayList pageIds = pagesByLevel.get(warmupTreeLevels[i]);
- int[] remainingPageIds = new int[pageIds.size()];
- for (int r = 0; r < repeats; r++) {
- for (int j = 0; j < pageIds.size(); j++) {
- remainingPageIds[j] = pageIds.get(j);
- }
-
- int remainingLength = pageIds.size();
- for (int j = 0; j < pageIds.size(); j++) {
- int index = Math.abs(rnd.nextInt()) % remainingLength;
- int pageId = remainingPageIds[index];
-
- // pin & latch then immediately unlatch & unpin
- ICachedPage page = bufferCache.pin(BufferedFileHandle
- .getDiskPageId(fileId, pageId), false);
- page.acquireReadLatch();
- page.releaseReadLatch();
- bufferCache.unpin(page);
-
- remainingPageIds[index] = remainingPageIds[remainingLength - 1];
- remainingLength--;
- }
- }
- }
- }
-
- bufferCache.closeFile(fileId);
- }
-}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/utility/TreeIndexStats.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/utility/TreeIndexStats.java
deleted file mode 100644
index 2754743..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/utility/TreeIndexStats.java
+++ /dev/null
@@ -1,147 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.common.utility;
-
-import java.text.DecimalFormat;
-
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-
-public class TreeIndexStats {
-
- private TreeIndexNodeTypeStats rootStats = new TreeIndexNodeTypeStats();
- private TreeIndexNodeTypeStats interiorStats = new TreeIndexNodeTypeStats();
- private TreeIndexNodeTypeStats leafStats = new TreeIndexNodeTypeStats();
-
- private int freePages = 0;
- private int metaPages = 0;
- private int treeLevels = 0;
-
- public void begin() {
- rootStats.clear();
- interiorStats.clear();
- leafStats.clear();
- freePages = 0;
- metaPages = 0;
- treeLevels = 0;
- }
-
- public void addRoot(ITreeIndexFrame frame) {
- treeLevels = frame.getLevel() + 1;
- rootStats.add(frame);
- }
-
- public void add(ITreeIndexFrame frame) {
- if (frame.isLeaf()) {
- leafStats.add(frame);
- } else if (frame.isInterior()) {
- interiorStats.add(frame);
- }
- }
-
- public void add(ITreeIndexMetaDataFrame metaFrame,
- IFreePageManager freePageManager) {
- if (freePageManager.isFreePage(metaFrame)) {
- freePages++;
- } else if (freePageManager.isMetaPage(metaFrame)) {
- metaPages++;
- }
- }
-
- public void end() {
- // nothing here currently
- }
-
- @Override
- public String toString() {
- StringBuilder strBuilder = new StringBuilder();
- DecimalFormat df = new DecimalFormat("#####.##");
-
- strBuilder.append("TREE LEVELS: " + treeLevels + "\n");
- strBuilder.append("FREE PAGES : " + freePages + "\n");
- strBuilder.append("META PAGES : " + metaPages + "\n");
- long totalPages = interiorStats.getNumPages() + leafStats.getNumPages()
- + freePages + metaPages;
- strBuilder.append("TOTAL PAGES : " + totalPages + "\n");
-
- strBuilder.append("\n");
- strBuilder.append("ROOT STATS" + "\n");
- strBuilder
- .append("NUM TUPLES: " + rootStats.getNumTuples() + "\n");
- strBuilder.append("FILL FACTOR : "
- + df.format(rootStats.getAvgFillFactor()) + "\n");
-
- if (interiorStats.getNumPages() > 0) {
- strBuilder.append("\n");
- strBuilder.append("INTERIOR STATS" + "\n");
- strBuilder.append("NUM PAGES: " + interiorStats.getNumPages()
- + "\n");
- strBuilder.append("NUM TUPLES: "
- + interiorStats.getNumTuples() + "\n");
- strBuilder.append("AVG TUPLES/PAGE: "
- + df.format(interiorStats.getAvgNumTuples()) + "\n");
- strBuilder.append("AVG FILL FACTOR: "
- + df.format(interiorStats.getAvgFillFactor()) + "\n");
- }
-
- if (leafStats.getNumPages() > 0) {
- strBuilder.append("\n");
- strBuilder.append("LEAF STATS" + "\n");
- strBuilder.append("NUM PAGES: "
- + df.format(leafStats.getNumPages()) + "\n");
- strBuilder.append("NUM TUPLES: "
- + df.format(leafStats.getNumTuples()) + "\n");
- strBuilder.append("AVG TUPLES/PAGE: "
- + df.format(leafStats.getAvgNumTuples()) + "\n");
- strBuilder.append("AVG FILL FACTOR: "
- + df.format(leafStats.getAvgFillFactor()) + "\n");
- }
-
- return strBuilder.toString();
- }
-
- public class TreeIndexNodeTypeStats {
- private long numTuples;
- private long sumTuplesSizes;
- private long numPages;
- private double sumFillFactors;
-
- public void clear() {
- numTuples = 0;
- sumTuplesSizes = 0;
- numPages = 0;
- }
-
- public void add(ITreeIndexFrame frame) {
- numPages++;
- numTuples += frame.getTupleCount();
- sumFillFactors += (double) (frame.getBuffer().capacity() - frame
- .getTotalFreeSpace())
- / (double) frame.getBuffer().capacity();
- }
-
- public long getNumTuples() {
- return numTuples;
- }
-
- public long getSumTupleSizes() {
- return sumTuplesSizes;
- }
-
- public long getNumPages() {
- return numPages;
- }
-
- public double getAvgNumTuples() {
- return (double) numTuples / (double) numPages;
- }
-
- public double getAvgTupleSize() {
- return (double) sumTuplesSizes / (double) numTuples;
- }
-
- public double getAvgFillFactor() {
- return sumFillFactors / numPages;
- }
- }
-
-}
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/utility/TreeIndexStatsGatherer.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/utility/TreeIndexStatsGatherer.java
deleted file mode 100644
index 9167732..0000000
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/utility/TreeIndexStatsGatherer.java
+++ /dev/null
@@ -1,73 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.common.utility;
-
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
-import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
-
-public class TreeIndexStatsGatherer {
-
- private final TreeIndexStats treeIndexStats = new TreeIndexStats();
- private final IBufferCache bufferCache;
- private final IFreePageManager freePageManager;
- private final int fileId;
- private final int rootPage;
-
- public TreeIndexStatsGatherer(IBufferCache bufferCache,
- IFreePageManager freePageManager, int fileId, int rootPage) {
- this.bufferCache = bufferCache;
- this.freePageManager = freePageManager;
- this.fileId = fileId;
- this.rootPage = rootPage;
- }
-
- public TreeIndexStats gatherStats(ITreeIndexFrame leafFrame,
- ITreeIndexFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame)
- throws HyracksDataException {
-
- bufferCache.openFile(fileId);
-
- treeIndexStats.begin();
-
- int maxPageId = freePageManager.getMaxPage(metaFrame);
- for (int pageId = 0; pageId <= maxPageId; pageId++) {
- ICachedPage page = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- page.acquireReadLatch();
- try {
- metaFrame.setPage(page);
- leafFrame.setPage(page);
- interiorFrame.setPage(page);
-
- if (leafFrame.isLeaf()) {
- if (pageId == rootPage) {
- treeIndexStats.addRoot(leafFrame);
- } else {
- treeIndexStats.add(leafFrame);
- }
- } else if (interiorFrame.isInterior()) {
- if (pageId == rootPage) {
- treeIndexStats.addRoot(interiorFrame);
- } else {
- treeIndexStats.add(interiorFrame);
- }
- } else {
- treeIndexStats.add(metaFrame, freePageManager);
- }
-
- } finally {
- page.releaseReadLatch();
- bufferCache.unpin(page);
- }
- }
-
- treeIndexStats.end();
-
- bufferCache.closeFile(fileId);
-
- return treeIndexStats;
- }
-}
diff --git a/hyracks-storage-am-invertedindex/.settings/org.eclipse.jdt.core.prefs b/hyracks-storage-am-invertedindex/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 1e91fb3..0000000
--- a/hyracks-storage-am-invertedindex/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Thu May 05 14:44:24 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-storage-am-invertedindex/.settings/org.maven.ide.eclipse.prefs b/hyracks-storage-am-invertedindex/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index e5e549a..0000000
--- a/hyracks-storage-am-invertedindex/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu May 05 14:44:18 PDT 2011
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-storage-am-invertedindex/pom.xml b/hyracks-storage-am-invertedindex/pom.xml
index 4578a46..32ce735 100644
--- a/hyracks-storage-am-invertedindex/pom.xml
+++ b/hyracks-storage-am-invertedindex/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-invertedindex</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,35 +27,35 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptor.java
new file mode 100644
index 0000000..1dc753e
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptor.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.api;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptor;
+
+public interface IInvertedIndexOperatorDescriptor extends ITreeIndexOperatorDescriptor {
+ public IBinaryComparatorFactory[] getInvListsComparatorFactories();
+
+ public ITypeTraits[] getInvListsTypeTraits();
+
+ public IFileSplitProvider getInvListsFileSplitProvider();
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptorHelper.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptorHelper.java
deleted file mode 100644
index b3afe4a..0000000
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptorHelper.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.invertedindex.api;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptorHelper;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
-
-public interface IInvertedIndexOperatorDescriptorHelper extends ITreeIndexOperatorDescriptorHelper {
- public IFileSplitProvider getInvIndexFileSplitProvider();
-
- public IBinaryComparatorFactory[] getInvIndexComparatorFactories();
-
- public ITypeTrait[] getInvIndexTypeTraits();
-
- public IIndexRegistryProvider<InvertedIndex> getInvIndexRegistryProvider();
-}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifierFactory.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifierFactory.java
new file mode 100644
index 0000000..92770d6
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifierFactory.java
@@ -0,0 +1,22 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.api;
+
+import java.io.Serializable;
+
+public interface IInvertedIndexSearchModifierFactory extends Serializable {
+ public IInvertedIndexSearchModifier createSearchModifier();
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
index 9db23ef..eab266d 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
@@ -16,103 +16,105 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.dataflow;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexOperatorDescriptorHelper;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.util.InvertedIndexUtils;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public abstract class AbstractInvertedIndexOperatorDescriptor extends AbstractSingleActivityOperatorDescriptor
- implements IInvertedIndexOperatorDescriptorHelper {
+ implements IInvertedIndexOperatorDescriptor {
private static final long serialVersionUID = 1L;
- // general
+ // General.
protected final IStorageManagerInterface storageManager;
+ protected final IIndexRegistryProvider<IIndex> indexRegistryProvider;
- // btree
- protected final IFileSplitProvider btreeFileSplitProvider;
- protected final IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider;
- protected final ITreeIndexFrameFactory interiorFrameFactory;
- protected final ITreeIndexFrameFactory leafFrameFactory;
- protected final ITypeTrait[] btreeTypeTraits;
+ // Btree.
+ protected final ITreeIndexFrameFactory btreeInteriorFrameFactory;
+ protected final ITreeIndexFrameFactory btreeLeafFrameFactory;
+ protected final ITypeTraits[] btreeTypeTraits;
protected final IBinaryComparatorFactory[] btreeComparatorFactories;
- protected final ITreeIndexOpHelperFactory opHelperFactory;
+ protected final IIndexDataflowHelperFactory btreeDataflowHelperFactory;
+ protected final IFileSplitProvider btreeFileSplitProvider;
- // inverted index
- protected final IFileSplitProvider invIndexFileSplitProvider;
- protected final IIndexRegistryProvider<InvertedIndex> invIndexRegistryProvider;
- protected final ITypeTrait[] invIndexTypeTraits;
- protected final IBinaryComparatorFactory[] invIndexComparatorFactories;
+ // Inverted index.
+ protected final ITypeTraits[] invListsTypeTraits;
+ protected final IBinaryComparatorFactory[] invListComparatorFactories;
+ protected final IFileSplitProvider invListsFileSplitProvider;
public AbstractInvertedIndexOperatorDescriptor(JobSpecification spec, int inputArity, int outputArity,
RecordDescriptor recDesc, IStorageManagerInterface storageManager,
- IFileSplitProvider btreeFileSplitProvider, IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
- ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
- ITypeTrait[] btreeTypeTraits, IBinaryComparatorFactory[] btreeComparatorFactories, float btreeFillFactor,
- ITreeIndexOpHelperFactory opHelperFactory, IFileSplitProvider invIndexFileSplitProvider,
- IIndexRegistryProvider<InvertedIndex> invIndexRegistryProvider, ITypeTrait[] invIndexTypeTraits,
- IBinaryComparatorFactory[] invIndexComparatorFactories) {
+ IFileSplitProvider btreeFileSplitProvider, IFileSplitProvider invListsFileSplitProvider,
+ IIndexRegistryProvider<IIndex> indexRegistryProvider, ITypeTraits[] tokenTypeTraits,
+ IBinaryComparatorFactory[] tokenComparatorFactories, ITypeTraits[] invListsTypeTraits,
+ IBinaryComparatorFactory[] invListComparatorFactories,
+ IIndexDataflowHelperFactory btreeDataflowHelperFactory) {
super(spec, inputArity, outputArity);
- // general
+ // General.
this.storageManager = storageManager;
+ this.indexRegistryProvider = indexRegistryProvider;
- // btree
+ // Btree.
+ this.btreeTypeTraits = InvertedIndexUtils.getBTreeTypeTraits(tokenTypeTraits);
+ ITreeIndexTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(btreeTypeTraits);
+ this.btreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ this.btreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ this.btreeComparatorFactories = tokenComparatorFactories;
+ this.btreeDataflowHelperFactory = btreeDataflowHelperFactory;
this.btreeFileSplitProvider = btreeFileSplitProvider;
- this.treeIndexRegistryProvider = treeIndexRegistryProvider;
- this.interiorFrameFactory = interiorFrameFactory;
- this.leafFrameFactory = leafFrameFactory;
- this.btreeTypeTraits = btreeTypeTraits;
- this.btreeComparatorFactories = btreeComparatorFactories;
- this.opHelperFactory = opHelperFactory;
- // inverted index
- this.invIndexFileSplitProvider = invIndexFileSplitProvider;
- this.invIndexRegistryProvider = invIndexRegistryProvider;
- this.invIndexTypeTraits = invIndexTypeTraits;
- this.invIndexComparatorFactories = invIndexComparatorFactories;
+ // Inverted index.
+ this.invListsTypeTraits = invListsTypeTraits;
+ this.invListComparatorFactories = invListComparatorFactories;
+ this.invListsFileSplitProvider = invListsFileSplitProvider;
- if (outputArity > 0)
+ if (outputArity > 0) {
recordDescriptors[0] = recDesc;
+ }
}
@Override
- public IFileSplitProvider getTreeIndexFileSplitProvider() {
+ public IFileSplitProvider getFileSplitProvider() {
return btreeFileSplitProvider;
}
+
+ @Override
+ public IFileSplitProvider getInvListsFileSplitProvider() {
+ return invListsFileSplitProvider;
+ }
@Override
public IBinaryComparatorFactory[] getTreeIndexComparatorFactories() {
return btreeComparatorFactories;
}
-
- @Override
- public IPrimitiveValueProviderFactory[] getTreeIndexValueProviderFactories() {
- return null;
- }
@Override
- public ITypeTrait[] getTreeIndexTypeTraits() {
+ public ITypeTraits[] getTreeIndexTypeTraits() {
return btreeTypeTraits;
}
@Override
public ITreeIndexFrameFactory getTreeIndexInteriorFactory() {
- return interiorFrameFactory;
+ return btreeInteriorFrameFactory;
}
@Override
public ITreeIndexFrameFactory getTreeIndexLeafFactory() {
- return leafFrameFactory;
+ return btreeLeafFrameFactory;
}
@Override
@@ -121,37 +123,27 @@
}
@Override
- public IIndexRegistryProvider<ITreeIndex> getTreeIndexRegistryProvider() {
- return treeIndexRegistryProvider;
- }
-
- @Override
public RecordDescriptor getRecordDescriptor() {
return recordDescriptors[0];
}
@Override
- public IIndexRegistryProvider<InvertedIndex> getInvIndexRegistryProvider() {
- return invIndexRegistryProvider;
+ public IBinaryComparatorFactory[] getInvListsComparatorFactories() {
+ return invListComparatorFactories;
}
@Override
- public IBinaryComparatorFactory[] getInvIndexComparatorFactories() {
- return invIndexComparatorFactories;
+ public ITypeTraits[] getInvListsTypeTraits() {
+ return invListsTypeTraits;
}
@Override
- public IFileSplitProvider getInvIndexFileSplitProvider() {
- return invIndexFileSplitProvider;
+ public IIndexRegistryProvider<IIndex> getIndexRegistryProvider() {
+ return indexRegistryProvider;
}
-
+
@Override
- public ITypeTrait[] getInvIndexTypeTraits() {
- return invIndexTypeTraits;
- }
-
- @Override
- public ITreeIndexOpHelperFactory getTreeIndexOpHelperFactory() {
- return opHelperFactory;
+ public IIndexDataflowHelperFactory getIndexDataflowHelperFactory() {
+ return btreeDataflowHelperFactory;
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java
index b5b1393..ffb94e7 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorDescriptor.java
@@ -20,7 +20,6 @@
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractSingleActivityOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
@@ -30,26 +29,26 @@
private static final long serialVersionUID = 1L;
private final IBinaryTokenizerFactory tokenizerFactory;
- // fields that will be tokenized
+ // Fields that will be tokenized
private final int[] tokenFields;
- // operator will emit these projected fields for each token, e.g., as
+ // operator will append these key fields to each token, e.g., as
// payload for an inverted list
- // WARNING: too many projected fields can cause significant data blowup
- private final int[] projFields;
+ // WARNING: too many key fields can cause significant data blowup.
+ private final int[] keyFields;
public BinaryTokenizerOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc,
- IBinaryTokenizerFactory tokenizerFactory, int[] tokenFields, int[] projFields) {
+ IBinaryTokenizerFactory tokenizerFactory, int[] tokenFields, int[] keyFields) {
super(spec, 1, 1);
this.tokenizerFactory = tokenizerFactory;
this.tokenFields = tokenFields;
- this.projFields = projFields;
+ this.keyFields = keyFields;
recordDescriptors[0] = recDesc;
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
- IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IRecordDescriptorProvider recordDescProvider,
+ int partition, int nPartitions) throws HyracksDataException {
return new BinaryTokenizerOperatorNodePushable(ctx, recordDescProvider.getInputRecordDescriptor(odId, 0),
- recordDescriptors[0], tokenizerFactory.createTokenizer(), tokenFields, projFields);
+ recordDescriptors[0], tokenizerFactory.createTokenizer(), tokenFields, keyFields);
}
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java
index 0af27d6..d00bea6 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/BinaryTokenizerOperatorNodePushable.java
@@ -59,7 +59,7 @@
public void open() throws HyracksDataException {
accessor = new FrameTupleAccessor(ctx.getFrameSize(), inputRecDesc);
writeBuffer = ctx.allocateFrame();
- builder = new ArrayTupleBuilder(outputRecDesc.getFields().length);
+ builder = new ArrayTupleBuilder(outputRecDesc.getFieldCount());
builderDos = builder.getDataOutput();
appender = new FrameTupleAppender(ctx.getFrameSize());
appender.reset(writeBuffer, true);
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
index 1ab72fe..e91aa08 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
@@ -19,16 +19,12 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
public class InvertedIndexBulkLoadOperatorDescriptor extends AbstractInvertedIndexOperatorDescriptor {
@@ -36,30 +32,22 @@
private static final long serialVersionUID = 1L;
private final int[] fieldPermutation;
- private final float btreeFillFactor;
- private final IInvertedListBuilder invListBuilder;
- public InvertedIndexBulkLoadOperatorDescriptor(JobSpecification spec, IStorageManagerInterface storageManager,
- int[] fieldPermutation, IFileSplitProvider btreeFileSplitProvider,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider, ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] btreeTypeTraits,
- IBinaryComparatorFactory[] btreeComparatorFactories, float btreeFillFactor,
- ITreeIndexOpHelperFactory opHelperFactory, IFileSplitProvider invIndexFileSplitProvider,
- IIndexRegistryProvider<InvertedIndex> invIndexRegistryProvider, ITypeTrait[] invIndexTypeTraits,
- IBinaryComparatorFactory[] invIndexComparatorFactories, IInvertedListBuilder invListBuilder) {
- super(spec, 1, 0, null, storageManager, btreeFileSplitProvider, treeIndexRegistryProvider,
- interiorFrameFactory, leafFrameFactory, btreeTypeTraits, btreeComparatorFactories, btreeFillFactor,
- opHelperFactory, invIndexFileSplitProvider, invIndexRegistryProvider, invIndexTypeTraits,
- invIndexComparatorFactories);
+ public InvertedIndexBulkLoadOperatorDescriptor(JobSpecification spec, int[] fieldPermutation,
+ IStorageManagerInterface storageManager, IFileSplitProvider btreeFileSplitProvider,
+ IFileSplitProvider invListsFileSplitProvider, IIndexRegistryProvider<IIndex> indexRegistryProvider,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenComparatorFactories,
+ ITypeTraits[] invListsTypeTraits, IBinaryComparatorFactory[] invListComparatorFactories,
+ IIndexDataflowHelperFactory btreeDataflowHelperFactory) {
+ super(spec, 1, 0, null, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
+ indexRegistryProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
+ invListComparatorFactories, btreeDataflowHelperFactory);
this.fieldPermutation = fieldPermutation;
- this.btreeFillFactor = btreeFillFactor;
- this.invListBuilder = invListBuilder;
}
@Override
- public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx, IOperatorEnvironment env,
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
- return new InvertedIndexBulkLoadOperatorNodePushable(this, ctx, partition, fieldPermutation, btreeFillFactor,
- invListBuilder, recordDescProvider);
+ return new InvertedIndexBulkLoadOperatorNodePushable(this, ctx, partition, fieldPermutation, recordDescProvider);
}
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
index 9d421b0..9dcabb4 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
@@ -22,18 +22,19 @@
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputSinkOperatorNodePushable;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexHelperOpenMode;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
import edu.uci.ics.hyracks.storage.am.common.dataflow.PermutingFrameTupleReference;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexOpHelper;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
public class InvertedIndexBulkLoadOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
- private final TreeIndexOpHelper treeIndexOpHelper;
- private float btreeFillFactor;
-
- private final InvertedIndexOpHelper invIndexOpHelper;
- protected final IInvertedListBuilder invListBuilder;
+ private final TreeIndexDataflowHelper btreeDataflowHelper;
+ private final InvertedIndexDataflowHelper invIndexDataflowHelper;
+ private final IInvertedListBuilder invListBuilder;
+ private InvertedIndex invIndex;
private InvertedIndex.BulkLoadContext bulkLoadCtx;
private final IHyracksTaskContext ctx;
@@ -44,45 +45,49 @@
private IRecordDescriptorProvider recordDescProvider;
public InvertedIndexBulkLoadOperatorNodePushable(AbstractInvertedIndexOperatorDescriptor opDesc,
- IHyracksTaskContext ctx, int partition, int[] fieldPermutation, float btreeFillFactor,
- IInvertedListBuilder invListBuilder, IRecordDescriptorProvider recordDescProvider) {
- treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory().createTreeIndexOpHelper(opDesc, ctx, partition,
- IndexHelperOpenMode.CREATE);
- invIndexOpHelper = new InvertedIndexOpHelper(opDesc, ctx, partition, IndexHelperOpenMode.CREATE);
- this.btreeFillFactor = btreeFillFactor;
+ IHyracksTaskContext ctx, int partition, int[] fieldPermutation, IRecordDescriptorProvider recordDescProvider) {
+ btreeDataflowHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory()
+ .createIndexDataflowHelper(opDesc, ctx, partition, true);
+ invIndexDataflowHelper = new InvertedIndexDataflowHelper(btreeDataflowHelper, opDesc, ctx, partition, true);
this.recordDescProvider = recordDescProvider;
this.ctx = ctx;
- this.invListBuilder = invListBuilder;
+ this.invListBuilder = new FixedSizeElementInvertedListBuilder(opDesc.getInvListsTypeTraits());
tuple.setFieldPermutation(fieldPermutation);
}
@Override
public void open() throws HyracksDataException {
- AbstractInvertedIndexOperatorDescriptor opDesc = (AbstractInvertedIndexOperatorDescriptor) treeIndexOpHelper
+ AbstractInvertedIndexOperatorDescriptor opDesc = (AbstractInvertedIndexOperatorDescriptor) btreeDataflowHelper
.getOperatorDescriptor();
RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
- accessor = new FrameTupleAccessor(treeIndexOpHelper.getHyracksTaskContext().getFrameSize(), recDesc);
+ accessor = new FrameTupleAccessor(btreeDataflowHelper.getHyracksTaskContext().getFrameSize(), recDesc);
- // btree
+ // BTree.
try {
- treeIndexOpHelper.init();
- treeIndexOpHelper.getTreeIndex().open(treeIndexOpHelper.getIndexFileId());
+ btreeDataflowHelper.init();
} catch (Exception e) {
- // cleanup in case of failure
- treeIndexOpHelper.deinit();
- throw new HyracksDataException(e);
+ // Cleanup in case of failure.
+ btreeDataflowHelper.deinit();
+ if (e instanceof HyracksDataException) {
+ throw (HyracksDataException) e;
+ } else {
+ throw new HyracksDataException(e);
+ }
}
- // inverted index
+ // Inverted Index.
try {
- invIndexOpHelper.init();
- invIndexOpHelper.getInvIndex().open(invIndexOpHelper.getInvIndexFileId());
- bulkLoadCtx = invIndexOpHelper.getInvIndex().beginBulkLoad(invListBuilder, ctx.getFrameSize(),
- btreeFillFactor);
+ invIndexDataflowHelper.init();
+ invIndex = (InvertedIndex) invIndexDataflowHelper.getIndex();
+ bulkLoadCtx = invIndex.beginBulkLoad(invListBuilder, ctx.getFrameSize(), BTree.DEFAULT_FILL_FACTOR);
} catch (Exception e) {
- // cleanup in case of failure
- invIndexOpHelper.deinit();
- throw new HyracksDataException(e);
+ // Cleanup in case of failure.
+ invIndexDataflowHelper.deinit();
+ if (e instanceof HyracksDataException) {
+ throw (HyracksDataException) e;
+ } else {
+ throw new HyracksDataException(e);
+ }
}
}
@@ -92,16 +97,26 @@
int tupleCount = accessor.getTupleCount();
for (int i = 0; i < tupleCount; i++) {
tuple.reset(accessor, i);
- invIndexOpHelper.getInvIndex().bulkLoadAddTuple(bulkLoadCtx, tuple);
+ try {
+ invIndex.bulkLoadAddTuple(bulkLoadCtx, tuple);
+ } catch (PageAllocationException e) {
+ throw new HyracksDataException(e);
+ }
}
}
@Override
public void close() throws HyracksDataException {
try {
- invIndexOpHelper.getInvIndex().endBulkLoad(bulkLoadCtx);
+ invIndex.endBulkLoad(bulkLoadCtx);
+ } catch (PageAllocationException e) {
+ throw new HyracksDataException(e);
} finally {
- treeIndexOpHelper.deinit();
+ try {
+ btreeDataflowHelper.deinit();
+ } finally {
+ invIndexDataflowHelper.deinit();
+ }
}
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java
new file mode 100644
index 0000000..71717e4
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexDataflowHelper.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.storage.am.invertedindex.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+
+public final class InvertedIndexDataflowHelper extends IndexDataflowHelper {
+ private final TreeIndexDataflowHelper btreeDataflowHelper;
+
+ public InvertedIndexDataflowHelper(TreeIndexDataflowHelper btreeDataflowHelper, IIndexOperatorDescriptor opDesc,
+ IHyracksTaskContext ctx, int partition, boolean createIfNotExists) {
+ super(opDesc, ctx, partition, createIfNotExists);
+ this.btreeDataflowHelper = btreeDataflowHelper;
+ }
+
+ public FileReference getFilereference() {
+ AbstractInvertedIndexOperatorDescriptor invIndexOpDesc = (AbstractInvertedIndexOperatorDescriptor) opDesc;
+ IFileSplitProvider fileSplitProvider = invIndexOpDesc.getInvListsFileSplitProvider();
+ return fileSplitProvider.getFileSplits()[partition].getLocalFile();
+ }
+
+ @Override
+ public IIndex createIndexInstance() throws HyracksDataException {
+ IInvertedIndexOperatorDescriptor invIndexOpDesc = (IInvertedIndexOperatorDescriptor) opDesc;
+ MultiComparator cmp = IndexUtils.createMultiComparator(invIndexOpDesc.getInvListsComparatorFactories());
+ // Assumes btreeDataflowHelper.init() has already been called.
+ BTree btree = (BTree) btreeDataflowHelper.getIndex();
+ return new InvertedIndex(opDesc.getStorageManager().getBufferCache(ctx), btree,
+ invIndexOpDesc.getInvListsTypeTraits(), cmp);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexOpHelper.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexOpHelper.java
deleted file mode 100644
index 1eb6757..0000000
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexOpHelper.java
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.storage.am.invertedindex.dataflow;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptorHelper;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexHelperOpenMode;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexOperatorDescriptorHelper;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-
-public final class InvertedIndexOpHelper {
-
- private InvertedIndex invIndex;
- private int invIndexFileId = -1;
- private int partition;
-
- private IInvertedIndexOperatorDescriptorHelper opDesc;
- private IHyracksTaskContext ctx;
-
- private IndexHelperOpenMode mode;
-
- public InvertedIndexOpHelper(IInvertedIndexOperatorDescriptorHelper opDesc, final IHyracksTaskContext ctx,
- int partition, IndexHelperOpenMode mode) {
- this.opDesc = opDesc;
- this.ctx = ctx;
- this.mode = mode;
- this.partition = partition;
- }
-
- public void init() throws HyracksDataException {
- IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
- IFileMapProvider fileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
- IFileSplitProvider fileSplitProvider = opDesc.getInvIndexFileSplitProvider();
-
- FileReference f = fileSplitProvider.getFileSplits()[partition].getLocalFile();
- boolean fileIsMapped = fileMapProvider.isMapped(f);
-
- switch (mode) {
-
- case OPEN: {
- if (!fileIsMapped) {
- throw new HyracksDataException("Trying to open inverted index from unmapped file " + f.toString());
- }
- }
- break;
-
- case CREATE:
- case ENLIST: {
- if (!fileIsMapped) {
- bufferCache.createFile(f);
- }
- }
- break;
-
- }
-
- int fileId = fileMapProvider.lookupFileId(f);
- try {
- bufferCache.openFile(fileId);
- } catch (HyracksDataException e) {
- // revert state of buffer cache since file failed to open
- if (!fileIsMapped) {
- bufferCache.deleteFile(fileId);
- }
- throw e;
- }
-
- // only set btreeFileId member when openFile() succeeds,
- // otherwise deinit() will try to close the file that failed to open
- invIndexFileId = fileId;
- IndexRegistry<InvertedIndex> invIndexRegistry = opDesc.getInvIndexRegistryProvider().getRegistry(ctx);
- invIndex = invIndexRegistry.get(invIndexFileId);
- if (invIndex == null) {
-
- // create new inverted index and register it
- invIndexRegistry.lock();
- try {
- // check if inverted index has already been registered by
- // another thread
- invIndex = invIndexRegistry.get(invIndexFileId);
- if (invIndex == null) {
- // this thread should create and register the inverted index
-
- IBinaryComparator[] comparators = new IBinaryComparator[opDesc.getInvIndexComparatorFactories().length];
- for (int i = 0; i < opDesc.getInvIndexComparatorFactories().length; i++) {
- comparators[i] = opDesc.getInvIndexComparatorFactories()[i].createBinaryComparator();
- }
-
- MultiComparator cmp = new MultiComparator(opDesc.getInvIndexTypeTraits(), comparators);
-
- // assumes btree has already been registered
- IFileSplitProvider btreeFileSplitProvider = opDesc.getTreeIndexFileSplitProvider();
- IndexRegistry<ITreeIndex> treeIndexRegistry = opDesc.getTreeIndexRegistryProvider()
- .getRegistry(ctx);
- FileReference btreeFile = btreeFileSplitProvider.getFileSplits()[partition].getLocalFile();
- boolean btreeFileIsMapped = fileMapProvider.isMapped(btreeFile);
- if (!btreeFileIsMapped) {
- throw new HyracksDataException(
- "Trying to create inverted index, but associated BTree file has not been mapped");
- }
- int btreeFileId = fileMapProvider.lookupFileId(f);
- BTree btree = (BTree) treeIndexRegistry.get(btreeFileId);
-
- invIndex = new InvertedIndex(bufferCache, btree, cmp);
- invIndex.open(invIndexFileId);
- invIndexRegistry.register(invIndexFileId, invIndex);
- }
- } finally {
- invIndexRegistry.unlock();
- }
- }
- }
-
- public void deinit() throws HyracksDataException {
- if (invIndexFileId != -1) {
- IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
- bufferCache.closeFile(invIndexFileId);
- }
- }
-
- public InvertedIndex getInvIndex() {
- return invIndex;
- }
-
- public ITreeIndexOperatorDescriptorHelper getOperatorDescriptor() {
- return opDesc;
- }
-
- public int getInvIndexFileId() {
- return invIndexFileId;
- }
-}
\ No newline at end of file
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java
new file mode 100644
index 0000000..bf9899f
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorDescriptor.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifierFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizerFactory;
+import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+
+public class InvertedIndexSearchOperatorDescriptor extends AbstractInvertedIndexOperatorDescriptor {
+ private static final long serialVersionUID = 1L;
+
+ private final int queryField;
+ private final IBinaryTokenizerFactory queryTokenizerFactory;
+ private final IInvertedIndexSearchModifierFactory searchModifierFactory;
+
+ public InvertedIndexSearchOperatorDescriptor(JobSpecification spec,
+ int queryField, IStorageManagerInterface storageManager, IFileSplitProvider btreeFileSplitProvider,
+ IFileSplitProvider invListsFileSplitProvider, IIndexRegistryProvider<IIndex> indexRegistryProvider,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenComparatorFactories,
+ ITypeTraits[] invListsTypeTraits, IBinaryComparatorFactory[] invListComparatorFactories,
+ IIndexDataflowHelperFactory btreeDataflowHelperFactory, IBinaryTokenizerFactory queryTokenizerFactory,
+ IInvertedIndexSearchModifierFactory searchModifierFactory, RecordDescriptor recDesc) {
+ super(spec, 1, 1, recDesc, storageManager, btreeFileSplitProvider, invListsFileSplitProvider,
+ indexRegistryProvider, tokenTypeTraits, tokenComparatorFactories, invListsTypeTraits,
+ invListComparatorFactories, btreeDataflowHelperFactory);
+ this.queryField = queryField;
+ this.queryTokenizerFactory = queryTokenizerFactory;
+ this.searchModifierFactory = searchModifierFactory;
+ }
+
+ @Override
+ public IOperatorNodePushable createPushRuntime(IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) throws HyracksDataException {
+ IBinaryTokenizer tokenizer = queryTokenizerFactory.createTokenizer();
+ IInvertedIndexSearchModifier searchModifier = searchModifierFactory.createSearchModifier();
+ return new InvertedIndexSearchOperatorNodePushable(this, ctx, partition, queryField, searchModifier, tokenizer,
+ recordDescProvider);
+ }
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java
new file mode 100644
index 0000000..ceaff1e
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexSearchOperatorNodePushable.java
@@ -0,0 +1,183 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.dataflow;
+
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexResultCursor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.OccurrenceThresholdPanicException;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.SearchResultCursor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
+import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
+
+public class InvertedIndexSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
+ private final TreeIndexDataflowHelper btreeDataflowHelper;
+ private final InvertedIndexDataflowHelper invIndexDataflowHelper;
+ private final IHyracksTaskContext ctx;
+ private final int queryField;
+ private FrameTupleAccessor accessor;
+ private FrameTupleReference tuple;
+ private IRecordDescriptorProvider recordDescProvider;
+ private InvertedIndex invIndex;
+
+ private final IInvertedIndexSearchModifier searchModifier;
+ private final IBinaryTokenizer queryTokenizer;
+ private TOccurrenceSearcher searcher;
+ private IInvertedIndexResultCursor resultCursor;
+
+ private ByteBuffer writeBuffer;
+ private FrameTupleAppender appender;
+ private ArrayTupleBuilder tb;
+ private DataOutput dos;
+
+ private final AbstractInvertedIndexOperatorDescriptor opDesc;
+
+ public InvertedIndexSearchOperatorNodePushable(AbstractInvertedIndexOperatorDescriptor opDesc,
+ IHyracksTaskContext ctx, int partition, int queryField, IInvertedIndexSearchModifier searchModifier,
+ IBinaryTokenizer queryTokenizer, IRecordDescriptorProvider recordDescProvider) {
+ this.opDesc = opDesc;
+ btreeDataflowHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory()
+ .createIndexDataflowHelper(opDesc, ctx, partition, false);
+ invIndexDataflowHelper = new InvertedIndexDataflowHelper(btreeDataflowHelper, opDesc, ctx, partition, false);
+ this.ctx = ctx;
+ this.queryField = queryField;
+ this.searchModifier = searchModifier;
+ this.queryTokenizer = queryTokenizer;
+ this.recordDescProvider = recordDescProvider;
+ }
+
+ @Override
+ public void open() throws HyracksDataException {
+ RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
+ accessor = new FrameTupleAccessor(btreeDataflowHelper.getHyracksTaskContext().getFrameSize(), recDesc);
+ tuple = new FrameTupleReference();
+ // BTree.
+ try {
+ btreeDataflowHelper.init();
+ } catch (Exception e) {
+ // Cleanup in case of failure/
+ btreeDataflowHelper.deinit();
+ if (e instanceof HyracksDataException) {
+ throw (HyracksDataException) e;
+ } else {
+ throw new HyracksDataException(e);
+ }
+ }
+ // Inverted Index.
+ try {
+ invIndexDataflowHelper.init();
+ invIndex = (InvertedIndex) invIndexDataflowHelper.getIndex();
+ } catch (Exception e) {
+ // Cleanup in case of failure.
+ invIndexDataflowHelper.deinit();
+ if (e instanceof HyracksDataException) {
+ throw (HyracksDataException) e;
+ } else {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ writeBuffer = btreeDataflowHelper.getHyracksTaskContext().allocateFrame();
+ tb = new ArrayTupleBuilder(opDesc.getInvListsTypeTraits().length);
+ dos = tb.getDataOutput();
+ appender = new FrameTupleAppender(btreeDataflowHelper.getHyracksTaskContext().getFrameSize());
+ appender.reset(writeBuffer, true);
+
+ searcher = new TOccurrenceSearcher(ctx, invIndex, queryTokenizer);
+ resultCursor = new SearchResultCursor(searcher.createResultFrameTupleAccessor(),
+ searcher.createResultTupleReference());
+
+ writer.open();
+ }
+
+ private void writeSearchResults() throws Exception {
+ while (resultCursor.hasNext()) {
+ resultCursor.next();
+ tb.reset();
+ ITupleReference invListElement = resultCursor.getTuple();
+ int invListFields = opDesc.getInvListsTypeTraits().length;
+ for (int i = 0; i < invListFields; i++) {
+ dos.write(invListElement.getFieldData(i), invListElement.getFieldStart(i),
+ invListElement.getFieldLength(i));
+ tb.addFieldEndOffset();
+ }
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ FrameUtils.flushFrame(writeBuffer, writer);
+ appender.reset(writeBuffer, true);
+ if (!appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize())) {
+ throw new IllegalStateException();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
+ accessor.reset(buffer);
+ int tupleCount = accessor.getTupleCount();
+ try {
+ for (int i = 0; i < tupleCount; i++) {
+ tuple.reset(accessor, i);
+ searcher.reset();
+ try {
+ searcher.reset();
+ searcher.search(resultCursor, tuple, queryField, searchModifier);
+ writeSearchResults();
+ } catch (OccurrenceThresholdPanicException e) {
+ // Ignore panic cases for now.
+ }
+ }
+ } catch (Exception e) {
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public void fail() throws HyracksDataException {
+ writer.fail();
+ }
+
+ @Override
+ public void close() throws HyracksDataException {
+ try {
+ if (appender.getTupleCount() > 0) {
+ FrameUtils.flushFrame(writeBuffer, writer);
+ }
+ writer.close();
+ } finally {
+ try {
+ btreeDataflowHelper.deinit();
+ } finally {
+ invIndexDataflowHelper.deinit();
+ }
+ }
+ }
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java
index 03fc9a1..643c105 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java
@@ -15,7 +15,7 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
@@ -26,10 +26,10 @@
private byte[] targetBuf;
private int pos;
- public FixedSizeElementInvertedListBuilder(ITypeTrait[] invListFields) {
+ public FixedSizeElementInvertedListBuilder(ITypeTraits[] invListFields) {
int tmp = 0;
for (int i = 0; i < invListFields.length; i++) {
- tmp += invListFields[i].getStaticallyKnownDataLength();
+ tmp += invListFields[i].getFixedLength();
}
listElementSize = tmp;
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
index f7ef56e..4158019 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
@@ -5,7 +5,7 @@
import java.io.DataInputStream;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
@@ -32,7 +32,7 @@
private ICachedPage[] pages = new ICachedPage[10];
private int[] elementIndexes = new int[10];
- public FixedSizeElementInvertedListCursor(IBufferCache bufferCache, int fileId, ITypeTrait[] invListFields) {
+ public FixedSizeElementInvertedListCursor(IBufferCache bufferCache, int fileId, ITypeTraits[] invListFields) {
this.bufferCache = bufferCache;
this.fileId = fileId;
this.currentElementIx = 0;
@@ -40,7 +40,7 @@
int tmp = 0;
for (int i = 0; i < invListFields.length; i++) {
- tmp += invListFields[i].getStaticallyKnownDataLength();
+ tmp += invListFields[i].getFixedLength();
}
elementSize = tmp;
this.currentOff = -elementSize;
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAccessor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAccessor.java
index 9858eb0..cbedc45 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAccessor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAccessor.java
@@ -19,29 +19,29 @@
import edu.uci.ics.hyracks.api.comm.FrameHelper;
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
public class FixedSizeFrameTupleAccessor implements IFrameTupleAccessor {
private final int frameSize;
private ByteBuffer buffer;
- private final ITypeTrait[] fields;
+ private final ITypeTraits[] fields;
private final int[] fieldStartOffsets;
private final int tupleSize;
- public FixedSizeFrameTupleAccessor(int frameSize, ITypeTrait[] fields) {
+ public FixedSizeFrameTupleAccessor(int frameSize, ITypeTraits[] fields) {
this.frameSize = frameSize;
this.fields = fields;
this.fieldStartOffsets = new int[fields.length];
this.fieldStartOffsets[0] = 0;
for (int i = 1; i < fields.length; i++) {
- fieldStartOffsets[i] = fieldStartOffsets[i - 1] + fields[i - 1].getStaticallyKnownDataLength();
+ fieldStartOffsets[i] = fieldStartOffsets[i - 1] + fields[i - 1].getFixedLength();
}
int tmp = 0;
for (int i = 0; i < fields.length; i++) {
- tmp += fields[i].getStaticallyKnownDataLength();
+ tmp += fields[i].getFixedLength();
}
tupleSize = tmp;
}
@@ -58,12 +58,12 @@
@Override
public int getFieldEndOffset(int tupleIndex, int fIdx) {
- return getTupleStartOffset(tupleIndex) + fieldStartOffsets[fIdx] + fields[fIdx].getStaticallyKnownDataLength();
+ return getTupleStartOffset(tupleIndex) + fieldStartOffsets[fIdx] + fields[fIdx].getFixedLength();
}
@Override
public int getFieldLength(int tupleIndex, int fIdx) {
- return fields[fIdx].getStaticallyKnownDataLength();
+ return fields[fIdx].getFixedLength();
}
@Override
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAppender.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAppender.java
index edc2304..489ec2e 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAppender.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAppender.java
@@ -18,7 +18,7 @@
import java.nio.ByteBuffer;
import edu.uci.ics.hyracks.api.comm.FrameHelper;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
public class FixedSizeFrameTupleAppender {
@@ -29,11 +29,11 @@
private int tupleCount;
private int tupleDataEndOffset;
- public FixedSizeFrameTupleAppender(int frameSize, ITypeTrait[] fields) {
+ public FixedSizeFrameTupleAppender(int frameSize, ITypeTraits[] fields) {
this.frameSize = frameSize;
int tmp = 0;
for (int i = 0; i < fields.length; i++) {
- tmp += fields[i].getStaticallyKnownDataLength();
+ tmp += fields[i].getFixedLength();
}
tupleSize = tmp;
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeTupleReference.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeTupleReference.java
index 248b81e..0656d69 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeTupleReference.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeTupleReference.java
@@ -15,22 +15,22 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
public class FixedSizeTupleReference implements ITupleReference {
- private final ITypeTrait[] typeTraits;
+ private final ITypeTraits[] typeTraits;
private final int[] fieldStartOffsets;
private byte[] data;
private int startOff;
- public FixedSizeTupleReference(ITypeTrait[] typeTraits) {
+ public FixedSizeTupleReference(ITypeTraits[] typeTraits) {
this.typeTraits = typeTraits;
this.fieldStartOffsets = new int[typeTraits.length];
this.fieldStartOffsets[0] = 0;
for (int i = 1; i < typeTraits.length; i++) {
- fieldStartOffsets[i] = fieldStartOffsets[i - 1] + typeTraits[i - 1].getStaticallyKnownDataLength();
+ fieldStartOffsets[i] = fieldStartOffsets[i - 1] + typeTraits[i - 1].getFixedLength();
}
}
@@ -51,7 +51,7 @@
@Override
public int getFieldLength(int fIdx) {
- return typeTraits[fIdx].getStaticallyKnownDataLength();
+ return typeTraits[fIdx].getFixedLength();
}
@Override
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
index 9eab110..986e57b 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
@@ -18,6 +18,7 @@
import java.nio.ByteBuffer;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
@@ -28,10 +29,13 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListCursor;
@@ -47,34 +51,41 @@
* implemented features: updates (insert/update/delete) Limitations: a query
* cannot exceed the size of a Hyracks frame
*/
-public class InvertedIndex {
+public class InvertedIndex implements IIndex {
private BTree btree;
private int rootPageId = 0;
private IBufferCache bufferCache;
private int fileId;
+ private final ITypeTraits[] invListTypeTraits;
private final MultiComparator invListCmp;
private final int numTokenFields;
private final int numInvListKeys;
- public InvertedIndex(IBufferCache bufferCache, BTree btree, MultiComparator invListCmp) {
+ public InvertedIndex(IBufferCache bufferCache, BTree btree, ITypeTraits[] invListTypeTraits, MultiComparator invListCmp) {
this.bufferCache = bufferCache;
this.btree = btree;
this.invListCmp = invListCmp;
+ this.invListTypeTraits = invListTypeTraits;
this.numTokenFields = btree.getMultiComparator().getKeyFieldCount();
this.numInvListKeys = invListCmp.getKeyFieldCount();
}
+ @Override
public void open(int fileId) {
this.fileId = fileId;
}
+ @Override
+ public void create(int indexFileId) throws HyracksDataException {
+ }
+
public void close() {
this.fileId = -1;
}
public BulkLoadContext beginBulkLoad(IInvertedListBuilder invListBuilder, int hyracksFrameSize,
- float btreeFillFactor) throws HyracksDataException {
+ float btreeFillFactor) throws HyracksDataException, TreeIndexException, PageAllocationException {
BulkLoadContext ctx = new BulkLoadContext(invListBuilder, hyracksFrameSize, btreeFillFactor);
ctx.init(rootPageId, fileId);
return ctx;
@@ -86,7 +97,7 @@
// the next invListCmp.getKeyFieldCount() fields in tuple are keys of the
// inverted list (e.g., primary key)
// key fields of inverted list are fixed size
- public void bulkLoadAddTuple(BulkLoadContext ctx, ITupleReference tuple) throws HyracksDataException {
+ public void bulkLoadAddTuple(BulkLoadContext ctx, ITupleReference tuple) throws HyracksDataException, PageAllocationException {
// first inverted list, copy token to baaos and start new list
if (ctx.currentInvListTokenBaaos.size() == 0) {
@@ -147,9 +158,9 @@
}
}
- public boolean openCursor(ITreeIndexCursor btreeCursor, RangePredicate btreePred, BTreeOpContext btreeOpCtx,
+ public boolean openCursor(ITreeIndexCursor btreeCursor, RangePredicate btreePred, ITreeIndexAccessor btreeAccessor,
IInvertedListCursor invListCursor) throws Exception {
- btree.search(btreeCursor, btreePred, btreeOpCtx);
+ btreeAccessor.search(btreeCursor, btreePred);
boolean ret = false;
if (btreeCursor.hasNext()) {
@@ -179,7 +190,7 @@
return ret;
}
- public void createAndInsertBTreeTuple(BulkLoadContext ctx) throws HyracksDataException {
+ public void createAndInsertBTreeTuple(BulkLoadContext ctx) throws HyracksDataException, PageAllocationException {
// build tuple
ctx.btreeTupleBuilder.reset();
ctx.btreeTupleBuilder.addField(ctx.currentInvListTokenBaaos.getByteArray(), 0,
@@ -197,10 +208,10 @@
// reset tuple reference
ctx.btreeFrameTupleReference.reset(ctx.btreeFrameTupleAccessor, 0);
- btree.bulkLoadAddTuple(ctx.btreeBulkLoadCtx, ctx.btreeFrameTupleReference);
+ btree.bulkLoadAddTuple(ctx.btreeFrameTupleReference, ctx.btreeBulkLoadCtx);
}
- public void endBulkLoad(BulkLoadContext ctx) throws HyracksDataException {
+ public void endBulkLoad(BulkLoadContext ctx) throws HyracksDataException, PageAllocationException {
// create entry in btree for last inverted list
createAndInsertBTreeTuple(ctx);
btree.endBulkLoad(ctx.btreeBulkLoadCtx);
@@ -218,6 +229,10 @@
public MultiComparator getInvListElementCmp() {
return invListCmp;
}
+
+ public ITypeTraits[] getTypeTraits() {
+ return invListTypeTraits;
+ }
public BTree getBTree() {
return btree;
@@ -235,8 +250,7 @@
private int currentInvListStartPageId;
private int currentInvListStartOffset;
private final ByteArrayAccessibleOutputStream currentInvListTokenBaaos = new ByteArrayAccessibleOutputStream();
- private final FixedSizeTupleReference currentInvListToken = new FixedSizeTupleReference(
- invListCmp.getTypeTraits());
+ private final FixedSizeTupleReference currentInvListToken = new FixedSizeTupleReference(invListTypeTraits);
private int currentPageId;
private ICachedPage currentPage;
@@ -247,7 +261,7 @@
this.invListBuilder = invListBuilder;
this.tokenCmp = btree.getMultiComparator();
this.btreeTupleBuffer = ByteBuffer.allocate(hyracksFrameSize);
- this.btreeTupleBuilder = new ArrayTupleBuilder(tokenCmp.getFieldCount());
+ this.btreeTupleBuilder = new ArrayTupleBuilder(btree.getFieldCount());
this.btreeTupleAppender = new FrameTupleAppender(hyracksFrameSize);
// TODO: serde never used, only need correct number of fields
// tuple contains (token, start page, end page, start offset, num
@@ -260,10 +274,8 @@
this.btreeFillFactor = btreeFillFactor;
}
- public void init(int startPageId, int fileId) throws HyracksDataException {
- btreeBulkLoadCtx = btree.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR,
- btree.getLeafFrameFactory().createFrame(), btree.getInteriorFrameFactory().createFrame(), btree
- .getFreePageManager().getMetaDataFrameFactory().createFrame());
+ public void init(int startPageId, int fileId) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ btreeBulkLoadCtx = btree.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR);
currentPageId = startPageId;
currentPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), true);
currentPage.acquireWriteLatch();
@@ -285,5 +297,5 @@
currentPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), true);
currentPage.acquireWriteLatch();
}
- };
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
index 7505ae0..b24d416 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
@@ -24,10 +24,10 @@
import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
@@ -36,12 +36,11 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexResultCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
@@ -69,18 +68,18 @@
protected final ITreeIndexCursor btreeCursor;
protected final FrameTupleReference searchKey = new FrameTupleReference();
protected final RangePredicate btreePred = new RangePredicate(true, null, null, true, true, null, null);
- protected final BTreeOpContext btreeOpCtx;
+ protected final ITreeIndexAccessor btreeAccessor;
protected RecordDescriptor queryTokenRecDesc = new RecordDescriptor(
new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE });
- protected ArrayTupleBuilder queryTokenBuilder = new ArrayTupleBuilder(queryTokenRecDesc.getFields().length);
+ protected ArrayTupleBuilder queryTokenBuilder = new ArrayTupleBuilder(queryTokenRecDesc.getFieldCount());
protected DataOutput queryTokenDos = queryTokenBuilder.getDataOutput();
protected FrameTupleAppender queryTokenAppender;
protected ByteBuffer queryTokenFrame;
protected final InvertedIndex invIndex;
protected final IBinaryTokenizer queryTokenizer;
- protected final ITypeTrait[] invListFieldsWithCount;
+ protected final ITypeTraits[] invListFieldsWithCount;
protected int occurrenceThreshold;
protected final int cursorCacheSize = 10;
@@ -95,20 +94,18 @@
leafFrame = invIndex.getBTree().getLeafFrameFactory().createFrame();
interiorFrame = invIndex.getBTree().getInteriorFrameFactory().createFrame();
- btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame);
- ITypeTrait[] invListFields = invIndex.getInvListElementCmp().getTypeTraits();
- invListFieldsWithCount = new TypeTrait[invListFields.length + 1];
+ btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame, false);
+ ITypeTraits[] invListFields = invIndex.getTypeTraits();
+ invListFieldsWithCount = new ITypeTraits[invListFields.length + 1];
int tmp = 0;
for (int i = 0; i < invListFields.length; i++) {
invListFieldsWithCount[i] = invListFields[i];
- tmp += invListFields[i].getStaticallyKnownDataLength();
+ tmp += invListFields[i].getFixedLength();
}
// using an integer for counting occurrences
- invListFieldsWithCount[invListFields.length] = new TypeTrait(4);
+ invListFieldsWithCount[invListFields.length] = IntegerPointable.TYPE_TRAITS;
invListKeyLength = tmp;
- btreeOpCtx = invIndex.getBTree().createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
-
resultFrameTupleApp = new FixedSizeFrameTupleAppender(ctx.getFrameSize(), invListFieldsWithCount);
resultFrameTupleAcc = new FixedSizeFrameTupleAccessor(ctx.getFrameSize(), invListFieldsWithCount);
resultTuple = new FixedSizeTupleReference(invListFieldsWithCount);
@@ -124,12 +121,13 @@
// pre-create cursor objects
for (int i = 0; i < cursorCacheSize; i++) {
invListCursorCache.add(new FixedSizeElementInvertedListCursor(invIndex.getBufferCache(), invIndex
- .getInvListsFileId(), invIndex.getInvListElementCmp().getTypeTraits()));
+ .getInvListsFileId(), invIndex.getTypeTraits()));
}
queryTokenAppender = new FrameTupleAppender(ctx.getFrameSize());
queryTokenFrame = ctx.allocateFrame();
+ btreeAccessor = invIndex.getBTree().createAccessor();
currentNumResults = 0;
}
@@ -143,12 +141,12 @@
currentNumResults = 0;
}
- public void search(IInvertedIndexResultCursor resultCursor, ITupleReference queryTuple, int queryFieldIndex,
+ public void search(IInvertedIndexResultCursor resultCursor, ITupleReference queryTuple, int queryField,
IInvertedIndexSearchModifier searchModifier) throws Exception {
queryTokenAppender.reset(queryTokenFrame, true);
- queryTokenizer.reset(queryTuple.getFieldData(queryFieldIndex), queryTuple.getFieldStart(queryFieldIndex),
- queryTuple.getFieldLength(queryFieldIndex));
+ queryTokenizer.reset(queryTuple.getFieldData(queryField), queryTuple.getFieldStart(queryField),
+ queryTuple.getFieldLength(queryField));
while (queryTokenizer.hasNext()) {
queryTokenizer.next();
@@ -174,14 +172,14 @@
int diff = numQueryTokens - invListCursorCache.size();
for (int i = 0; i < diff; i++) {
invListCursorCache.add(new FixedSizeElementInvertedListCursor(invIndex.getBufferCache(), invIndex
- .getInvListsFileId(), invIndex.getInvListElementCmp().getTypeTraits()));
+ .getInvListsFileId(), invIndex.getTypeTraits()));
}
}
invListCursors.clear();
for (int i = 0; i < numQueryTokens; i++) {
searchKey.reset(queryTokenAccessor, i);
- invIndex.openCursor(btreeCursor, btreePred, btreeOpCtx, invListCursorCache.get(i));
+ invIndex.openCursor(btreeCursor, btreePred, btreeAccessor, invListCursorCache.get(i));
invListCursors.add(invListCursorCache.get(i));
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/ConjunctiveSearchModifierFactory.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/ConjunctiveSearchModifierFactory.java
new file mode 100644
index 0000000..0db6008
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/ConjunctiveSearchModifierFactory.java
@@ -0,0 +1,28 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers;
+
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifierFactory;
+
+public class ConjunctiveSearchModifierFactory implements IInvertedIndexSearchModifierFactory {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IInvertedIndexSearchModifier createSearchModifier() {
+ return new ConjunctiveSearchModifier();
+ }
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/EditDistanceSearchModifierFactory.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/EditDistanceSearchModifierFactory.java
new file mode 100644
index 0000000..128d9db
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/EditDistanceSearchModifierFactory.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers;
+
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifierFactory;
+
+public class EditDistanceSearchModifierFactory implements IInvertedIndexSearchModifierFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final int gramLength;
+ private final int edThresh;
+
+ public EditDistanceSearchModifierFactory(int gramLength, int edThresh) {
+ this.gramLength = gramLength;
+ this.edThresh = edThresh;
+ }
+
+ @Override
+ public IInvertedIndexSearchModifier createSearchModifier() {
+ return new EditDistanceSearchModifier(gramLength, edThresh);
+ }
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifierFactory.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifierFactory.java
new file mode 100644
index 0000000..bd27c03
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifierFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers;
+
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifierFactory;
+
+public class JaccardSearchModifierFactory implements IInvertedIndexSearchModifierFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ private final float jaccThresh;
+
+ public JaccardSearchModifierFactory(float jaccThresh) {
+ this.jaccThresh = jaccThresh;
+ }
+
+ @Override
+ public IInvertedIndexSearchModifier createSearchModifier() {
+ return new JaccardSearchModifier(jaccThresh);
+ }
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8StringBinaryTokenizer.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8StringBinaryTokenizer.java
index bbb32d6..be6bfb0 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8StringBinaryTokenizer.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8StringBinaryTokenizer.java
@@ -19,60 +19,59 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
-public abstract class AbstractUTF8StringBinaryTokenizer implements
- IBinaryTokenizer {
+public abstract class AbstractUTF8StringBinaryTokenizer implements IBinaryTokenizer {
- protected byte[] data;
- protected int start;
- protected int length;
- protected int tokenLength;
- protected int index;
- protected int utf8Length;
+ protected byte[] data;
+ protected int start;
+ protected int length;
+ protected int tokenLength;
+ protected int index;
+ protected int utf8Length;
- protected final IntArray tokensStart;
- protected final IntArray tokensLength;
- protected final IToken token;
+ protected final IntArray tokensStart;
+ protected final IntArray tokensLength;
+ protected final IToken token;
- protected final boolean ignoreTokenCount;
- protected final boolean sourceHasTypeTag;
+ protected final boolean ignoreTokenCount;
+ protected final boolean sourceHasTypeTag;
- public AbstractUTF8StringBinaryTokenizer(boolean ignoreTokenCount,
- boolean sourceHasTypeTag, ITokenFactory tokenFactory) {
- this.ignoreTokenCount = ignoreTokenCount;
- this.sourceHasTypeTag = sourceHasTypeTag;
- if (!ignoreTokenCount) {
- tokensStart = new IntArray();
- tokensLength = new IntArray();
- } else {
- tokensStart = null;
- tokensLength = null;
- }
- token = tokenFactory.createToken();
- }
+ public AbstractUTF8StringBinaryTokenizer(boolean ignoreTokenCount, boolean sourceHasTypeTag,
+ ITokenFactory tokenFactory) {
+ this.ignoreTokenCount = ignoreTokenCount;
+ this.sourceHasTypeTag = sourceHasTypeTag;
+ if (!ignoreTokenCount) {
+ tokensStart = new IntArray();
+ tokensLength = new IntArray();
+ } else {
+ tokensStart = null;
+ tokensLength = null;
+ }
+ token = tokenFactory.createToken();
+ }
- @Override
- public IToken getToken() {
- return token;
- }
+ @Override
+ public IToken getToken() {
+ return token;
+ }
- @Override
- public void reset(byte[] data, int start, int length) {
- this.start = start;
- index = this.start;
- if (sourceHasTypeTag) {
- index++; // skip type tag
- }
- utf8Length = StringUtils.getUTFLen(data, index);
- index += 2; // skip utf8 length indicator
- this.data = data;
- this.length = length + start;
+ @Override
+ public void reset(byte[] data, int start, int length) {
+ this.start = start;
+ index = this.start;
+ if (sourceHasTypeTag) {
+ index++; // skip type tag
+ }
+ utf8Length = UTF8StringPointable.getUTFLen(data, index);
+ index += 2; // skip utf8 length indicator
+ this.data = data;
+ this.length = length + start;
- tokenLength = 0;
- if (!ignoreTokenCount) {
- tokensStart.reset();
- tokensLength.reset();
- }
- }
+ tokenLength = 0;
+ if (!ignoreTokenCount) {
+ tokensStart.reset();
+ tokensLength.reset();
+ }
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8Token.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8Token.java
index 92d6ac2..65afa65 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8Token.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/AbstractUTF8Token.java
@@ -16,91 +16,89 @@
*
* Author: Alexander Behm <abehm (at) ics.uci.edu>
*/
-
package edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers;
import java.io.DataOutput;
import java.io.IOException;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
public abstract class AbstractUTF8Token implements IToken {
- public static final int GOLDEN_RATIO_32 = 0x09e3779b9;
+ public static final int GOLDEN_RATIO_32 = 0x09e3779b9;
- protected int length;
- protected int tokenLength;
- protected int start;
- protected int tokenCount;
- protected byte[] data;
- protected final byte tokenTypeTag;
- protected final byte countTypeTag;
+ protected int length;
+ protected int tokenLength;
+ protected int start;
+ protected int tokenCount;
+ protected byte[] data;
+ protected final byte tokenTypeTag;
+ protected final byte countTypeTag;
- public AbstractUTF8Token() {
- tokenTypeTag = -1;
- countTypeTag = -1;
- }
+ public AbstractUTF8Token() {
+ tokenTypeTag = -1;
+ countTypeTag = -1;
+ }
- public AbstractUTF8Token(byte tokenTypeTag, byte countTypeTag) {
- this.tokenTypeTag = tokenTypeTag;
- this.countTypeTag = countTypeTag;
- }
+ public AbstractUTF8Token(byte tokenTypeTag, byte countTypeTag) {
+ this.tokenTypeTag = tokenTypeTag;
+ this.countTypeTag = countTypeTag;
+ }
- @Override
- public byte[] getData() {
- return data;
- }
+ @Override
+ public byte[] getData() {
+ return data;
+ }
- @Override
- public int getLength() {
- return length;
- }
+ @Override
+ public int getLength() {
+ return length;
+ }
- public int getLowerCaseUTF8Len(int size) {
- int lowerCaseUTF8Len = 0;
- int pos = start;
- for (int i = 0; i < size; i++) {
- char c = StringUtils.toLowerCase(StringUtils.charAt(data, pos));
- lowerCaseUTF8Len += StringUtils.getModifiedUTF8Len(c);
- pos += StringUtils.charSize(data, pos);
- }
- return lowerCaseUTF8Len;
- }
+ public int getLowerCaseUTF8Len(int size) {
+ int lowerCaseUTF8Len = 0;
+ int pos = start;
+ for (int i = 0; i < size; i++) {
+ char c = Character.toLowerCase(UTF8StringPointable.charAt(data, pos));
+ lowerCaseUTF8Len += UTF8StringPointable.getModifiedUTF8Len(c);
+ pos += UTF8StringPointable.charSize(data, pos);
+ }
+ return lowerCaseUTF8Len;
+ }
- @Override
- public int getStart() {
- return start;
- }
+ @Override
+ public int getStart() {
+ return start;
+ }
- @Override
- public int getTokenLength() {
- return tokenLength;
- }
+ @Override
+ public int getTokenLength() {
+ return tokenLength;
+ }
- public void handleCountTypeTag(DataOutput dos) throws IOException {
- if (countTypeTag > 0) {
- dos.write(countTypeTag);
- }
- }
+ public void handleCountTypeTag(DataOutput dos) throws IOException {
+ if (countTypeTag > 0) {
+ dos.write(countTypeTag);
+ }
+ }
- public void handleTokenTypeTag(DataOutput dos) throws IOException {
- if (tokenTypeTag > 0) {
- dos.write(tokenTypeTag);
- }
- }
+ public void handleTokenTypeTag(DataOutput dos) throws IOException {
+ if (tokenTypeTag > 0) {
+ dos.write(tokenTypeTag);
+ }
+ }
- @Override
- public void reset(byte[] data, int start, int length, int tokenLength,
- int tokenCount) {
- this.data = data;
- this.start = start;
- this.length = length;
- this.tokenLength = tokenLength;
- this.tokenCount = tokenCount;
- }
+ @Override
+ public void reset(byte[] data, int start, int length, int tokenLength, int tokenCount) {
+ this.data = data;
+ this.start = start;
+ this.length = length;
+ this.tokenLength = tokenLength;
+ this.tokenCount = tokenCount;
+ }
- @Override
- public void serializeTokenCount(DataOutput dos) throws IOException {
- handleCountTypeTag(dos);
- dos.writeInt(tokenCount);
- }
-}
+ @Override
+ public void serializeTokenCount(DataOutput dos) throws IOException {
+ handleCountTypeTag(dos);
+ dos.writeInt(tokenCount);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/DelimitedUTF8StringBinaryTokenizer.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/DelimitedUTF8StringBinaryTokenizer.java
index de9ad2c..9dacde6 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/DelimitedUTF8StringBinaryTokenizer.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/DelimitedUTF8StringBinaryTokenizer.java
@@ -19,69 +19,63 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
-public class DelimitedUTF8StringBinaryTokenizer extends
- AbstractUTF8StringBinaryTokenizer {
+public class DelimitedUTF8StringBinaryTokenizer extends AbstractUTF8StringBinaryTokenizer {
- public DelimitedUTF8StringBinaryTokenizer(boolean ignoreTokenCount,
- boolean sourceHasTypeTag, ITokenFactory tokenFactory) {
- super(ignoreTokenCount, sourceHasTypeTag, tokenFactory);
- }
+ public DelimitedUTF8StringBinaryTokenizer(boolean ignoreTokenCount, boolean sourceHasTypeTag,
+ ITokenFactory tokenFactory) {
+ super(ignoreTokenCount, sourceHasTypeTag, tokenFactory);
+ }
- @Override
- public boolean hasNext() {
- // skip delimiters
- while (index < length && isSeparator(StringUtils.charAt(data, index))) {
- index += StringUtils.charSize(data, index);
- }
- return index < length;
- }
+ @Override
+ public boolean hasNext() {
+ // skip delimiters
+ while (index < length && isSeparator(UTF8StringPointable.charAt(data, index))) {
+ index += UTF8StringPointable.charSize(data, index);
+ }
+ return index < length;
+ }
- private boolean isSeparator(char c) {
- return !(Character.isLetterOrDigit(c)
- || Character.getType(c) == Character.OTHER_LETTER || Character
- .getType(c) == Character.OTHER_NUMBER);
- }
+ private boolean isSeparator(char c) {
+ return !(Character.isLetterOrDigit(c) || Character.getType(c) == Character.OTHER_LETTER || Character.getType(c) == Character.OTHER_NUMBER);
+ }
- @Override
- public void next() {
- tokenLength = 0;
- int currentTokenStart = index;
- while (index < length && !isSeparator(StringUtils.charAt(data, index))) {
- index += StringUtils.charSize(data, index);
- tokenLength++;
- }
- int tokenCount = 1;
- if (tokenLength > 0 && !ignoreTokenCount) {
- // search if we got the same token before
- for (int i = 0; i < tokensStart.length(); ++i) {
- if (tokenLength == tokensLength.get(i)) {
- int tokenStart = tokensStart.get(i);
- tokenCount++; // assume we found it
- int offset = 0;
- int currLength = 0;
- while (currLength < tokenLength) {
- // case insensitive comparison
- if (StringUtils.toLowerCase(StringUtils.charAt(data,
- currentTokenStart + offset)) != StringUtils
- .toLowerCase(StringUtils.charAt(data,
- tokenStart + offset))) {
- tokenCount--;
- break;
- }
- offset += StringUtils.charSize(data, currentTokenStart
- + offset);
- currLength++;
- }
- }
- }
- // add the new token to the list of seen tokens
- tokensStart.add(currentTokenStart);
- tokensLength.add(tokenLength);
- }
+ @Override
+ public void next() {
+ tokenLength = 0;
+ int currentTokenStart = index;
+ while (index < length && !isSeparator(UTF8StringPointable.charAt(data, index))) {
+ index += UTF8StringPointable.charSize(data, index);
+ tokenLength++;
+ }
+ int tokenCount = 1;
+ if (tokenLength > 0 && !ignoreTokenCount) {
+ // search if we got the same token before
+ for (int i = 0; i < tokensStart.length(); ++i) {
+ if (tokenLength == tokensLength.get(i)) {
+ int tokenStart = tokensStart.get(i);
+ tokenCount++; // assume we found it
+ int offset = 0;
+ int currLength = 0;
+ while (currLength < tokenLength) {
+ // case insensitive comparison
+ if (Character.toLowerCase(UTF8StringPointable.charAt(data, currentTokenStart + offset)) != Character
+ .toLowerCase(UTF8StringPointable.charAt(data, tokenStart + offset))) {
+ tokenCount--;
+ break;
+ }
+ offset += UTF8StringPointable.charSize(data, currentTokenStart + offset);
+ currLength++;
+ }
+ }
+ }
+ // add the new token to the list of seen tokens
+ tokensStart.add(currentTokenStart);
+ tokensLength.add(tokenLength);
+ }
- // set token
- token.reset(data, currentTokenStart, index, tokenLength, tokenCount);
- }
-}
+ // set token
+ token.reset(data, currentTokenStart, index, tokenLength, tokenCount);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/HashedUTF8NGramToken.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/HashedUTF8NGramToken.java
index 25d1a2c..b7bb828 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/HashedUTF8NGramToken.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/HashedUTF8NGramToken.java
@@ -22,43 +22,43 @@
import java.io.DataOutput;
import java.io.IOException;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
public class HashedUTF8NGramToken extends UTF8NGramToken {
- public HashedUTF8NGramToken(byte tokenTypeTag, byte countTypeTag) {
- super(tokenTypeTag, countTypeTag);
- }
+ public HashedUTF8NGramToken(byte tokenTypeTag, byte countTypeTag) {
+ super(tokenTypeTag, countTypeTag);
+ }
- @Override
- public void serializeToken(DataOutput dos) throws IOException {
- handleTokenTypeTag(dos);
+ @Override
+ public void serializeToken(DataOutput dos) throws IOException {
+ handleTokenTypeTag(dos);
- int hash = GOLDEN_RATIO_32;
+ int hash = GOLDEN_RATIO_32;
- // pre chars
- for (int i = 0; i < numPreChars; i++) {
- hash ^= PRECHAR;
- hash *= GOLDEN_RATIO_32;
- }
+ // pre chars
+ for (int i = 0; i < numPreChars; i++) {
+ hash ^= PRECHAR;
+ hash *= GOLDEN_RATIO_32;
+ }
- // regular chars
- int numRegGrams = tokenLength - numPreChars - numPostChars;
- int pos = start;
- for (int i = 0; i < numRegGrams; i++) {
- hash ^= StringUtils.toLowerCase(StringUtils.charAt(data, pos));
- hash *= GOLDEN_RATIO_32;
- pos += StringUtils.charSize(data, pos);
- }
+ // regular chars
+ int numRegGrams = tokenLength - numPreChars - numPostChars;
+ int pos = start;
+ for (int i = 0; i < numRegGrams; i++) {
+ hash ^= Character.toLowerCase(UTF8StringPointable.charAt(data, pos));
+ hash *= GOLDEN_RATIO_32;
+ pos += UTF8StringPointable.charSize(data, pos);
+ }
- // post chars
- for (int i = 0; i < numPostChars; i++) {
- hash ^= POSTCHAR;
- hash *= GOLDEN_RATIO_32;
- }
+ // post chars
+ for (int i = 0; i < numPostChars; i++) {
+ hash ^= POSTCHAR;
+ hash *= GOLDEN_RATIO_32;
+ }
- // token count
- hash += tokenCount;
+ // token count
+ hash += tokenCount;
- dos.writeInt(hash);
- }
+ dos.writeInt(hash);
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/HashedUTF8WordToken.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/HashedUTF8WordToken.java
index 55237ce..42ed053 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/HashedUTF8WordToken.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/HashedUTF8WordToken.java
@@ -22,67 +22,66 @@
import java.io.DataOutput;
import java.io.IOException;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
public class HashedUTF8WordToken extends UTF8WordToken {
- private int hash = 0;
+ private int hash = 0;
- public HashedUTF8WordToken(byte tokenTypeTag, byte countTypeTag) {
- super(tokenTypeTag, countTypeTag);
- }
+ public HashedUTF8WordToken(byte tokenTypeTag, byte countTypeTag) {
+ super(tokenTypeTag, countTypeTag);
+ }
- @Override
- public boolean equals(Object o) {
- if (o == null) {
- return false;
- }
- if (!(o instanceof IToken)) {
- return false;
- }
- IToken t = (IToken) o;
- if (t.getTokenLength() != tokenLength) {
- return false;
- }
- int offset = 0;
- for (int i = 0; i < tokenLength; i++) {
- if (StringUtils.charAt(t.getData(), t.getStart() + offset) != StringUtils
- .charAt(data, start + offset)) {
- return false;
- }
- offset += StringUtils.charSize(data, start + offset);
- }
- return true;
- }
+ @Override
+ public boolean equals(Object o) {
+ if (o == null) {
+ return false;
+ }
+ if (!(o instanceof IToken)) {
+ return false;
+ }
+ IToken t = (IToken) o;
+ if (t.getTokenLength() != tokenLength) {
+ return false;
+ }
+ int offset = 0;
+ for (int i = 0; i < tokenLength; i++) {
+ if (UTF8StringPointable.charAt(t.getData(), t.getStart() + offset) != UTF8StringPointable.charAt(data,
+ start + offset)) {
+ return false;
+ }
+ offset += UTF8StringPointable.charSize(data, start + offset);
+ }
+ return true;
+ }
- @Override
- public int hashCode() {
- return hash;
- }
+ @Override
+ public int hashCode() {
+ return hash;
+ }
- @Override
- public void reset(byte[] data, int start, int length, int tokenLength,
- int tokenCount) {
- super.reset(data, start, length, tokenLength, tokenCount);
+ @Override
+ public void reset(byte[] data, int start, int length, int tokenLength, int tokenCount) {
+ super.reset(data, start, length, tokenLength, tokenCount);
- // pre-compute hash value using JAQL-like string hashing
- int pos = start;
- hash = GOLDEN_RATIO_32;
- for (int i = 0; i < tokenLength; i++) {
- hash ^= StringUtils.toLowerCase(StringUtils.charAt(data, pos));
- hash *= GOLDEN_RATIO_32;
- pos += StringUtils.charSize(data, pos);
- }
- hash += tokenCount;
- }
+ // pre-compute hash value using JAQL-like string hashing
+ int pos = start;
+ hash = GOLDEN_RATIO_32;
+ for (int i = 0; i < tokenLength; i++) {
+ hash ^= Character.toLowerCase(UTF8StringPointable.charAt(data, pos));
+ hash *= GOLDEN_RATIO_32;
+ pos += UTF8StringPointable.charSize(data, pos);
+ }
+ hash += tokenCount;
+ }
- @Override
- public void serializeToken(DataOutput dos) throws IOException {
- if (tokenTypeTag > 0) {
- dos.write(tokenTypeTag);
- }
+ @Override
+ public void serializeToken(DataOutput dos) throws IOException {
+ if (tokenTypeTag > 0) {
+ dos.write(tokenTypeTag);
+ }
- // serialize hash value
- dos.writeInt(hash);
- }
+ // serialize hash value
+ dos.writeInt(hash);
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizer.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizer.java
index 2a13f83..fdfc02f 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizer.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizer.java
@@ -19,105 +19,100 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers;
-import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
-public class NGramUTF8StringBinaryTokenizer extends
- AbstractUTF8StringBinaryTokenizer {
+public class NGramUTF8StringBinaryTokenizer extends AbstractUTF8StringBinaryTokenizer {
- private int gramLength;
- private boolean usePrePost;
+ private int gramLength;
+ private boolean usePrePost;
- private int gramNum;
- private int totalGrams;
+ private int gramNum;
+ private int totalGrams;
- private final INGramToken concreteToken;
+ private final INGramToken concreteToken;
- public NGramUTF8StringBinaryTokenizer(int gramLength, boolean usePrePost,
- boolean ignoreTokenCount, boolean sourceHasTypeTag,
- ITokenFactory tokenFactory) {
- super(ignoreTokenCount, sourceHasTypeTag, tokenFactory);
- this.gramLength = gramLength;
- this.usePrePost = usePrePost;
- concreteToken = (INGramToken) token;
- }
+ public NGramUTF8StringBinaryTokenizer(int gramLength, boolean usePrePost, boolean ignoreTokenCount,
+ boolean sourceHasTypeTag, ITokenFactory tokenFactory) {
+ super(ignoreTokenCount, sourceHasTypeTag, tokenFactory);
+ this.gramLength = gramLength;
+ this.usePrePost = usePrePost;
+ concreteToken = (INGramToken) token;
+ }
- @Override
- public boolean hasNext() {
- if (gramNum < totalGrams) {
- return true;
- } else {
- return false;
- }
- }
+ @Override
+ public boolean hasNext() {
+ if (gramNum < totalGrams) {
+ return true;
+ } else {
+ return false;
+ }
+ }
- @Override
- public void next() {
- int currentTokenStart = index;
- int tokenCount = 1;
- int numPreChars = 0;
- int numPostChars = 0;
- if (usePrePost) {
- numPreChars = Math.max(gramLength - gramNum - 1, 0);
- numPostChars = (gramNum > totalGrams - gramLength) ? gramLength
- - totalGrams + gramNum : 0;
- }
- gramNum++;
+ @Override
+ public void next() {
+ int currentTokenStart = index;
+ int tokenCount = 1;
+ int numPreChars = 0;
+ int numPostChars = 0;
+ if (usePrePost) {
+ numPreChars = Math.max(gramLength - gramNum - 1, 0);
+ numPostChars = (gramNum > totalGrams - gramLength) ? gramLength - totalGrams + gramNum : 0;
+ }
+ gramNum++;
- concreteToken.setNumPrePostChars(numPreChars, numPostChars);
- if (numPreChars == 0) {
- index += StringUtils.charSize(data, index);
- }
+ concreteToken.setNumPrePostChars(numPreChars, numPostChars);
+ if (numPreChars == 0) {
+ index += UTF8StringPointable.charSize(data, index);
+ }
- // compute token count
- // ignore pre and post grams for duplicate detection
- if (!ignoreTokenCount && numPreChars == 0 && numPostChars == 0) {
- int tmpIndex = start;
- while (tmpIndex < currentTokenStart) {
- tokenCount++; // assume found
- int offset = 0;
- for (int j = 0; j < gramLength; j++) {
- if (StringUtils.toLowerCase(StringUtils.charAt(data,
- currentTokenStart + offset)) != StringUtils
- .toLowerCase(StringUtils.charAt(data, tmpIndex
- + offset))) {
- tokenCount--;
- break;
- }
- offset += StringUtils.charSize(data, tmpIndex + offset);
- }
- tmpIndex += StringUtils.charSize(data, tmpIndex);
- }
- }
+ // compute token count
+ // ignore pre and post grams for duplicate detection
+ if (!ignoreTokenCount && numPreChars == 0 && numPostChars == 0) {
+ int tmpIndex = start;
+ while (tmpIndex < currentTokenStart) {
+ tokenCount++; // assume found
+ int offset = 0;
+ for (int j = 0; j < gramLength; j++) {
+ if (Character.toLowerCase(UTF8StringPointable.charAt(data, currentTokenStart + offset)) != Character
+ .toLowerCase(UTF8StringPointable.charAt(data, tmpIndex + offset))) {
+ tokenCount--;
+ break;
+ }
+ offset += UTF8StringPointable.charSize(data, tmpIndex + offset);
+ }
+ tmpIndex += UTF8StringPointable.charSize(data, tmpIndex);
+ }
+ }
- // set token
- token.reset(data, currentTokenStart, length, gramLength, tokenCount);
- }
+ // set token
+ token.reset(data, currentTokenStart, length, gramLength, tokenCount);
+ }
- @Override
- public void reset(byte[] data, int start, int length) {
- super.reset(data, start, length);
- gramNum = 0;
+ @Override
+ public void reset(byte[] data, int start, int length) {
+ super.reset(data, start, length);
+ gramNum = 0;
- int numChars = 0;
- int pos = index;
- int end = pos + utf8Length;
- while (pos < end) {
- numChars++;
- pos += StringUtils.charSize(data, pos);
- }
+ int numChars = 0;
+ int pos = index;
+ int end = pos + utf8Length;
+ while (pos < end) {
+ numChars++;
+ pos += UTF8StringPointable.charSize(data, pos);
+ }
- if (usePrePost) {
- totalGrams = numChars + gramLength - 1;
- } else {
- totalGrams = numChars - gramLength + 1;
- }
- }
+ if (usePrePost) {
+ totalGrams = numChars + gramLength - 1;
+ } else {
+ totalGrams = numChars - gramLength + 1;
+ }
+ }
- public void setGramlength(int gramLength) {
- this.gramLength = gramLength;
- }
+ public void setGramlength(int gramLength) {
+ this.gramLength = gramLength;
+ }
- public void setPrePost(boolean usePrePost) {
- this.usePrePost = usePrePost;
- }
+ public void setPrePost(boolean usePrePost) {
+ this.usePrePost = usePrePost;
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/UTF8NGramToken.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/UTF8NGramToken.java
index 6b6406f..59cadc8 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/UTF8NGramToken.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/UTF8NGramToken.java
@@ -22,65 +22,65 @@
import java.io.DataOutput;
import java.io.IOException;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
public class UTF8NGramToken extends AbstractUTF8Token implements INGramToken {
- public final static char PRECHAR = '#';
+ public final static char PRECHAR = '#';
- public final static char POSTCHAR = '$';
+ public final static char POSTCHAR = '$';
- protected int numPreChars;
- protected int numPostChars;
+ protected int numPreChars;
+ protected int numPostChars;
- public UTF8NGramToken(byte tokenTypeTag, byte countTypeTag) {
- super(tokenTypeTag, countTypeTag);
- }
+ public UTF8NGramToken(byte tokenTypeTag, byte countTypeTag) {
+ super(tokenTypeTag, countTypeTag);
+ }
- @Override
- public int getNumPostChars() {
- return numPreChars;
- }
+ @Override
+ public int getNumPostChars() {
+ return numPreChars;
+ }
- @Override
- public int getNumPreChars() {
- return numPostChars;
- }
+ @Override
+ public int getNumPreChars() {
+ return numPostChars;
+ }
- @Override
- public void serializeToken(DataOutput dos) throws IOException {
- handleTokenTypeTag(dos);
+ @Override
+ public void serializeToken(DataOutput dos) throws IOException {
+ handleTokenTypeTag(dos);
- // regular chars
- int numRegChars = tokenLength - numPreChars - numPostChars;
+ // regular chars
+ int numRegChars = tokenLength - numPreChars - numPostChars;
- // assuming pre and post char need 1-byte each in utf8
- int tokenUTF8Len = getLowerCaseUTF8Len(numRegChars) + numPreChars
- + numPostChars;
+ // assuming pre and post char need 1-byte each in utf8
+ int tokenUTF8Len = getLowerCaseUTF8Len(numRegChars) + numPreChars + numPostChars;
- // write utf8 length indicator
- StringUtils.writeUTF8Len(tokenUTF8Len, dos);
+ // write utf8 length indicator
+ StringUtils.writeUTF8Len(tokenUTF8Len, dos);
- // pre chars
- for (int i = 0; i < numPreChars; i++) {
- StringUtils.writeCharAsModifiedUTF8(PRECHAR, dos);
- }
+ // pre chars
+ for (int i = 0; i < numPreChars; i++) {
+ StringUtils.writeCharAsModifiedUTF8(PRECHAR, dos);
+ }
- int pos = start;
- for (int i = 0; i < numRegChars; i++) {
- char c = StringUtils.toLowerCase(StringUtils.charAt(data, pos));
- StringUtils.writeCharAsModifiedUTF8(c, dos);
- pos += StringUtils.charSize(data, pos);
- }
+ int pos = start;
+ for (int i = 0; i < numRegChars; i++) {
+ char c = Character.toLowerCase(UTF8StringPointable.charAt(data, pos));
+ StringUtils.writeCharAsModifiedUTF8(c, dos);
+ pos += UTF8StringPointable.charSize(data, pos);
+ }
- // post chars
- for (int i = 0; i < numPostChars; i++) {
- StringUtils.writeCharAsModifiedUTF8(POSTCHAR, dos);
- }
- }
+ // post chars
+ for (int i = 0; i < numPostChars; i++) {
+ StringUtils.writeCharAsModifiedUTF8(POSTCHAR, dos);
+ }
+ }
- public void setNumPrePostChars(int numPreChars, int numPostChars) {
- this.numPreChars = numPreChars;
- this.numPostChars = numPostChars;
- }
+ public void setNumPrePostChars(int numPreChars, int numPostChars) {
+ this.numPreChars = numPreChars;
+ this.numPostChars = numPostChars;
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/UTF8WordToken.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/UTF8WordToken.java
index 25e0cd3..97a1e12 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/UTF8WordToken.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/tokenizers/UTF8WordToken.java
@@ -22,25 +22,26 @@
import java.io.DataOutput;
import java.io.IOException;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.data.util.StringUtils;
public class UTF8WordToken extends AbstractUTF8Token {
- public UTF8WordToken(byte tokenTypeTag, byte countTypeTag) {
- super(tokenTypeTag, countTypeTag);
- }
+ public UTF8WordToken(byte tokenTypeTag, byte countTypeTag) {
+ super(tokenTypeTag, countTypeTag);
+ }
- @Override
- public void serializeToken(DataOutput dos) throws IOException {
- handleTokenTypeTag(dos);
+ @Override
+ public void serializeToken(DataOutput dos) throws IOException {
+ handleTokenTypeTag(dos);
- int tokenUTF8Len = getLowerCaseUTF8Len(tokenLength);
- StringUtils.writeUTF8Len(tokenUTF8Len, dos);
- int pos = start;
- for (int i = 0; i < tokenLength; i++) {
- char c = StringUtils.toLowerCase(StringUtils.charAt(data, pos));
- StringUtils.writeCharAsModifiedUTF8(c, dos);
- pos += StringUtils.charSize(data, pos);
- }
- }
+ int tokenUTF8Len = getLowerCaseUTF8Len(tokenLength);
+ StringUtils.writeUTF8Len(tokenUTF8Len, dos);
+ int pos = start;
+ for (int i = 0; i < tokenLength; i++) {
+ char c = Character.toLowerCase(UTF8StringPointable.charAt(data, pos));
+ StringUtils.writeCharAsModifiedUTF8(c, dos);
+ pos += UTF8StringPointable.charSize(data, pos);
+ }
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/util/InvertedIndexUtils.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/util/InvertedIndexUtils.java
new file mode 100644
index 0000000..a1d1f06
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/util/InvertedIndexUtils.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.util;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+
+public class InvertedIndexUtils {
+ // Type traits to be appended to the token type trait which finally form the BTree field type traits.
+ private static final ITypeTraits[] btreeValueTypeTraits = new ITypeTraits[4];
+ static {
+ // startPageId
+ btreeValueTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ // endPageId
+ btreeValueTypeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ // startOff
+ btreeValueTypeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ // numElements
+ btreeValueTypeTraits[3] = IntegerPointable.TYPE_TRAITS;
+ }
+
+ public static ITypeTraits[] getBTreeTypeTraits(ITypeTraits[] tokenTypeTraits) {
+ ITypeTraits[] btreeTypeTraits = new ITypeTraits[tokenTypeTraits.length + btreeValueTypeTraits.length];
+ // Set key type traits.
+ for (int i = 0; i < tokenTypeTraits.length; i++) {
+ btreeTypeTraits[i] = tokenTypeTraits[i];
+ }
+ // Set value type traits.
+ for (int i = 0; i < btreeValueTypeTraits.length; i++) {
+ btreeTypeTraits[i + tokenTypeTraits.length] = btreeValueTypeTraits[i];
+ }
+ return btreeTypeTraits;
+ }
+}
diff --git a/hyracks-storage-am-rtree/.classpath b/hyracks-storage-am-rtree/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-storage-am-rtree/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-storage-am-rtree/.project b/hyracks-storage-am-rtree/.project
deleted file mode 100644
index 4989318..0000000
--- a/hyracks-storage-am-rtree/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-storage-am-rtree</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-storage-am-rtree/.settings/org.eclipse.jdt.core.prefs b/hyracks-storage-am-rtree/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 375e12e..0000000
--- a/hyracks-storage-am-rtree/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:07 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-storage-am-rtree/.settings/org.maven.ide.eclipse.prefs b/hyracks-storage-am-rtree/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index ba4f536..0000000
--- a/hyracks-storage-am-rtree/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Wed Feb 02 19:49:19 PST 2011
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-storage-am-rtree/pom.xml b/hyracks-storage-am-rtree/pom.xml
index 10f118cf..9abdd00 100644
--- a/hyracks-storage-am-rtree/pom.xml
+++ b/hyracks-storage-am-rtree/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-rtree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,28 +27,28 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeFrame.java
index e3b9b96..dd57986 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeFrame.java
@@ -15,33 +15,25 @@
package edu.uci.ics.hyracks.storage.am.rtree.api;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.Rectangle;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.TupleEntryArrayList;
public interface IRTreeFrame extends ITreeIndexFrame {
- public ITreeIndexTupleReference createTupleReference();
+ public void computeMBR(ISplitKey splitKey);
- public void computeMBR(ISplitKey splitKey, MultiComparator cmp);
+ public void delete(int tupleIndex, MultiComparator cmp);
- public void insert(ITupleReference tuple, MultiComparator cmp,
- int tupleIndex) throws Exception;
+ public long getPageNsn();
- public void delete(int tupleIndex, MultiComparator cmp) throws Exception;
-
- public int getPageNsn();
-
- public void setPageNsn(int pageNsn);
+ public void setPageNsn(long pageNsn);
public int getRightPage();
public void setRightPage(int rightPage);
- public void adjustMBR(ITreeIndexTupleReference[] tuples, MultiComparator cmp);
+ public void adjustMBR(ITreeIndexTupleReference[] tuples);
}
\ No newline at end of file
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
index 4c251d2..2b3065d 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/api/IRTreeInteriorFrame.java
@@ -23,7 +23,7 @@
public boolean findBestChild(ITupleReference tuple, MultiComparator cmp);
- public int getBestChildPageId(MultiComparator cmp);
+ public int getBestChildPageId();
public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex,
MultiComparator cmp);
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
new file mode 100644
index 0000000..51abb79
--- /dev/null
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelper.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
+import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+public class RTreeDataflowHelper extends TreeIndexDataflowHelper {
+
+ public RTreeDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx, int partition,
+ boolean createIfNotExists) {
+ super(opDesc, ctx, partition, createIfNotExists);
+ }
+
+ @Override
+ public ITreeIndex createIndexInstance() throws HyracksDataException {
+ MultiComparator cmp = IndexUtils.createMultiComparator(treeOpDesc.getTreeIndexComparatorFactories());
+ IBufferCache bufferCache = treeOpDesc.getStorageManager().getBufferCache(ctx);
+ ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, indexFileId, 0,
+ metaDataFrameFactory);
+ return new RTree(bufferCache, treeOpDesc.getTreeIndexTypeTraits().length, cmp, freePageManager,
+ treeOpDesc.getTreeIndexInteriorFactory(), treeOpDesc.getTreeIndexLeafFactory());
+ }
+}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java
new file mode 100644
index 0000000..8cd4e95
--- /dev/null
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeDataflowHelperFactory.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree.dataflow;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexDataflowHelper;
+
+public class RTreeDataflowHelperFactory implements IIndexDataflowHelperFactory {
+
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IndexDataflowHelper createIndexDataflowHelper(IIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
+ int partition, boolean createIfNotExists) {
+ return new RTreeDataflowHelper(opDesc, ctx, partition, createIfNotExists);
+ }
+}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeOpHelper.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeOpHelper.java
deleted file mode 100644
index cb7c9d9..0000000
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeOpHelper.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.rtree.dataflow;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptorHelper;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexHelperOpenMode;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexOpHelper;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-
-public class RTreeOpHelper extends TreeIndexOpHelper {
-
- public RTreeOpHelper(ITreeIndexOperatorDescriptorHelper opDesc,
- IHyracksTaskContext ctx, int partition, IndexHelperOpenMode mode) {
- super(opDesc, ctx, partition, mode);
- }
-
- public ITreeIndex createTreeIndex() throws HyracksDataException {
- IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(
- ctx);
- ITreeIndexMetaDataFrameFactory metaDataFrameFactory = new LIFOMetaDataFrameFactory();
- IFreePageManager freePageManager = new LinkedListFreePageManager(
- bufferCache, indexFileId, 0, metaDataFrameFactory);
-
- return new RTree(bufferCache, freePageManager,
- opDesc.getTreeIndexInteriorFactory(),
- opDesc.getTreeIndexLeafFactory(), cmp);
- }
-
- public MultiComparator createMultiComparator(IBinaryComparator[] comparators)
- throws HyracksDataException {
- IPrimitiveValueProvider[] keyValueProvider = new IPrimitiveValueProvider[opDesc
- .getTreeIndexValueProviderFactories().length];
- for (int i = 0; i < opDesc.getTreeIndexComparatorFactories().length; i++) {
- keyValueProvider[i] = opDesc.getTreeIndexValueProviderFactories()[i]
- .createPrimitiveValueProvider();
- }
- return new MultiComparator(opDesc.getTreeIndexTypeTraits(),
- comparators, keyValueProvider);
- }
-}
\ No newline at end of file
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeOpHelperFactory.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeOpHelperFactory.java
deleted file mode 100644
index 6afc37e..0000000
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeOpHelperFactory.java
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.rtree.dataflow;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptorHelper;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexHelperOpenMode;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexOpHelper;
-
-public class RTreeOpHelperFactory implements ITreeIndexOpHelperFactory {
-
- private static final long serialVersionUID = 1L;
-
- @Override
- public TreeIndexOpHelper createTreeIndexOpHelper(
- ITreeIndexOperatorDescriptorHelper opDesc, IHyracksTaskContext ctx,
- int partition, IndexHelperOpenMode mode) {
- return new RTreeOpHelper(opDesc, ctx, partition, mode);
- }
-}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
index f5741c0..8cb4e55 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorDescriptor.java
@@ -19,48 +19,37 @@
import edu.uci.ics.hyracks.api.dataflow.IOperatorNodePushable;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IRecordDescriptorProvider;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.job.IOperatorEnvironment;
import edu.uci.ics.hyracks.api.job.JobSpecification;
import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-public class RTreeSearchOperatorDescriptor extends
- AbstractTreeIndexOperatorDescriptor {
+public class RTreeSearchOperatorDescriptor extends AbstractTreeIndexOperatorDescriptor {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 1L;
- private int[] keyFields; // fields in input tuple to be used as keys
+ private int[] keyFields; // fields in input tuple to be used as keys
- public RTreeSearchOperatorDescriptor(JobSpecification spec,
- RecordDescriptor recDesc, IStorageManagerInterface storageManager,
- IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
- IFileSplitProvider fileSplitProvider,
- ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] typeTraits,
- IBinaryComparatorFactory[] comparatorFactories,
- IPrimitiveValueProviderFactory[] valueProviderFactories,
- int[] keyFields, ITreeIndexOpHelperFactory opHelperFactory) {
- super(spec, 1, 1, recDesc, storageManager, treeIndexRegistryProvider,
- fileSplitProvider, interiorFrameFactory, leafFrameFactory,
- typeTraits, comparatorFactories, valueProviderFactories,
- opHelperFactory);
- this.keyFields = keyFields;
- }
+ public RTreeSearchOperatorDescriptor(JobSpecification spec, RecordDescriptor recDesc,
+ IStorageManagerInterface storageManager, IIndexRegistryProvider<IIndex> indexRegistryProvider,
+ IFileSplitProvider fileSplitProvider, ITreeIndexFrameFactory interiorFrameFactory,
+ ITreeIndexFrameFactory leafFrameFactory, ITypeTraits[] typeTraits,
+ IBinaryComparatorFactory[] comparatorFactories, int[] keyFields,
+ IIndexDataflowHelperFactory dataflowHelperFactory) {
+ super(spec, 1, 1, recDesc, storageManager, indexRegistryProvider, fileSplitProvider, interiorFrameFactory,
+ leafFrameFactory, typeTraits, comparatorFactories, dataflowHelperFactory);
+ this.keyFields = keyFields;
+ }
- @Override
- public IOperatorNodePushable createPushRuntime(
- final IHyracksTaskContext ctx, final IOperatorEnvironment env,
- IRecordDescriptorProvider recordDescProvider, int partition,
- int nPartitions) {
- return new RTreeSearchOperatorNodePushable(this, ctx, partition,
- recordDescProvider, keyFields);
- }
+ @Override
+ public IOperatorNodePushable createPushRuntime(final IHyracksTaskContext ctx,
+ IRecordDescriptorProvider recordDescProvider, int partition, int nPartitions) {
+ return new RTreeSearchOperatorNodePushable(this, ctx, partition, recordDescProvider, keyFields);
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
index a1e5162..9f62b8f 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/dataflow/RTreeSearchOperatorNodePushable.java
@@ -29,24 +29,21 @@
import edu.uci.ics.hyracks.dataflow.common.comm.util.FrameUtils;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.dataflow.AbstractTreeIndexOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexHelperOpenMode;
import edu.uci.ics.hyracks.storage.am.common.dataflow.PermutingFrameTupleReference;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexOpHelper;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.TreeIndexDataflowHelper;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeOpContext;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
public class RTreeSearchOperatorNodePushable extends AbstractUnaryInputUnaryOutputOperatorNodePushable {
- private TreeIndexOpHelper treeIndexOpHelper;
+ private TreeIndexDataflowHelper treeIndexHelper;
private FrameTupleAccessor accessor;
private ByteBuffer writeBuffer;
@@ -61,15 +58,14 @@
private ITreeIndexCursor cursor;
private ITreeIndexFrame interiorFrame;
private ITreeIndexFrame leafFrame;
- private RTreeOpContext opCtx;
+ private ITreeIndexAccessor indexAccessor;
private RecordDescriptor recDesc;
public RTreeSearchOperatorNodePushable(AbstractTreeIndexOperatorDescriptor opDesc, IHyracksTaskContext ctx,
int partition, IRecordDescriptorProvider recordDescProvider, int[] keyFields) {
- treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory().createTreeIndexOpHelper(opDesc, ctx, partition,
- IndexHelperOpenMode.OPEN);
-
+ treeIndexHelper = (TreeIndexDataflowHelper) opDesc.getIndexDataflowHelperFactory().createIndexDataflowHelper(
+ opDesc, ctx, partition, false);
this.recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
if (keyFields != null && keyFields.length > 0) {
searchKey = new PermutingFrameTupleReference();
@@ -79,52 +75,37 @@
@Override
public void open() throws HyracksDataException {
- AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexOpHelper
+ AbstractTreeIndexOperatorDescriptor opDesc = (AbstractTreeIndexOperatorDescriptor) treeIndexHelper
.getOperatorDescriptor();
- accessor = new FrameTupleAccessor(treeIndexOpHelper.getHyracksTaskContext().getFrameSize(), recDesc);
-
+ accessor = new FrameTupleAccessor(treeIndexHelper.getHyracksTaskContext().getFrameSize(), recDesc);
interiorFrame = opDesc.getTreeIndexInteriorFactory().createFrame();
leafFrame = opDesc.getTreeIndexLeafFactory().createFrame();
cursor = new RTreeSearchCursor((IRTreeInteriorFrame) interiorFrame, (IRTreeLeafFrame) leafFrame);
-
try {
-
- treeIndexOpHelper.init();
+ treeIndexHelper.init();
writer.open();
try {
- rtree = (RTree) treeIndexOpHelper.getTreeIndex();
-
+ rtree = (RTree) treeIndexHelper.getIndex();
int keySearchFields = rtree.getCmp().getComparators().length;
-
IBinaryComparator[] keySearchComparators = new IBinaryComparator[keySearchFields];
for (int i = 0; i < keySearchFields; i++) {
keySearchComparators[i] = rtree.getCmp().getComparators()[i];
}
-
- IPrimitiveValueProvider[] keyValueProvider = new IPrimitiveValueProvider[keySearchFields];
- for (int i = 0; i < keySearchFields; i++) {
- keyValueProvider[i] = rtree.getCmp().getValueProviders()[i];
- }
-
- cmp = new MultiComparator(rtree.getCmp().getTypeTraits(), keySearchComparators, keyValueProvider);
-
+ cmp = new MultiComparator(keySearchComparators);
searchPred = new SearchPredicate(searchKey, cmp);
-
- writeBuffer = treeIndexOpHelper.getHyracksTaskContext().allocateFrame();
- tb = new ArrayTupleBuilder(rtree.getCmp().getFieldCount());
+ writeBuffer = treeIndexHelper.getHyracksTaskContext().allocateFrame();
+ tb = new ArrayTupleBuilder(rtree.getFieldCount());
dos = tb.getDataOutput();
- appender = new FrameTupleAppender(treeIndexOpHelper.getHyracksTaskContext().getFrameSize());
+ appender = new FrameTupleAppender(treeIndexHelper.getHyracksTaskContext().getFrameSize());
appender.reset(writeBuffer, true);
-
- opCtx = rtree.createOpContext(IndexOp.SEARCH, treeIndexOpHelper.getLeafFrame(),
- treeIndexOpHelper.getInteriorFrame(), null);
+ indexAccessor = rtree.createAccessor();
} catch (Exception e) {
writer.fail();
throw e;
}
} catch (Exception e) {
- treeIndexOpHelper.deinit();
+ treeIndexHelper.deinit();
throw new HyracksDataException(e);
}
}
@@ -161,7 +142,7 @@
searchPred.setSearchKey(searchKey);
cursor.reset();
- rtree.search(cursor, searchPred, opCtx);
+ indexAccessor.search(cursor, searchPred);
writeSearchResults();
}
} catch (Exception e) {
@@ -182,7 +163,7 @@
throw new HyracksDataException(e);
}
} finally {
- treeIndexOpHelper.deinit();
+ treeIndexHelper.deinit();
}
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
index e809884..546a4d4 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
@@ -19,6 +19,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
@@ -34,7 +35,7 @@
public abstract class RTreeNSMFrame extends TreeIndexNSMFrame implements
IRTreeFrame {
protected static final int pageNsnOff = smFlagOff + 1;
- protected static final int rightPageOff = pageNsnOff + 4;
+ protected static final int rightPageOff = pageNsnOff + 8;
protected ITreeIndexTupleReference[] tuples;
protected ITreeIndexTupleReference cmpFrameTuple;
@@ -48,11 +49,12 @@
protected static final int nearMinimumOverlapFactor = 32;
private static final double doubleEpsilon = computeDoubleEpsilon();
private static final int numTuplesEntries = 100;
-
- public RTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, int keyFieldCount) {
- super(tupleWriter, new UnorderedSlotManager());
- this.tuples = new ITreeIndexTupleReference[keyFieldCount];
- for (int i = 0; i < keyFieldCount; i++) {
+ protected final IPrimitiveValueProvider[] keyValueProviders;
+
+ public RTreeNSMFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders) {
+ super(tupleWriter, new UnorderedSlotManager());
+ this.tuples = new ITreeIndexTupleReference[keyValueProviders.length];
+ for (int i = 0; i < keyValueProviders.length; i++) {
this.tuples[i] = tupleWriter.createTupleReference();
}
cmpFrameTuple = tupleWriter.createTupleReference();
@@ -63,8 +65,9 @@
numTuplesEntries);
rec = new Rectangle[4];
for (int i = 0; i < 4; i++) {
- rec[i] = new Rectangle(keyFieldCount / 2);
+ rec[i] = new Rectangle(keyValueProviders.length / 2);
}
+ this.keyValueProviders = keyValueProviders;
}
private static double computeDoubleEpsilon() {
@@ -83,7 +86,7 @@
@Override
public void initBuffer(byte level) {
super.initBuffer(level);
- buf.putInt(pageNsnOff, 0);
+ buf.putLong(pageNsnOff, 0);
buf.putInt(rightPageOff, -1);
}
@@ -92,13 +95,13 @@
}
@Override
- public void setPageNsn(int pageNsn) {
- buf.putInt(pageNsnOff, pageNsn);
+ public void setPageNsn(long pageNsn) {
+ buf.putLong(pageNsnOff, pageNsn);
}
@Override
- public int getPageNsn() {
- return buf.getInt(pageNsnOff);
+ public long getPageNsn() {
+ return buf.getLong(pageNsnOff);
}
@Override
@@ -141,22 +144,21 @@
}
public void generateDist(ITupleReference tuple,
- TupleEntryArrayList entries, Rectangle rec, int start, int end,
- MultiComparator cmp) {
+ TupleEntryArrayList entries, Rectangle rec, int start, int end) {
int j = 0;
while (entries.get(j).getTupleIndex() == -1) {
j++;
}
frameTuple.resetByTupleIndex(this, entries.get(j).getTupleIndex());
- rec.set(frameTuple, cmp);
+ rec.set(frameTuple, keyValueProviders);
for (int i = start; i < end; ++i) {
if (i != j) {
if (entries.get(i).getTupleIndex() != -1) {
frameTuple.resetByTupleIndex(this, entries.get(i)
.getTupleIndex());
- rec.enlarge(frameTuple, cmp);
+ rec.enlarge(frameTuple, keyValueProviders);
} else {
- rec.enlarge(tuple, cmp);
+ rec.enlarge(tuple, keyValueProviders);
}
}
}
@@ -167,28 +169,20 @@
return tupleWriter.createTupleReference();
}
- public void adjustMBRImpl(ITreeIndexTupleReference[] tuples,
- MultiComparator cmp) {
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ public void adjustMBRImpl(ITreeIndexTupleReference[] tuples) {
+ int maxFieldPos = keyValueProviders.length / 2;
for (int i = 1; i < getTupleCount(); i++) {
frameTuple.resetByTupleIndex(this, i);
for (int j = 0; j < maxFieldPos; j++) {
int k = maxFieldPos + j;
- int c = cmp.getComparators()[j].compare(
- frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j),
- frameTuple.getFieldLength(j),
- tuples[j].getFieldData(j), tuples[j].getFieldStart(j),
- tuples[j].getFieldLength(j));
- if (c < 0) {
+ double valA = keyValueProviders[j].getValue(frameTuple.getFieldData(j), frameTuple.getFieldStart(j));
+ double valB = keyValueProviders[j].getValue(tuples[j].getFieldData(j), tuples[j].getFieldStart(j));
+ if (valA < valB) {
tuples[j].resetByTupleIndex(this, i);
}
- c = cmp.getComparators()[k].compare(frameTuple.getFieldData(k),
- frameTuple.getFieldStart(k),
- frameTuple.getFieldLength(k),
- tuples[k].getFieldData(k), tuples[k].getFieldStart(k),
- tuples[k].getFieldLength(k));
- if (c > 0) {
+ valA = keyValueProviders[k].getValue(frameTuple.getFieldData(k), frameTuple.getFieldStart(k));
+ valB = keyValueProviders[k].getValue(tuples[k].getFieldData(k), tuples[k].getFieldStart(k));
+ if (valA > valB) {
tuples[k].resetByTupleIndex(this, i);
}
}
@@ -196,18 +190,17 @@
}
@Override
- public void computeMBR(ISplitKey splitKey, MultiComparator cmp) {
+ public void computeMBR(ISplitKey splitKey) {
RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter);
- frameTuple.setFieldCount(cmp.getFieldCount());
int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
frameTuple.resetByTupleOffset(buf, tupleOff);
int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0,
- cmp.getKeyFieldCount());
+ keyValueProviders.length);
splitKey.initData(splitKeySize);
- this.adjustMBR(tuples, cmp);
+ this.adjustMBR(tuples);
rTreeTupleWriterLeftFrame.writeTupleFields(tuples, 0,
rTreeSplitKey.getLeftPageBuffer(), 0);
rTreeSplitKey.getLeftTuple().resetByTupleOffset(
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
index 38d2a46..f3e6be2 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrame.java
@@ -15,20 +15,20 @@
package edu.uci.ics.hyracks.storage.am.rtree.frames;
-import java.io.ByteArrayInputStream;
-import java.io.DataInput;
-import java.io.DataInputStream;
import java.util.ArrayList;
import java.util.Collections;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.SlotOffTupleOff;
@@ -40,685 +40,579 @@
import edu.uci.ics.hyracks.storage.am.rtree.impls.UnorderedSlotManager;
import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriter;
-public class RTreeNSMInteriorFrame extends RTreeNSMFrame implements
- IRTreeInteriorFrame {
+public class RTreeNSMInteriorFrame extends RTreeNSMFrame implements IRTreeInteriorFrame {
- private static final int childPtrSize = 4;
+ private static final int childPtrSize = 4;
+ private static IBinaryComparator childPtrCmp = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY)
+ .createBinaryComparator();
- public RTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter,
- int keyFieldCount) {
- super(tupleWriter, keyFieldCount);
- }
+ public RTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders) {
+ super(tupleWriter, keyValueProviders);
+ frameTuple.setFieldCount(keyValueProviders.length);
+ }
- @Override
- public String printKeys(MultiComparator cmp,
- ISerializerDeserializer[] fields) throws HyracksDataException {
- StringBuilder strBuilder = new StringBuilder();
- int tupleCount = buf.getInt(tupleCountOff);
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- for (int i = 0; i < tupleCount; i++) {
- frameTuple.resetByTupleIndex(this, i);
- for (int j = 0; j < cmp.getKeyFieldCount(); j++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j),
- frameTuple.getFieldLength(j));
- DataInput dataIn = new DataInputStream(inStream);
- Object o = fields[j].deserialize(dataIn);
- strBuilder.append(o.toString() + " ");
- }
- strBuilder.append(" | ");
- }
- strBuilder.append("\n");
- return strBuilder.toString();
- }
+ @Override
+ public boolean findBestChild(ITupleReference tuple, MultiComparator cmp) {
+ cmpFrameTuple.setFieldCount(cmp.getKeyFieldCount());
+ frameTuple.setFieldCount(cmp.getKeyFieldCount());
- @Override
- public boolean findBestChild(ITupleReference tuple, MultiComparator cmp) {
- cmpFrameTuple.setFieldCount(cmp.getKeyFieldCount());
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ int bestChild = 0;
+ double minEnlargedArea = Double.MAX_VALUE;
- int bestChild = 0;
- double minEnlargedArea = Double.MAX_VALUE;
+ // the children pointers in the node point to leaves
+ if (getLevel() == 1) {
+ // find least overlap enlargement, use minimum enlarged area to
+ // break tie, if tie still exists use minimum area to break it
+ for (int i = 0; i < getTupleCount(); ++i) {
+ frameTuple.resetByTupleIndex(this, i);
+ double enlargedArea = enlargedArea(frameTuple, tuple, cmp);
+ tupleEntries1.add(i, enlargedArea);
+ if (enlargedArea < minEnlargedArea) {
+ minEnlargedArea = enlargedArea;
+ bestChild = i;
+ }
+ }
+ if (minEnlargedArea < RTreeNSMFrame.doubleEpsilon() || minEnlargedArea > RTreeNSMFrame.doubleEpsilon()) {
+ minEnlargedArea = Double.MAX_VALUE;
+ int k;
+ if (getTupleCount() > nearMinimumOverlapFactor) {
+ // sort the entries based on their area enlargement needed
+ // to include the object
+ tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount());
+ k = nearMinimumOverlapFactor;
+ } else {
+ k = getTupleCount();
+ }
- // the children pointers in the node point to leaves
- if (getLevel() == 1) {
- // find least overlap enlargement, use minimum enlarged area to
- // break tie, if tie still exists use minimum area to break it
- for (int i = 0; i < getTupleCount(); ++i) {
- frameTuple.resetByTupleIndex(this, i);
- double enlargedArea = enlargedArea(frameTuple, tuple, cmp);
- tupleEntries1.add(i, enlargedArea);
- if (enlargedArea < minEnlargedArea) {
- minEnlargedArea = enlargedArea;
- bestChild = i;
- }
- }
- if (minEnlargedArea < RTreeNSMFrame.doubleEpsilon()
- || minEnlargedArea > RTreeNSMFrame.doubleEpsilon()) {
- minEnlargedArea = Double.MAX_VALUE;
- int k;
- if (getTupleCount() > nearMinimumOverlapFactor) {
- // sort the entries based on their area enlargement needed
- // to include the object
- tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount());
- k = nearMinimumOverlapFactor;
- } else {
- k = getTupleCount();
- }
+ double minOverlap = Double.MAX_VALUE;
+ int id = 0;
+ for (int i = 0; i < k; ++i) {
+ double difference = 0.0;
+ for (int j = 0; j < getTupleCount(); ++j) {
+ frameTuple.resetByTupleIndex(this, j);
+ cmpFrameTuple.resetByTupleIndex(this, tupleEntries1.get(i).getTupleIndex());
- double minOverlap = Double.MAX_VALUE;
- int id = 0;
- for (int i = 0; i < k; ++i) {
- double difference = 0.0;
- for (int j = 0; j < getTupleCount(); ++j) {
- frameTuple.resetByTupleIndex(this, j);
- cmpFrameTuple.resetByTupleIndex(this, tupleEntries1
- .get(i).getTupleIndex());
+ int c = pointerCmp(frameTuple, cmpFrameTuple, cmp);
+ if (c != 0) {
+ double intersection = overlappedArea(frameTuple, tuple, cmpFrameTuple, cmp);
+ if (intersection != 0.0) {
+ difference += intersection - overlappedArea(frameTuple, null, cmpFrameTuple, cmp);
+ }
+ } else {
+ id = j;
+ }
+ }
- int c = pointerCmp(frameTuple, cmpFrameTuple, cmp);
- if (c != 0) {
- double intersection = overlappedArea(frameTuple,
- tuple, cmpFrameTuple, cmp);
- if (intersection != 0.0) {
- difference += intersection
- - overlappedArea(frameTuple, null,
- cmpFrameTuple, cmp);
- }
- } else {
- id = j;
- }
- }
+ double enlargedArea = enlargedArea(cmpFrameTuple, tuple, cmp);
+ if (difference < minOverlap) {
+ minOverlap = difference;
+ minEnlargedArea = enlargedArea;
+ bestChild = id;
+ } else if (difference == minOverlap) {
+ if (enlargedArea < minEnlargedArea) {
+ minEnlargedArea = enlargedArea;
+ bestChild = id;
+ } else if (enlargedArea == minEnlargedArea) {
+ double area = area(cmpFrameTuple, cmp);
+ frameTuple.resetByTupleIndex(this, bestChild);
+ double minArea = area(frameTuple, cmp);
+ if (area < minArea) {
+ bestChild = id;
+ }
+ }
+ }
+ }
+ }
+ } else { // find minimum enlarged area, use minimum area to break tie
+ for (int i = 0; i < getTupleCount(); i++) {
+ frameTuple.resetByTupleIndex(this, i);
+ double enlargedArea = enlargedArea(frameTuple, tuple, cmp);
+ if (enlargedArea < minEnlargedArea) {
+ minEnlargedArea = enlargedArea;
+ bestChild = i;
+ } else if (enlargedArea == minEnlargedArea) {
+ double area = area(frameTuple, cmp);
+ frameTuple.resetByTupleIndex(this, bestChild);
+ double minArea = area(frameTuple, cmp);
+ if (area < minArea) {
+ bestChild = i;
+ }
+ }
+ }
+ }
+ tupleEntries1.clear();
- double enlargedArea = enlargedArea(cmpFrameTuple, tuple,
- cmp);
- if (difference < minOverlap) {
- minOverlap = difference;
- minEnlargedArea = enlargedArea;
- bestChild = id;
- } else if (difference == minOverlap) {
- if (enlargedArea < minEnlargedArea) {
- minEnlargedArea = enlargedArea;
- bestChild = id;
- } else if (enlargedArea == minEnlargedArea) {
- double area = area(cmpFrameTuple, cmp);
- frameTuple.resetByTupleIndex(this, bestChild);
- double minArea = area(frameTuple, cmp);
- if (area < minArea) {
- bestChild = id;
- }
- }
- }
- }
- }
- } else { // find minimum enlarged area, use minimum area to break tie
- for (int i = 0; i < getTupleCount(); i++) {
- frameTuple.resetByTupleIndex(this, i);
- double enlargedArea = enlargedArea(frameTuple, tuple, cmp);
- if (enlargedArea < minEnlargedArea) {
- minEnlargedArea = enlargedArea;
- bestChild = i;
- } else if (enlargedArea == minEnlargedArea) {
- double area = area(frameTuple, cmp);
- frameTuple.resetByTupleIndex(this, bestChild);
- double minArea = area(frameTuple, cmp);
- if (area < minArea) {
- bestChild = i;
- }
- }
- }
- }
- tupleEntries1.clear();
+ frameTuple.resetByTupleIndex(this, bestChild);
+ if (minEnlargedArea > 0.0) {
+ return true;
+ } else {
+ return false;
+ }
+ }
- frameTuple.resetByTupleIndex(this, bestChild);
- if (minEnlargedArea > 0.0) {
- return true;
- } else {
- return false;
- }
- }
+ @Override
+ public int getBestChildPageId() {
+ return buf.getInt(getChildPointerOff(frameTuple));
+ }
- @Override
- public int getBestChildPageId(MultiComparator cmp) {
- return buf.getInt(getChildPointerOff(frameTuple, cmp));
- }
+ @Override
+ public int findTupleByPointer(ITupleReference tuple, MultiComparator cmp) {
+ frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ for (int i = 0; i < getTupleCount(); i++) {
+ frameTuple.resetByTupleIndex(this, i);
+ int c = pointerCmp(frameTuple, tuple, cmp);
+ if (c == 0) {
+ return i;
+ }
+ }
+ return -1;
+ }
- @Override
- public int findTupleByPointer(ITupleReference tuple, MultiComparator cmp) {
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- for (int i = 0; i < getTupleCount(); i++) {
- frameTuple.resetByTupleIndex(this, i);
- int c = pointerCmp(frameTuple, tuple, cmp);
- if (c == 0) {
- return i;
- }
- }
- return -1;
- }
+ @Override
+ public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex, MultiComparator cmp) {
+ frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i), frameTuple.getFieldData(j), frameTuple.getFieldStart(j),
+ frameTuple.getFieldLength(j));
+ if (c > 0) {
+ return -1;
+ }
+ c = cmp.getComparators()[i].compare(tuple.getFieldData(j), tuple.getFieldStart(j), tuple.getFieldLength(j),
+ frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
+ if (c < 0) {
+ return -1;
+ }
+ }
+ return buf.getInt(getChildPointerOff(frameTuple));
+ }
- @Override
- public int getChildPageIdIfIntersect(ITupleReference tuple, int tupleIndex,
- MultiComparator cmp) {
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- frameTuple.resetByTupleIndex(this, tupleIndex);
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(tuple.getFieldData(i),
- tuple.getFieldStart(i), tuple.getFieldLength(i),
- frameTuple.getFieldData(j), frameTuple.getFieldStart(j),
- frameTuple.getFieldLength(j));
- if (c > 0) {
- return -1;
- }
- c = cmp.getComparators()[i].compare(tuple.getFieldData(j),
- tuple.getFieldStart(j), tuple.getFieldLength(j),
- frameTuple.getFieldData(i), frameTuple.getFieldStart(i),
- frameTuple.getFieldLength(i));
- if (c < 0) {
- return -1;
- }
- }
- return buf.getInt(getChildPointerOff(frameTuple, cmp));
- }
+ @Override
+ public int findTupleByPointer(ITupleReference tuple, PathList traverseList, int parentIndex, MultiComparator cmp) {
+ frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ for (int i = 0; i < getTupleCount(); i++) {
+ frameTuple.resetByTupleIndex(this, i);
- @Override
- public int findTupleByPointer(ITupleReference tuple, PathList traverseList,
- int parentIndex, MultiComparator cmp) {
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- for (int i = 0; i < getTupleCount(); i++) {
- frameTuple.resetByTupleIndex(this, i);
+ int c = pointerCmp(frameTuple, tuple, cmp);
+ if (c == 0) {
+ return i;
+ } else {
+ int pageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(cmp.getKeyFieldCount() - 1),
+ getChildPointerOff(frameTuple));
+ traverseList.add(pageId, -1, parentIndex);
+ }
+ }
+ return -1;
+ }
- int c = pointerCmp(frameTuple, tuple, cmp);
- if (c == 0) {
- return i;
- } else {
- int pageId = IntegerSerializerDeserializer.getInt(
- frameTuple.getFieldData(cmp.getKeyFieldCount() - 1),
- getChildPointerOff(frameTuple, cmp));
- traverseList.add(pageId, -1, parentIndex);
- }
- }
- return -1;
- }
+ @Override
+ public boolean compact() {
+ resetSpaceParams();
- @Override
- public boolean compact(MultiComparator cmp) {
- resetSpaceParams();
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ int tupleCount = buf.getInt(tupleCountOff);
+ int freeSpace = buf.getInt(freeSpaceOff);
- int tupleCount = buf.getInt(tupleCountOff);
- int freeSpace = buf.getInt(freeSpaceOff);
+ ArrayList<SlotOffTupleOff> sortedTupleOffs = new ArrayList<SlotOffTupleOff>();
+ sortedTupleOffs.ensureCapacity(tupleCount);
+ for (int i = 0; i < tupleCount; i++) {
+ int slotOff = slotManager.getSlotOff(i);
+ int tupleOff = slotManager.getTupleOff(slotOff);
+ sortedTupleOffs.add(new SlotOffTupleOff(i, slotOff, tupleOff));
+ }
+ Collections.sort(sortedTupleOffs);
- ArrayList<SlotOffTupleOff> sortedTupleOffs = new ArrayList<SlotOffTupleOff>();
- sortedTupleOffs.ensureCapacity(tupleCount);
- for (int i = 0; i < tupleCount; i++) {
- int slotOff = slotManager.getSlotOff(i);
- int tupleOff = slotManager.getTupleOff(slotOff);
- sortedTupleOffs.add(new SlotOffTupleOff(i, slotOff, tupleOff));
- }
- Collections.sort(sortedTupleOffs);
+ for (int i = 0; i < sortedTupleOffs.size(); i++) {
+ int tupleOff = sortedTupleOffs.get(i).tupleOff;
+ frameTuple.resetByTupleOffset(buf, tupleOff);
- for (int i = 0; i < sortedTupleOffs.size(); i++) {
- int tupleOff = sortedTupleOffs.get(i).tupleOff;
- frameTuple.resetByTupleOffset(buf, tupleOff);
+ int tupleEndOff = frameTuple.getFieldStart(frameTuple.getFieldCount() - 1)
+ + frameTuple.getFieldLength(frameTuple.getFieldCount() - 1);
+ int tupleLength = tupleEndOff - tupleOff + childPtrSize;
+ System.arraycopy(buf.array(), tupleOff, buf.array(), freeSpace, tupleLength);
- int tupleEndOff = frameTuple.getFieldStart(frameTuple
- .getFieldCount() - 1)
- + frameTuple.getFieldLength(frameTuple.getFieldCount() - 1);
- int tupleLength = tupleEndOff - tupleOff + childPtrSize;
- System.arraycopy(buf.array(), tupleOff, buf.array(), freeSpace,
- tupleLength);
+ slotManager.setSlot(sortedTupleOffs.get(i).slotOff, freeSpace);
+ freeSpace += tupleLength;
+ }
- slotManager.setSlot(sortedTupleOffs.get(i).slotOff, freeSpace);
- freeSpace += tupleLength;
- }
+ buf.putInt(freeSpaceOff, freeSpace);
+ buf.putInt(totalFreeSpaceOff, buf.capacity() - freeSpace - tupleCount * slotManager.getSlotSize());
- buf.putInt(freeSpaceOff, freeSpace);
- buf.putInt(totalFreeSpaceOff, buf.capacity() - freeSpace - tupleCount
- * slotManager.getSlotSize());
+ return false;
+ }
- return false;
- }
+ @Override
+ public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple) {
+ int bytesRequired = tupleWriter.bytesRequired(tuple) + childPtrSize; // for
+ // the
+ // child
+ // pointer
+ if (bytesRequired + slotManager.getSlotSize() <= buf.capacity() - buf.getInt(freeSpaceOff)
+ - (buf.getInt(tupleCountOff) * slotManager.getSlotSize()))
+ return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
+ else if (bytesRequired + slotManager.getSlotSize() <= buf.getInt(totalFreeSpaceOff))
+ return FrameOpSpaceStatus.SUFFICIENT_SPACE;
+ else
+ return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
+ }
- @Override
- public FrameOpSpaceStatus hasSpaceInsert(ITupleReference tuple,
- MultiComparator cmp) {
- int bytesRequired = tupleWriter.bytesRequired(tuple) + childPtrSize; // for
- // the
- // child
- // pointer
- if (bytesRequired + slotManager.getSlotSize() <= buf.capacity()
- - buf.getInt(freeSpaceOff)
- - (buf.getInt(tupleCountOff) * slotManager.getSlotSize()))
- return FrameOpSpaceStatus.SUFFICIENT_CONTIGUOUS_SPACE;
- else if (bytesRequired + slotManager.getSlotSize() <= buf
- .getInt(totalFreeSpaceOff))
- return FrameOpSpaceStatus.SUFFICIENT_SPACE;
- else
- return FrameOpSpaceStatus.INSUFFICIENT_SPACE;
- }
+ @Override
+ public void adjustKey(ITupleReference tuple, int tupleIndex, MultiComparator cmp) {
+ frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ if (tupleIndex == -1) {
+ tupleIndex = findTupleByPointer(tuple, cmp);
+ }
+ if (tupleIndex != -1) {
+ tupleWriter.writeTuple(tuple, buf.array(), getTupleOffset(tupleIndex));
+ }
+ }
- @Override
- public void adjustKey(ITupleReference tuple, int tupleIndex,
- MultiComparator cmp) {
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- if (tupleIndex == -1) {
- tupleIndex = findTupleByPointer(tuple, cmp);
- }
- if (tupleIndex != -1) {
- tupleWriter.writeTuple(tuple, buf, getTupleOffset(tupleIndex));
- }
- }
+ private int pointerCmp(ITupleReference tupleA, ITupleReference tupleB, MultiComparator cmp) {
+ return childPtrCmp
+ .compare(tupleA.getFieldData(cmp.getKeyFieldCount() - 1), getChildPointerOff(tupleA), childPtrSize,
+ tupleB.getFieldData(cmp.getKeyFieldCount() - 1), getChildPointerOff(tupleB), childPtrSize);
+ }
- private int pointerCmp(ITupleReference tupleA, ITupleReference tupleB,
- MultiComparator cmp) {
- return cmp.getIntCmp().compare(
- tupleA.getFieldData(cmp.getKeyFieldCount() - 1),
- getChildPointerOff(tupleA, cmp), childPtrSize,
- tupleB.getFieldData(cmp.getKeyFieldCount() - 1),
- getChildPointerOff(tupleB, cmp), childPtrSize);
- }
+ @Override
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException {
+ RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
+ RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter);
+ RTreeTypeAwareTupleWriter rTreeTupleWriterRightFrame = ((RTreeTypeAwareTupleWriter) rightFrame.getTupleWriter());
- @Override
- public int split(ITreeIndexFrame rightFrame, ITupleReference tuple,
- MultiComparator cmp, ISplitKey splitKey) throws Exception {
+ // calculations are based on the R*-tree paper
+ int m = (int) Math.floor((getTupleCount() + 1) * splitFactor);
+ int splitDistribution = getTupleCount() - (2 * m) + 2;
- rightFrame.setPageTupleFieldCount(cmp.getKeyFieldCount());
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ // to calculate the minimum margin in order to pick the split axis
+ double minMargin = Double.MAX_VALUE;
+ int splitAxis = 0, sortOrder = 0;
- RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
- RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter);
- RTreeTypeAwareTupleWriter rTreeTupleWriterRightFrame = ((RTreeTypeAwareTupleWriter) rightFrame
- .getTupleWriter());
+ int maxFieldPos = keyValueProviders.length / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ for (int k = 0; k < getTupleCount(); ++k) {
- // calculations are based on the R*-tree paper
- int m = (int) Math.floor((getTupleCount() + 1) * splitFactor);
- int splitDistribution = getTupleCount() - (2 * m) + 2;
+ frameTuple.resetByTupleIndex(this, k);
+ double LowerKey = keyValueProviders[i]
+ .getValue(frameTuple.getFieldData(i), frameTuple.getFieldStart(i));
+ double UpperKey = keyValueProviders[j]
+ .getValue(frameTuple.getFieldData(j), frameTuple.getFieldStart(j));
- // to calculate the minimum margin in order to pick the split axis
- double minMargin = Double.MAX_VALUE;
- int splitAxis = 0, sortOrder = 0;
+ tupleEntries1.add(k, LowerKey);
+ tupleEntries2.add(k, UpperKey);
+ }
+ double LowerKey = keyValueProviders[i].getValue(tuple.getFieldData(i), tuple.getFieldStart(i));
+ double UpperKey = keyValueProviders[j].getValue(tuple.getFieldData(j), tuple.getFieldStart(j));
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- for (int k = 0; k < getTupleCount(); ++k) {
+ tupleEntries1.add(-1, LowerKey);
+ tupleEntries2.add(-1, UpperKey);
- frameTuple.resetByTupleIndex(this, k);
- double LowerKey = cmp.getValueProviders()[i]
- .getValue(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i));
- double UpperKey = cmp.getValueProviders()[j]
- .getValue(frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j));
+ tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
+ tupleEntries2.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
- tupleEntries1.add(k, LowerKey);
- tupleEntries2.add(k, UpperKey);
- }
- double LowerKey = cmp.getValueProviders()[i].getValue(
- tuple.getFieldData(i), tuple.getFieldStart(i));
- double UpperKey = cmp.getValueProviders()[j].getValue(
- tuple.getFieldData(j), tuple.getFieldStart(j));
+ double lowerMargin = 0.0, upperMargin = 0.0;
+ // generate distribution
+ for (int k = 1; k <= splitDistribution; ++k) {
+ int d = m - 1 + k;
- tupleEntries1.add(-1, LowerKey);
- tupleEntries2.add(-1, UpperKey);
+ generateDist(tuple, tupleEntries1, rec[0], 0, d);
+ generateDist(tuple, tupleEntries2, rec[1], 0, d);
+ generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
+ generateDist(tuple, tupleEntries2, rec[3], d, getTupleCount() + 1);
- tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
- tupleEntries2.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
+ // calculate the margin of the distributions
+ lowerMargin += rec[0].margin() + rec[2].margin();
+ upperMargin += rec[1].margin() + rec[3].margin();
+ }
+ double margin = Math.min(lowerMargin, upperMargin);
- double lowerMargin = 0.0, upperMargin = 0.0;
- // generate distribution
- for (int k = 1; k <= splitDistribution; ++k) {
- int d = m - 1 + k;
+ // store minimum margin as split axis
+ if (margin < minMargin) {
+ minMargin = margin;
+ splitAxis = i;
+ sortOrder = (lowerMargin < upperMargin) ? 0 : 2;
+ }
- generateDist(tuple, tupleEntries1, rec[0], 0, d, cmp);
- generateDist(tuple, tupleEntries2, rec[1], 0, d, cmp);
- generateDist(tuple, tupleEntries1, rec[2], d,
- getTupleCount() + 1, cmp);
- generateDist(tuple, tupleEntries2, rec[3], d,
- getTupleCount() + 1, cmp);
-
- // calculate the margin of the distributions
- lowerMargin += rec[0].margin() + rec[2].margin();
- upperMargin += rec[1].margin() + rec[3].margin();
- }
- double margin = Math.min(lowerMargin, upperMargin);
-
- // store minimum margin as split axis
- if (margin < minMargin) {
- minMargin = margin;
- splitAxis = i;
- sortOrder = (lowerMargin < upperMargin) ? 0 : 2;
- }
-
- tupleEntries1.clear();
- tupleEntries2.clear();
- }
+ tupleEntries1.clear();
+ tupleEntries2.clear();
+ }
- for (int i = 0; i < getTupleCount(); ++i) {
- frameTuple.resetByTupleIndex(this, i);
- double key = cmp.getValueProviders()[splitAxis + sortOrder]
- .getValue(frameTuple.getFieldData(splitAxis + sortOrder),
- frameTuple.getFieldStart(splitAxis + sortOrder));
- tupleEntries1.add(i, key);
- }
- double key = cmp.getValueProviders()[splitAxis + sortOrder].getValue(
- tuple.getFieldData(splitAxis + sortOrder),
- tuple.getFieldStart(splitAxis + sortOrder));
- tupleEntries1.add(-1, key);
- tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
+ for (int i = 0; i < getTupleCount(); ++i) {
+ frameTuple.resetByTupleIndex(this, i);
+ double key = keyValueProviders[splitAxis + sortOrder].getValue(
+ frameTuple.getFieldData(splitAxis + sortOrder), frameTuple.getFieldStart(splitAxis + sortOrder));
+ tupleEntries1.add(i, key);
+ }
+ double key = keyValueProviders[splitAxis + sortOrder].getValue(tuple.getFieldData(splitAxis + sortOrder),
+ tuple.getFieldStart(splitAxis + sortOrder));
+ tupleEntries1.add(-1, key);
+ tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
- double minArea = Double.MAX_VALUE;
- double minOverlap = Double.MAX_VALUE;
- int splitPoint = 0;
- for (int i = 1; i <= splitDistribution; ++i) {
- int d = m - 1 + i;
+ double minArea = Double.MAX_VALUE;
+ double minOverlap = Double.MAX_VALUE;
+ int splitPoint = 0;
+ for (int i = 1; i <= splitDistribution; ++i) {
+ int d = m - 1 + i;
- generateDist(tuple, tupleEntries1, rec[0], 0, d, cmp);
- generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1,
- cmp);
+ generateDist(tuple, tupleEntries1, rec[0], 0, d);
+ generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
- double overlap = rec[0].overlappedArea(rec[2]);
- if (overlap < minOverlap) {
- splitPoint = d;
- minOverlap = overlap;
- minArea = rec[0].area() + rec[2].area();
- } else if (overlap == minOverlap) {
- double area = rec[0].area() + rec[2].area();
- if (area < minArea) {
- splitPoint = d;
- minArea = area;
- }
- }
- }
- int startIndex, endIndex;
- if (splitPoint < (getTupleCount() + 1) / 2) {
- startIndex = 0;
- endIndex = splitPoint;
- } else {
- startIndex = splitPoint;
- endIndex = (getTupleCount() + 1);
- }
- boolean tupleInserted = false;
- int totalBytes = 0, numOfDeletedTuples = 0;
- for (int i = startIndex; i < endIndex; i++) {
- if (tupleEntries1.get(i).getTupleIndex() != -1) {
- frameTuple.resetByTupleIndex(this, tupleEntries1.get(i)
- .getTupleIndex());
- rightFrame.insert(frameTuple, cmp, -1);
- ((UnorderedSlotManager) slotManager).modifySlot(slotManager
- .getSlotOff(tupleEntries1.get(i).getTupleIndex()), -1);
- totalBytes += tupleWriter.bytesRequired(frameTuple)
- + childPtrSize;
- numOfDeletedTuples++;
- } else {
- rightFrame.insert(tuple, cmp, -1);
- tupleInserted = true;
- }
- }
+ double overlap = rec[0].overlappedArea(rec[2]);
+ if (overlap < minOverlap) {
+ splitPoint = d;
+ minOverlap = overlap;
+ minArea = rec[0].area() + rec[2].area();
+ } else if (overlap == minOverlap) {
+ double area = rec[0].area() + rec[2].area();
+ if (area < minArea) {
+ splitPoint = d;
+ minArea = area;
+ }
+ }
+ }
+ int startIndex, endIndex;
+ if (splitPoint < (getTupleCount() + 1) / 2) {
+ startIndex = 0;
+ endIndex = splitPoint;
+ } else {
+ startIndex = splitPoint;
+ endIndex = (getTupleCount() + 1);
+ }
+ boolean tupleInserted = false;
+ int totalBytes = 0, numOfDeletedTuples = 0;
+ for (int i = startIndex; i < endIndex; i++) {
+ if (tupleEntries1.get(i).getTupleIndex() != -1) {
+ frameTuple.resetByTupleIndex(this, tupleEntries1.get(i).getTupleIndex());
+ rightFrame.insert(frameTuple, -1);
+ ((UnorderedSlotManager) slotManager).modifySlot(
+ slotManager.getSlotOff(tupleEntries1.get(i).getTupleIndex()), -1);
+ totalBytes += tupleWriter.bytesRequired(frameTuple) + childPtrSize;
+ numOfDeletedTuples++;
+ } else {
+ rightFrame.insert(tuple, -1);
+ tupleInserted = true;
+ }
+ }
- ((UnorderedSlotManager) slotManager).deleteEmptySlots();
+ ((UnorderedSlotManager) slotManager).deleteEmptySlots();
- // maintain space information
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff)
- + totalBytes + (slotManager.getSlotSize() * numOfDeletedTuples));
+ // maintain space information
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + totalBytes
+ + (slotManager.getSlotSize() * numOfDeletedTuples));
- // compact both pages
- rightFrame.compact(cmp);
- compact(cmp);
+ // compact both pages
+ rightFrame.compact();
+ compact();
- if (!tupleInserted) {
- insert(tuple, cmp, -1);
- }
+ if (!tupleInserted) {
+ insert(tuple, -1);
+ }
- int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
- frameTuple.resetByTupleOffset(buf, tupleOff);
- int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0,
- cmp.getKeyFieldCount());
+ int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, keyValueProviders.length);
- splitKey.initData(splitKeySize);
- this.adjustMBR(tuples, cmp);
- rTreeTupleWriterLeftFrame.writeTupleFields(tuples, 0,
- rTreeSplitKey.getLeftPageBuffer(), 0);
- rTreeSplitKey.getLeftTuple().resetByTupleOffset(
- rTreeSplitKey.getLeftPageBuffer(), 0);
+ splitKey.initData(splitKeySize);
+ this.adjustMBR(tuples);
+ rTreeTupleWriterLeftFrame.writeTupleFields(tuples, 0, rTreeSplitKey.getLeftPageBuffer(), 0);
+ rTreeSplitKey.getLeftTuple().resetByTupleOffset(rTreeSplitKey.getLeftPageBuffer(), 0);
- ((IRTreeFrame) rightFrame).adjustMBR(
- ((RTreeNSMFrame) rightFrame).getTuples(), cmp);
- rTreeTupleWriterRightFrame.writeTupleFields(
- ((RTreeNSMFrame) rightFrame).getTuples(), 0,
- rTreeSplitKey.getRightPageBuffer(), 0);
- rTreeSplitKey.getRightTuple().resetByTupleOffset(
- rTreeSplitKey.getRightPageBuffer(), 0);
+ ((IRTreeFrame) rightFrame).adjustMBR(((RTreeNSMFrame) rightFrame).getTuples());
+ rTreeTupleWriterRightFrame.writeTupleFields(((RTreeNSMFrame) rightFrame).getTuples(), 0,
+ rTreeSplitKey.getRightPageBuffer(), 0);
+ rTreeSplitKey.getRightTuple().resetByTupleOffset(rTreeSplitKey.getRightPageBuffer(), 0);
- tupleEntries1.clear();
- tupleEntries2.clear();
- return 0;
- }
+ tupleEntries1.clear();
+ tupleEntries2.clear();
+ }
- private int getChildPointerOff(ITupleReference tuple, MultiComparator cmp) {
- return tuple.getFieldStart(cmp.getKeyFieldCount() - 1)
- + tuple.getFieldLength(cmp.getKeyFieldCount() - 1);
- }
+ private int getChildPointerOff(ITupleReference tuple) {
+ return tuple.getFieldStart(tuple.getFieldCount() - 1) + tuple.getFieldLength(tuple.getFieldCount() - 1);
+ }
- @Override
- public void insert(ITupleReference tuple, MultiComparator cmp,
- int tupleIndex) throws Exception {
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- slotManager.insertSlot(-1, buf.getInt(freeSpaceOff));
- int freeSpace = buf.getInt(freeSpaceOff);
- int bytesWritten = tupleWriter.writeTupleFields(tuple, 0,
- cmp.getKeyFieldCount(), buf, freeSpace);
- System.arraycopy(tuple.getFieldData(cmp.getKeyFieldCount() - 1),
- getChildPointerOff(tuple, cmp), buf.array(), freeSpace
- + bytesWritten, childPtrSize);
- int tupleSize = bytesWritten + childPtrSize;
+ @Override
+ public void insert(ITupleReference tuple, int tupleIndex) {
+ frameTuple.setFieldCount(tuple.getFieldCount());
+ slotManager.insertSlot(-1, buf.getInt(freeSpaceOff));
+ int freeSpace = buf.getInt(freeSpaceOff);
+ int bytesWritten = tupleWriter.writeTupleFields(tuple, 0, tuple.getFieldCount(), buf, freeSpace);
+ System.arraycopy(tuple.getFieldData(tuple.getFieldCount() - 1), getChildPointerOff(tuple), buf.array(),
+ freeSpace + bytesWritten, childPtrSize);
+ int tupleSize = bytesWritten + childPtrSize;
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
- buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + tupleSize);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - tupleSize
- - slotManager.getSlotSize());
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
+ buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + tupleSize);
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - tupleSize - slotManager.getSlotSize());
- }
+ }
- @Override
- public void delete(int tupleIndex, MultiComparator cmp) throws Exception {
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- int slotOff = slotManager.getSlotOff(tupleIndex);
+ @Override
+ public void delete(int tupleIndex, MultiComparator cmp) {
+ frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ int slotOff = slotManager.getSlotOff(tupleIndex);
- int tupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.resetByTupleOffset(buf, tupleOff);
- int tupleSize = tupleWriter.bytesRequired(frameTuple);
+ int tupleOff = slotManager.getTupleOff(slotOff);
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ int tupleSize = tupleWriter.bytesRequired(frameTuple);
- // perform deletion (we just do a memcpy to overwrite the slot)
- int slotStartOff = slotManager.getSlotEndOff();
- int length = slotOff - slotStartOff;
- System.arraycopy(buf.array(), slotStartOff, buf.array(), slotStartOff
- + slotManager.getSlotSize(), length);
+ // perform deletion (we just do a memcpy to overwrite the slot)
+ int slotStartOff = slotManager.getSlotEndOff();
+ int length = slotOff - slotStartOff;
+ System.arraycopy(buf.array(), slotStartOff, buf.array(), slotStartOff + slotManager.getSlotSize(), length);
- // maintain space information
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + tupleSize
- + childPtrSize + slotManager.getSlotSize());
- }
+ // maintain space information
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
+ buf.putInt(totalFreeSpaceOff,
+ buf.getInt(totalFreeSpaceOff) + tupleSize + childPtrSize + slotManager.getSlotSize());
+ }
- @Override
- public boolean recomputeMBR(ITupleReference tuple, int tupleIndex,
- MultiComparator cmp) {
- frameTuple.setFieldCount(cmp.getKeyFieldCount());
- frameTuple.resetByTupleIndex(this, tupleIndex);
+ @Override
+ public boolean recomputeMBR(ITupleReference tuple, int tupleIndex, MultiComparator cmp) {
+ frameTuple.setFieldCount(cmp.getKeyFieldCount());
+ frameTuple.resetByTupleIndex(this, tupleIndex);
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), frameTuple.getFieldLength(i),
- tuple.getFieldData(i), tuple.getFieldStart(i),
- tuple.getFieldLength(i));
- if (c != 0) {
- return true;
- }
- c = cmp.getComparators()[j].compare(frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j), frameTuple.getFieldLength(j),
- tuple.getFieldData(j), tuple.getFieldStart(j),
- tuple.getFieldLength(j));
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(frameTuple.getFieldData(i), frameTuple.getFieldStart(i),
+ frameTuple.getFieldLength(i), tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ if (c != 0) {
+ return true;
+ }
+ c = cmp.getComparators()[j].compare(frameTuple.getFieldData(j), frameTuple.getFieldStart(j),
+ frameTuple.getFieldLength(j), tuple.getFieldData(j), tuple.getFieldStart(j),
+ tuple.getFieldLength(j));
- if (c != 0) {
- return true;
- }
- }
- return false;
- }
+ if (c != 0) {
+ return true;
+ }
+ }
+ return false;
+ }
- private double overlappedArea(ITupleReference tuple1,
- ITupleReference tupleToBeInserted, ITupleReference tuple2,
- MultiComparator cmp) {
- double area = 1.0;
- double f1, f2;
+ private double overlappedArea(ITupleReference tuple1, ITupleReference tupleToBeInserted, ITupleReference tuple2,
+ MultiComparator cmp) {
+ double area = 1.0;
+ double f1, f2;
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- double pHigh1, pLow1;
- if (tupleToBeInserted != null) {
- int c = cmp.getComparators()[i].compare(tuple1.getFieldData(i),
- tuple1.getFieldStart(i), tuple1.getFieldLength(i),
- tupleToBeInserted.getFieldData(i),
- tupleToBeInserted.getFieldStart(i),
- tupleToBeInserted.getFieldLength(i));
- if (c < 0) {
- pLow1 = cmp.getValueProviders()[i].getValue(
- tuple1.getFieldData(i), tuple1.getFieldStart(i));
- } else {
- pLow1 = cmp.getValueProviders()[i].getValue(
- tupleToBeInserted.getFieldData(i),
- tupleToBeInserted.getFieldStart(i));
- }
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ double pHigh1, pLow1;
+ if (tupleToBeInserted != null) {
+ int c = cmp.getComparators()[i].compare(tuple1.getFieldData(i), tuple1.getFieldStart(i),
+ tuple1.getFieldLength(i), tupleToBeInserted.getFieldData(i),
+ tupleToBeInserted.getFieldStart(i), tupleToBeInserted.getFieldLength(i));
+ if (c < 0) {
+ pLow1 = keyValueProviders[i].getValue(tuple1.getFieldData(i), tuple1.getFieldStart(i));
+ } else {
+ pLow1 = keyValueProviders[i].getValue(tupleToBeInserted.getFieldData(i),
+ tupleToBeInserted.getFieldStart(i));
+ }
- c = cmp.getComparators()[j].compare(tuple1.getFieldData(j),
- tuple1.getFieldStart(j), tuple1.getFieldLength(j),
- tupleToBeInserted.getFieldData(j),
- tupleToBeInserted.getFieldStart(j),
- tupleToBeInserted.getFieldLength(j));
- if (c > 0) {
- pHigh1 = cmp.getValueProviders()[j].getValue(
- tuple1.getFieldData(j), tuple1.getFieldStart(j));
- } else {
- pHigh1 = cmp.getValueProviders()[j].getValue(
- tupleToBeInserted.getFieldData(j),
- tupleToBeInserted.getFieldStart(j));
- }
- } else {
- pLow1 = cmp.getValueProviders()[i].getValue(
- tuple1.getFieldData(i), tuple1.getFieldStart(i));
- pHigh1 = cmp.getValueProviders()[j].getValue(
- tuple1.getFieldData(j), tuple1.getFieldStart(j));
- }
+ c = cmp.getComparators()[j].compare(tuple1.getFieldData(j), tuple1.getFieldStart(j),
+ tuple1.getFieldLength(j), tupleToBeInserted.getFieldData(j),
+ tupleToBeInserted.getFieldStart(j), tupleToBeInserted.getFieldLength(j));
+ if (c > 0) {
+ pHigh1 = keyValueProviders[j].getValue(tuple1.getFieldData(j), tuple1.getFieldStart(j));
+ } else {
+ pHigh1 = keyValueProviders[j].getValue(tupleToBeInserted.getFieldData(j),
+ tupleToBeInserted.getFieldStart(j));
+ }
+ } else {
+ pLow1 = keyValueProviders[i].getValue(tuple1.getFieldData(i), tuple1.getFieldStart(i));
+ pHigh1 = keyValueProviders[j].getValue(tuple1.getFieldData(j), tuple1.getFieldStart(j));
+ }
- double pLow2 = cmp.getValueProviders()[i].getValue(
- tuple2.getFieldData(i), tuple2.getFieldStart(i));
- double pHigh2 = cmp.getValueProviders()[j].getValue(
- tuple2.getFieldData(j), tuple2.getFieldStart(j));
+ double pLow2 = keyValueProviders[i].getValue(tuple2.getFieldData(i), tuple2.getFieldStart(i));
+ double pHigh2 = keyValueProviders[j].getValue(tuple2.getFieldData(j), tuple2.getFieldStart(j));
- if (pLow1 > pHigh2 || pHigh1 < pLow2) {
- return 0.0;
- }
+ if (pLow1 > pHigh2 || pHigh1 < pLow2) {
+ return 0.0;
+ }
- f1 = Math.max(pLow1, pLow2);
- f2 = Math.min(pHigh1, pHigh2);
- area *= f2 - f1;
- }
- return area;
- }
+ f1 = Math.max(pLow1, pLow2);
+ f2 = Math.min(pHigh1, pHigh2);
+ area *= f2 - f1;
+ }
+ return area;
+ }
- private double enlargedArea(ITupleReference tuple,
- ITupleReference tupleToBeInserted, MultiComparator cmp) {
- double areaBeforeEnlarge = area(tuple, cmp);
- double areaAfterEnlarge = 1.0;
+ private double enlargedArea(ITupleReference tuple, ITupleReference tupleToBeInserted, MultiComparator cmp) {
+ double areaBeforeEnlarge = area(tuple, cmp);
+ double areaAfterEnlarge = 1.0;
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- double pHigh, pLow;
- int c = cmp.getComparators()[i].compare(tuple.getFieldData(i),
- tuple.getFieldStart(i), tuple.getFieldLength(i),
- tupleToBeInserted.getFieldData(i),
- tupleToBeInserted.getFieldStart(i),
- tupleToBeInserted.getFieldLength(i));
- if (c < 0) {
- pLow = cmp.getValueProviders()[i].getValue(
- tuple.getFieldData(i), tuple.getFieldStart(i));
- } else {
- pLow = cmp.getValueProviders()[i].getValue(
- tupleToBeInserted.getFieldData(i),
- tupleToBeInserted.getFieldStart(i));
- }
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ double pHigh, pLow;
+ int c = cmp.getComparators()[i].compare(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i), tupleToBeInserted.getFieldData(i), tupleToBeInserted.getFieldStart(i),
+ tupleToBeInserted.getFieldLength(i));
+ if (c < 0) {
+ pLow = keyValueProviders[i].getValue(tuple.getFieldData(i), tuple.getFieldStart(i));
+ } else {
+ pLow = keyValueProviders[i].getValue(tupleToBeInserted.getFieldData(i),
+ tupleToBeInserted.getFieldStart(i));
+ }
- c = cmp.getComparators()[j].compare(tuple.getFieldData(j),
- tuple.getFieldStart(j), tuple.getFieldLength(j),
- tupleToBeInserted.getFieldData(j),
- tupleToBeInserted.getFieldStart(j),
- tupleToBeInserted.getFieldLength(j));
- if (c > 0) {
- pHigh = cmp.getValueProviders()[j].getValue(
- tuple.getFieldData(j), tuple.getFieldStart(j));
- } else {
- pHigh = cmp.getValueProviders()[j].getValue(
- tupleToBeInserted.getFieldData(j),
- tupleToBeInserted.getFieldStart(j));
- }
- areaAfterEnlarge *= pHigh - pLow;
- }
- return areaAfterEnlarge - areaBeforeEnlarge;
- }
+ c = cmp.getComparators()[j].compare(tuple.getFieldData(j), tuple.getFieldStart(j), tuple.getFieldLength(j),
+ tupleToBeInserted.getFieldData(j), tupleToBeInserted.getFieldStart(j),
+ tupleToBeInserted.getFieldLength(j));
+ if (c > 0) {
+ pHigh = keyValueProviders[j].getValue(tuple.getFieldData(j), tuple.getFieldStart(j));
+ } else {
+ pHigh = keyValueProviders[j].getValue(tupleToBeInserted.getFieldData(j),
+ tupleToBeInserted.getFieldStart(j));
+ }
+ areaAfterEnlarge *= pHigh - pLow;
+ }
+ return areaAfterEnlarge - areaBeforeEnlarge;
+ }
- private double area(ITupleReference tuple, MultiComparator cmp) {
- double area = 1.0;
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- area *= cmp.getValueProviders()[j].getValue(tuple.getFieldData(j),
- tuple.getFieldStart(j))
- - cmp.getValueProviders()[i].getValue(
- tuple.getFieldData(i), tuple.getFieldStart(i));
- }
- return area;
- }
+ private double area(ITupleReference tuple, MultiComparator cmp) {
+ double area = 1.0;
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ area *= keyValueProviders[j].getValue(tuple.getFieldData(j), tuple.getFieldStart(j))
+ - keyValueProviders[i].getValue(tuple.getFieldData(i), tuple.getFieldStart(i));
+ }
+ return area;
+ }
- @Override
- public void enlarge(ITupleReference tuple, MultiComparator cmp) {
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), frameTuple.getFieldLength(i),
- tuple.getFieldData(i), tuple.getFieldStart(i),
- tuple.getFieldLength(i));
- if (c > 0) {
- System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i),
- frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i), tuple.getFieldLength(i));
- }
- c = cmp.getComparators()[j].compare(frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j), frameTuple.getFieldLength(j),
- tuple.getFieldData(j), tuple.getFieldStart(j),
- tuple.getFieldLength(j));
- if (c < 0) {
- System.arraycopy(tuple.getFieldData(j), tuple.getFieldStart(j),
- frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j), tuple.getFieldLength(j));
- }
- }
- }
+ @Override
+ public void enlarge(ITupleReference tuple, MultiComparator cmp) {
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(frameTuple.getFieldData(i), frameTuple.getFieldStart(i),
+ frameTuple.getFieldLength(i), tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ if (c > 0) {
+ System.arraycopy(tuple.getFieldData(i), tuple.getFieldStart(i), frameTuple.getFieldData(i),
+ frameTuple.getFieldStart(i), tuple.getFieldLength(i));
+ }
+ c = cmp.getComparators()[j].compare(frameTuple.getFieldData(j), frameTuple.getFieldStart(j),
+ frameTuple.getFieldLength(j), tuple.getFieldData(j), tuple.getFieldStart(j),
+ tuple.getFieldLength(j));
+ if (c < 0) {
+ System.arraycopy(tuple.getFieldData(j), tuple.getFieldStart(j), frameTuple.getFieldData(j),
+ frameTuple.getFieldStart(j), tuple.getFieldLength(j));
+ }
+ }
+ }
- @Override
- public void adjustMBR(ITreeIndexTupleReference[] tuples, MultiComparator cmp) {
- for (int i = 0; i < tuples.length; i++) {
- tuples[i].setFieldCount(cmp.getKeyFieldCount());
- tuples[i].resetByTupleIndex(this, 0);
- }
+ @Override
+ public void adjustMBR(ITreeIndexTupleReference[] tuples) {
+ for (int i = 0; i < tuples.length; i++) {
+ tuples[i].setFieldCount(keyValueProviders.length);
+ tuples[i].resetByTupleIndex(this, 0);
+ }
- adjustMBRImpl(tuples, cmp);
- }
-}
\ No newline at end of file
+ adjustMBRImpl(tuples);
+ }
+}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
index 100b4d8..943a179 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMInteriorFrameFactory.java
@@ -15,29 +15,37 @@
package edu.uci.ics.hyracks.storage.am.rtree.frames;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
public class RTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory {
- private static final long serialVersionUID = 1L;
- private ITreeIndexTupleWriterFactory tupleWriterFactory;
- private int keyFieldCount;
+ private static final long serialVersionUID = 1L;
+ private final ITreeIndexTupleWriterFactory tupleWriterFactory;
+ private final IPrimitiveValueProviderFactory[] keyValueProviderFactories;
- public RTreeNSMInteriorFrameFactory(
- ITreeIndexTupleWriterFactory tupleWriterFactory, int keyFieldCount) {
- this.tupleWriterFactory = tupleWriterFactory;
- if (keyFieldCount % 2 != 0) {
- throw new IllegalArgumentException(
- "The key has different number of dimensions.");
- }
- this.keyFieldCount = keyFieldCount;
- }
+ public RTreeNSMInteriorFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory, IPrimitiveValueProviderFactory[] keyValueProviderFactories) {
+ this.tupleWriterFactory = tupleWriterFactory;
+ if (keyValueProviderFactories.length % 2 != 0) {
+ throw new IllegalArgumentException("The key has different number of dimensions.");
+ }
+ this.keyValueProviderFactories = keyValueProviderFactories;
+ }
+
+ @Override
+ public IRTreeInteriorFrame createFrame() {
+ IPrimitiveValueProvider[] keyValueProviders = new IPrimitiveValueProvider[keyValueProviderFactories.length];
+ for (int i = 0; i < keyValueProviders.length; i++) {
+ keyValueProviders[i] = keyValueProviderFactories[i].createPrimitiveValueProvider();
+ }
+ return new RTreeNSMInteriorFrame(tupleWriterFactory.createTupleWriter(), keyValueProviders);
+ }
@Override
- public IRTreeInteriorFrame createFrame() {
- return new RTreeNSMInteriorFrame(
- tupleWriterFactory.createTupleWriter(), keyFieldCount);
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return tupleWriterFactory;
}
-}
\ No newline at end of file
+}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
index fa2ad32..b2ae5fc 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrame.java
@@ -16,10 +16,12 @@
package edu.uci.ics.hyracks.storage.am.rtree.frames;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
import edu.uci.ics.hyracks.storage.am.common.api.ISplitKey;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
@@ -30,263 +32,225 @@
public class RTreeNSMLeafFrame extends RTreeNSMFrame implements IRTreeLeafFrame {
- public RTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter,
- int keyFieldCount) {
- super(tupleWriter, keyFieldCount);
- }
+ public RTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter, IPrimitiveValueProvider[] keyValueProviders) {
+ super(tupleWriter, keyValueProviders);
+ }
- @Override
- public int findTupleIndex(ITupleReference tuple, MultiComparator cmp) {
- frameTuple.setFieldCount(cmp.getFieldCount());
- return slotManager.findTupleIndex(tuple, frameTuple, cmp, null, null);
- }
+ @Override
+ public int findTupleIndex(ITupleReference tuple, MultiComparator cmp) {
+ return slotManager.findTupleIndex(tuple, frameTuple, cmp, null, null);
+ }
- @Override
- public boolean intersect(ITupleReference tuple, int tupleIndex,
- MultiComparator cmp) {
- frameTuple.setFieldCount(cmp.getFieldCount());
- frameTuple.resetByTupleIndex(this, tupleIndex);
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(tuple.getFieldData(i),
- tuple.getFieldStart(i), tuple.getFieldLength(i),
- frameTuple.getFieldData(j), frameTuple.getFieldStart(j),
- frameTuple.getFieldLength(j));
- if (c > 0) {
- return false;
- }
- c = cmp.getComparators()[i].compare(tuple.getFieldData(j),
- tuple.getFieldStart(j), tuple.getFieldLength(j),
- frameTuple.getFieldData(i), frameTuple.getFieldStart(i),
- frameTuple.getFieldLength(i));
+ @Override
+ public boolean intersect(ITupleReference tuple, int tupleIndex, MultiComparator cmp) {
+ frameTuple.resetByTupleIndex(this, tupleIndex);
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i), frameTuple.getFieldData(j), frameTuple.getFieldStart(j),
+ frameTuple.getFieldLength(j));
+ if (c > 0) {
+ return false;
+ }
+ c = cmp.getComparators()[i].compare(tuple.getFieldData(j), tuple.getFieldStart(j), tuple.getFieldLength(j),
+ frameTuple.getFieldData(i), frameTuple.getFieldStart(i), frameTuple.getFieldLength(i));
- if (c < 0) {
- return false;
- }
- }
- return true;
- }
+ if (c < 0) {
+ return false;
+ }
+ }
+ return true;
+ }
- @Override
- public int split(ITreeIndexFrame rightFrame, ITupleReference tuple,
- MultiComparator cmp, ISplitKey splitKey) throws Exception {
+ @Override
+ public void split(ITreeIndexFrame rightFrame, ITupleReference tuple, ISplitKey splitKey) throws TreeIndexException {
- rightFrame.setPageTupleFieldCount(cmp.getFieldCount());
- frameTuple.setFieldCount(cmp.getFieldCount());
+ RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
+ RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter);
+ RTreeTypeAwareTupleWriter rTreeTupleWriterRightFrame = ((RTreeTypeAwareTupleWriter) rightFrame.getTupleWriter());
- RTreeSplitKey rTreeSplitKey = ((RTreeSplitKey) splitKey);
- RTreeTypeAwareTupleWriter rTreeTupleWriterLeftFrame = ((RTreeTypeAwareTupleWriter) tupleWriter);
- RTreeTypeAwareTupleWriter rTreeTupleWriterRightFrame = ((RTreeTypeAwareTupleWriter) rightFrame
- .getTupleWriter());
+ // calculations are based on the R*-tree paper
+ int m = (int) Math.floor((getTupleCount() + 1) * splitFactor);
+ int splitDistribution = getTupleCount() - (2 * m) + 2;
- // calculations are based on the R*-tree paper
- int m = (int) Math.floor((getTupleCount() + 1) * splitFactor);
- int splitDistribution = getTupleCount() - (2 * m) + 2;
+ // to calculate the minimum margin in order to pick the split axis
+ double minMargin = Double.MAX_VALUE;
+ int splitAxis = 0, sortOrder = 0;
- // to calculate the minimum margin in order to pick the split axis
- double minMargin = Double.MAX_VALUE;
- int splitAxis = 0, sortOrder = 0;
+ int maxFieldPos = keyValueProviders.length / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ for (int k = 0; k < getTupleCount(); ++k) {
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- for (int k = 0; k < getTupleCount(); ++k) {
+ frameTuple.resetByTupleIndex(this, k);
- frameTuple.resetByTupleIndex(this, k);
+ double LowerKey = keyValueProviders[i].getValue(frameTuple.getFieldData(i),
+ frameTuple.getFieldStart(i));
+ double UpperKey = keyValueProviders[j].getValue(frameTuple.getFieldData(j),
+ frameTuple.getFieldStart(j));
- double LowerKey = cmp.getValueProviders()[i]
- .getValue(frameTuple.getFieldData(i),
- frameTuple.getFieldStart(i));
- double UpperKey = cmp.getValueProviders()[j]
- .getValue(frameTuple.getFieldData(j),
- frameTuple.getFieldStart(j));
+ tupleEntries1.add(k, LowerKey);
+ tupleEntries2.add(k, UpperKey);
+ }
+ double LowerKey = keyValueProviders[i].getValue(tuple.getFieldData(i), tuple.getFieldStart(i));
+ double UpperKey = keyValueProviders[j].getValue(tuple.getFieldData(j), tuple.getFieldStart(j));
- tupleEntries1.add(k, LowerKey);
- tupleEntries2.add(k, UpperKey);
- }
- double LowerKey = cmp.getValueProviders()[i].getValue(
- tuple.getFieldData(i), tuple.getFieldStart(i));
- double UpperKey = cmp.getValueProviders()[j].getValue(
- tuple.getFieldData(j), tuple.getFieldStart(j));
+ tupleEntries1.add(-1, LowerKey);
+ tupleEntries2.add(-1, UpperKey);
- tupleEntries1.add(-1, LowerKey);
- tupleEntries2.add(-1, UpperKey);
+ tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
+ tupleEntries2.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
- tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
- tupleEntries2.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
+ double lowerMargin = 0.0, upperMargin = 0.0;
+ // generate distribution
+ for (int k = 1; k <= splitDistribution; ++k) {
+ int d = m - 1 + k;
- double lowerMargin = 0.0, upperMargin = 0.0;
- // generate distribution
- for (int k = 1; k <= splitDistribution; ++k) {
- int d = m - 1 + k;
+ generateDist(tuple, tupleEntries1, rec[0], 0, d);
+ generateDist(tuple, tupleEntries2, rec[1], 0, d);
+ generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
+ generateDist(tuple, tupleEntries2, rec[3], d, getTupleCount() + 1);
- generateDist(tuple, tupleEntries1, rec[0], 0, d, cmp);
- generateDist(tuple, tupleEntries2, rec[1], 0, d, cmp);
- generateDist(tuple, tupleEntries1, rec[2], d,
- getTupleCount() + 1, cmp);
- generateDist(tuple, tupleEntries2, rec[3], d,
- getTupleCount() + 1, cmp);
+ // calculate the margin of the distributions
+ lowerMargin += rec[0].margin() + rec[2].margin();
+ upperMargin += rec[1].margin() + rec[3].margin();
+ }
+ double margin = Math.min(lowerMargin, upperMargin);
- // calculate the margin of the distributions
- lowerMargin += rec[0].margin() + rec[2].margin();
- upperMargin += rec[1].margin() + rec[3].margin();
- }
- double margin = Math.min(lowerMargin, upperMargin);
+ // store minimum margin as split axis
+ if (margin < minMargin) {
+ minMargin = margin;
+ splitAxis = i;
+ sortOrder = (lowerMargin < upperMargin) ? 0 : 2;
+ }
- // store minimum margin as split axis
- if (margin < minMargin) {
- minMargin = margin;
- splitAxis = i;
- sortOrder = (lowerMargin < upperMargin) ? 0 : 2;
- }
+ tupleEntries1.clear();
+ tupleEntries2.clear();
+ }
- tupleEntries1.clear();
- tupleEntries2.clear();
- }
+ for (int i = 0; i < getTupleCount(); ++i) {
+ frameTuple.resetByTupleIndex(this, i);
+ double key = keyValueProviders[splitAxis + sortOrder].getValue(
+ frameTuple.getFieldData(splitAxis + sortOrder), frameTuple.getFieldStart(splitAxis + sortOrder));
+ tupleEntries1.add(i, key);
+ }
+ double key = keyValueProviders[splitAxis + sortOrder].getValue(tuple.getFieldData(splitAxis + sortOrder),
+ tuple.getFieldStart(splitAxis + sortOrder));
+ tupleEntries1.add(-1, key);
+ tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
- for (int i = 0; i < getTupleCount(); ++i) {
- frameTuple.resetByTupleIndex(this, i);
- double key = cmp.getValueProviders()[splitAxis + sortOrder]
- .getValue(frameTuple.getFieldData(splitAxis + sortOrder),
- frameTuple.getFieldStart(splitAxis + sortOrder));
- tupleEntries1.add(i, key);
- }
- double key = cmp.getValueProviders()[splitAxis + sortOrder].getValue(
- tuple.getFieldData(splitAxis + sortOrder),
- tuple.getFieldStart(splitAxis + sortOrder));
- tupleEntries1.add(-1, key);
- tupleEntries1.sort(EntriesOrder.ASCENDING, getTupleCount() + 1);
+ double minArea = Double.MAX_VALUE;
+ double minOverlap = Double.MAX_VALUE;
+ int splitPoint = 0;
+ for (int i = 1; i <= splitDistribution; ++i) {
+ int d = m - 1 + i;
- double minArea = Double.MAX_VALUE;
- double minOverlap = Double.MAX_VALUE;
- int splitPoint = 0;
- for (int i = 1; i <= splitDistribution; ++i) {
- int d = m - 1 + i;
+ generateDist(tuple, tupleEntries1, rec[0], 0, d);
+ generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1);
- generateDist(tuple, tupleEntries1, rec[0], 0, d, cmp);
- generateDist(tuple, tupleEntries1, rec[2], d, getTupleCount() + 1,
- cmp);
+ double overlap = rec[0].overlappedArea(rec[2]);
+ if (overlap < minOverlap) {
+ splitPoint = d;
+ minOverlap = overlap;
+ minArea = rec[0].area() + rec[2].area();
+ } else if (overlap == minOverlap) {
+ double area = rec[0].area() + rec[2].area();
+ if (area < minArea) {
+ splitPoint = d;
+ minArea = area;
+ }
+ }
+ }
+ int startIndex, endIndex;
+ if (splitPoint < (getTupleCount() + 1) / 2) {
+ startIndex = 0;
+ endIndex = splitPoint;
+ } else {
+ startIndex = splitPoint;
+ endIndex = (getTupleCount() + 1);
+ }
+ boolean tupleInserted = false;
+ int totalBytes = 0, numOfDeletedTuples = 0;
+ for (int i = startIndex; i < endIndex; i++) {
+ if (tupleEntries1.get(i).getTupleIndex() != -1) {
+ frameTuple.resetByTupleIndex(this, tupleEntries1.get(i).getTupleIndex());
+ rightFrame.insert(frameTuple, -1);
+ ((UnorderedSlotManager) slotManager).modifySlot(
+ slotManager.getSlotOff(tupleEntries1.get(i).getTupleIndex()), -1);
+ totalBytes += tupleWriter.bytesRequired(frameTuple);
+ numOfDeletedTuples++;
+ } else {
+ rightFrame.insert(tuple, -1);
+ tupleInserted = true;
+ }
+ }
- double overlap = rec[0].overlappedArea(rec[2]);
- if (overlap < minOverlap) {
- splitPoint = d;
- minOverlap = overlap;
- minArea = rec[0].area() + rec[2].area();
- } else if (overlap == minOverlap) {
- double area = rec[0].area() + rec[2].area();
- if (area < minArea) {
- splitPoint = d;
- minArea = area;
- }
- }
- }
- int startIndex, endIndex;
- if (splitPoint < (getTupleCount() + 1) / 2) {
- startIndex = 0;
- endIndex = splitPoint;
- } else {
- startIndex = splitPoint;
- endIndex = (getTupleCount() + 1);
- }
- boolean tupleInserted = false;
- int totalBytes = 0, numOfDeletedTuples = 0;
- for (int i = startIndex; i < endIndex; i++) {
- if (tupleEntries1.get(i).getTupleIndex() != -1) {
- frameTuple.resetByTupleIndex(this, tupleEntries1.get(i)
- .getTupleIndex());
- rightFrame.insert(frameTuple, cmp, -1);
- ((UnorderedSlotManager) slotManager).modifySlot(slotManager
- .getSlotOff(tupleEntries1.get(i).getTupleIndex()), -1);
- totalBytes += tupleWriter.bytesRequired(frameTuple);
- numOfDeletedTuples++;
- } else {
- rightFrame.insert(tuple, cmp, -1);
- tupleInserted = true;
- }
- }
+ ((UnorderedSlotManager) slotManager).deleteEmptySlots();
- ((UnorderedSlotManager) slotManager).deleteEmptySlots();
+ // maintain space information
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + totalBytes
+ + (slotManager.getSlotSize() * numOfDeletedTuples));
- // maintain space information
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff)
- + totalBytes + (slotManager.getSlotSize() * numOfDeletedTuples));
+ // compact both pages
+ rightFrame.compact();
+ compact();
- // compact both pages
- rightFrame.compact(cmp);
- compact(cmp);
+ if (!tupleInserted) {
+ insert(tuple, -1);
+ }
- if (!tupleInserted) {
- insert(tuple, cmp, -1);
- }
+ int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0, keyValueProviders.length);
- int tupleOff = slotManager.getTupleOff(slotManager.getSlotEndOff());
- frameTuple.resetByTupleOffset(buf, tupleOff);
- int splitKeySize = tupleWriter.bytesRequired(frameTuple, 0,
- cmp.getKeyFieldCount());
+ splitKey.initData(splitKeySize);
+ this.adjustMBR(tuples);
+ rTreeTupleWriterLeftFrame.writeTupleFields(tuples, 0, rTreeSplitKey.getLeftPageBuffer(), 0);
+ rTreeSplitKey.getLeftTuple().resetByTupleOffset(rTreeSplitKey.getLeftPageBuffer(), 0);
- splitKey.initData(splitKeySize);
- this.adjustMBR(tuples, cmp);
- rTreeTupleWriterLeftFrame.writeTupleFields(tuples, 0,
- rTreeSplitKey.getLeftPageBuffer(), 0);
- rTreeSplitKey.getLeftTuple().resetByTupleOffset(
- rTreeSplitKey.getLeftPageBuffer(), 0);
+ ((IRTreeFrame) rightFrame).adjustMBR(((RTreeNSMFrame) rightFrame).getTuples());
+ rTreeTupleWriterRightFrame.writeTupleFields(((RTreeNSMFrame) rightFrame).getTuples(), 0,
+ rTreeSplitKey.getRightPageBuffer(), 0);
+ rTreeSplitKey.getRightTuple().resetByTupleOffset(rTreeSplitKey.getRightPageBuffer(), 0);
- ((IRTreeFrame) rightFrame).adjustMBR(
- ((RTreeNSMFrame) rightFrame).getTuples(), cmp);
- rTreeTupleWriterRightFrame.writeTupleFields(
- ((RTreeNSMFrame) rightFrame).getTuples(), 0,
- rTreeSplitKey.getRightPageBuffer(), 0);
- rTreeSplitKey.getRightTuple().resetByTupleOffset(
- rTreeSplitKey.getRightPageBuffer(), 0);
+ tupleEntries1.clear();
+ tupleEntries2.clear();
+ }
- tupleEntries1.clear();
- tupleEntries2.clear();
- return 0;
- }
+ @Override
+ public void insert(ITupleReference tuple, int tupleIndex) {
+ slotManager.insertSlot(-1, buf.getInt(freeSpaceOff));
+ int bytesWritten = tupleWriter.writeTuple(tuple, buf.array(), buf.getInt(freeSpaceOff));
- @Override
- public void insert(ITupleReference tuple, MultiComparator cmp,
- int tupleIndex) throws Exception {
- frameTuple.setFieldCount(cmp.getFieldCount());
- slotManager.insertSlot(-1, buf.getInt(freeSpaceOff));
- int bytesWritten = tupleWriter.writeTuple(tuple, buf,
- buf.getInt(freeSpaceOff));
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
+ buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) - bytesWritten - slotManager.getSlotSize());
+ }
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) + 1);
- buf.putInt(freeSpaceOff, buf.getInt(freeSpaceOff) + bytesWritten);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff)
- - bytesWritten - slotManager.getSlotSize());
- }
+ @Override
+ public void delete(int tupleIndex, MultiComparator cmp) {
+ int slotOff = slotManager.getSlotOff(tupleIndex);
- @Override
- public void delete(int tupleIndex, MultiComparator cmp) throws Exception {
- frameTuple.setFieldCount(cmp.getFieldCount());
- int slotOff = slotManager.getSlotOff(tupleIndex);
+ int tupleOff = slotManager.getTupleOff(slotOff);
+ frameTuple.resetByTupleOffset(buf, tupleOff);
+ int tupleSize = tupleWriter.bytesRequired(frameTuple);
- int tupleOff = slotManager.getTupleOff(slotOff);
- frameTuple.resetByTupleOffset(buf, tupleOff);
- int tupleSize = tupleWriter.bytesRequired(frameTuple);
+ // perform deletion (we just do a memcpy to overwrite the slot)
+ int slotStartOff = slotManager.getSlotEndOff();
+ int length = slotOff - slotStartOff;
+ System.arraycopy(buf.array(), slotStartOff, buf.array(), slotStartOff + slotManager.getSlotSize(), length);
- // perform deletion (we just do a memcpy to overwrite the slot)
- int slotStartOff = slotManager.getSlotEndOff();
- int length = slotOff - slotStartOff;
- System.arraycopy(buf.array(), slotStartOff, buf.array(), slotStartOff
- + slotManager.getSlotSize(), length);
+ // maintain space information
+ buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
+ buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + tupleSize + slotManager.getSlotSize());
+ }
- // maintain space information
- buf.putInt(tupleCountOff, buf.getInt(tupleCountOff) - 1);
- buf.putInt(totalFreeSpaceOff, buf.getInt(totalFreeSpaceOff) + tupleSize
- + slotManager.getSlotSize());
- }
-
- @Override
- public void adjustMBR(ITreeIndexTupleReference[] tuples, MultiComparator cmp) {
- for (int i = 0; i < tuples.length; i++) {
- tuples[i].setFieldCount(cmp.getFieldCount());
- tuples[i].resetByTupleIndex(this, 0);
- }
-
- adjustMBRImpl(tuples, cmp);
- }
-}
\ No newline at end of file
+ @Override
+ public void adjustMBR(ITreeIndexTupleReference[] tuples) {
+ for (int i = 0; i < tuples.length; i++) {
+ tuples[i].resetByTupleIndex(this, 0);
+ }
+ adjustMBRImpl(tuples);
+ }
+}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
index 51a047e..e31148f 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMLeafFrameFactory.java
@@ -15,29 +15,37 @@
package edu.uci.ics.hyracks.storage.am.rtree.frames;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
public class RTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory {
- private static final long serialVersionUID = 1L;
- private ITreeIndexTupleWriterFactory tupleWriterFactory;
- private int keyFieldCount;
+ private static final long serialVersionUID = 1L;
+ private final ITreeIndexTupleWriterFactory tupleWriterFactory;
+ private final IPrimitiveValueProviderFactory[] keyValueProviderFactories;
- public RTreeNSMLeafFrameFactory(
- ITreeIndexTupleWriterFactory tupleWriterFactory, int keyFieldCount) {
- this.tupleWriterFactory = tupleWriterFactory;
- if (keyFieldCount % 2 != 0) {
- throw new IllegalArgumentException(
- "The key has different number of dimensions.");
- }
- this.keyFieldCount = keyFieldCount;
- }
+ public RTreeNSMLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory, IPrimitiveValueProviderFactory[] keyValueProviderFactories) {
+ this.tupleWriterFactory = tupleWriterFactory;
+ if (keyValueProviderFactories.length % 2 != 0) {
+ throw new IllegalArgumentException("The key has different number of dimensions.");
+ }
+ this.keyValueProviderFactories = keyValueProviderFactories;
+ }
+
+ @Override
+ public IRTreeLeafFrame createFrame() {
+ IPrimitiveValueProvider[] keyValueProviders = new IPrimitiveValueProvider[keyValueProviderFactories.length];
+ for (int i = 0; i < keyValueProviders.length; i++) {
+ keyValueProviders[i] = keyValueProviderFactories[i].createPrimitiveValueProvider();
+ }
+ return new RTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter(), keyValueProviders);
+ }
@Override
- public IRTreeLeafFrame createFrame() {
- return new RTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter(),
- keyFieldCount);
+ public ITreeIndexTupleWriterFactory getTupleWriterFactory() {
+ return tupleWriterFactory;
}
-}
\ No newline at end of file
+}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/PathList.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/PathList.java
index d66d0a0..4f86111 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/PathList.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/PathList.java
@@ -16,15 +16,16 @@
package edu.uci.ics.hyracks.storage.am.rtree.impls;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IntArrayList;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.LongArrayList;
public class PathList {
private IntArrayList pageIds;
- private IntArrayList pageLsns;
+ private LongArrayList pageLsns;
private IntArrayList pageIndexes;
public PathList(int initialCapacity, int growth) {
pageIds = new IntArrayList(initialCapacity, growth);
- pageLsns = new IntArrayList(initialCapacity, growth);
+ pageLsns = new LongArrayList(initialCapacity, growth);
pageIndexes = new IntArrayList(initialCapacity, growth);
}
@@ -36,7 +37,7 @@
return pageIds.first();
}
- public void add(int pageId, int pageLsn, int pageIndex) {
+ public void add(int pageId, long pageLsn, int pageIndex) {
pageIds.add(pageId);
pageLsns.add(pageLsn);
pageIndexes.add(pageIndex);
@@ -46,7 +47,7 @@
return pageIds.getFirst();
}
- public int getFirstPageLsn() {
+ public long getFirstPageLsn() {
return pageLsns.getFirst();
}
@@ -58,7 +59,7 @@
return pageIds.getLast();
}
- public int getLastPageLsn() {
+ public long getLastPageLsn() {
return pageLsns.getLast();
}
@@ -70,7 +71,7 @@
return pageIds.get(i);
}
- public int getPageLsn(int i) {
+ public long getPageLsn(int i) {
return pageLsns.get(i);
}
@@ -78,7 +79,7 @@
return pageIndexes.get(i);
}
- public void setPageLsn(int i, int pageLsn) {
+ public void setPageLsn(int i, long pageLsn) {
pageLsns.set(i, pageLsn);
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
index 03b2062..5f78dcc 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTree.java
@@ -16,7 +16,6 @@
package edu.uci.ics.hyracks.storage.am.rtree.impls;
import java.util.ArrayList;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -26,17 +25,22 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ISearchPredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IndexType;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.FrameOpSpaceStatus;
import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexUtils;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
@@ -47,972 +51,1035 @@
public class RTree implements ITreeIndex {
- private boolean created = false;
- private boolean loaded = false;
- private final int rootPage = 1; // the root page never changes
+ private boolean created = false;
+ private boolean loaded = false;
+ private final int rootPage = 1; // the root page never changes
- private final AtomicInteger globalNsn; // Global node sequence number
- private int numOfPages = 1;
- private final ReadWriteLock treeLatch;
+ private final AtomicLong globalNsn; // Global node sequence number
+ private int numOfPages = 1;
+ private final ReadWriteLock treeLatch;
- private final IFreePageManager freePageManager;
- private final IBufferCache bufferCache;
- private int fileId;
+ private final IFreePageManager freePageManager;
+ private final IBufferCache bufferCache;
+ private int fileId;
- private final SearchPredicate diskOrderScanPredicate;
- private final ITreeIndexFrameFactory interiorFrameFactory;
- private final ITreeIndexFrameFactory leafFrameFactory;
- private final MultiComparator cmp;
+ private final SearchPredicate diskOrderScanPredicate;
+ private final ITreeIndexFrameFactory interiorFrameFactory;
+ private final ITreeIndexFrameFactory leafFrameFactory;
+ private final int fieldCount;
+ private final MultiComparator cmp;
- public int rootSplits = 0;
- public int[] splitsByLevel = new int[500];
- public AtomicLong readLatchesAcquired = new AtomicLong();
- public AtomicLong readLatchesReleased = new AtomicLong();
- public AtomicLong writeLatchesAcquired = new AtomicLong();
- public AtomicLong writeLatchesReleased = new AtomicLong();
- public AtomicLong pins = new AtomicLong();
- public AtomicLong unpins = new AtomicLong();
- public byte currentLevel = 0;
+ public int rootSplits = 0;
+ public int[] splitsByLevel = new int[500];
+ public AtomicLong readLatchesAcquired = new AtomicLong();
+ public AtomicLong readLatchesReleased = new AtomicLong();
+ public AtomicLong writeLatchesAcquired = new AtomicLong();
+ public AtomicLong writeLatchesReleased = new AtomicLong();
+ public AtomicLong pins = new AtomicLong();
+ public AtomicLong unpins = new AtomicLong();
+ public byte currentLevel = 0;
- public RTree(IBufferCache bufferCache, IFreePageManager freePageManager,
- ITreeIndexFrameFactory interiorFrameFactory,
- ITreeIndexFrameFactory leafFrameFactory, MultiComparator cmp) {
- this.bufferCache = bufferCache;
- this.freePageManager = freePageManager;
- this.interiorFrameFactory = interiorFrameFactory;
- this.leafFrameFactory = leafFrameFactory;
- this.cmp = cmp;
- globalNsn = new AtomicInteger();
- this.treeLatch = new ReentrantReadWriteLock(true);
- this.diskOrderScanPredicate = new SearchPredicate(null, cmp);
+ // TODO: is MultiComparator needed at all?
+ public RTree(IBufferCache bufferCache, int fieldCount, MultiComparator cmp, IFreePageManager freePageManager,
+ ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory) {
+ this.bufferCache = bufferCache;
+ this.fieldCount = fieldCount;
+ this.cmp = cmp;
+ this.freePageManager = freePageManager;
+ this.interiorFrameFactory = interiorFrameFactory;
+ this.leafFrameFactory = leafFrameFactory;
+ globalNsn = new AtomicLong();
+ this.treeLatch = new ReentrantReadWriteLock(true);
+ this.diskOrderScanPredicate = new SearchPredicate(null, cmp);
+ }
+
+ public void incrementGlobalNsn() {
+ globalNsn.incrementAndGet();
+ }
+
+ public long getGlobalNsn() {
+ return globalNsn.get();
+ }
+
+ public void incrementReadLatchesAcquired() {
+ readLatchesAcquired.incrementAndGet();
+ }
+
+ public void incrementReadLatchesReleased() {
+ readLatchesReleased.incrementAndGet();
+ }
+
+ public void incrementWriteLatchesAcquired() {
+ writeLatchesAcquired.incrementAndGet();
+ }
+
+ public void incrementWriteLatchesReleased() {
+ writeLatchesReleased.incrementAndGet();
+ }
+
+ public void incrementPins() {
+ pins.incrementAndGet();
+ }
+
+ public void incrementUnpins() {
+ unpins.incrementAndGet();
+ }
+
+ public String printStats() {
+ StringBuilder strBuilder = new StringBuilder();
+ strBuilder.append("\n");
+ strBuilder.append("ROOTSPLITS: " + rootSplits + "\n");
+ strBuilder.append("SPLITS BY LEVEL\n");
+ for (int i = 0; i < currentLevel; i++) {
+ strBuilder.append(String.format("%3d ", i) + String.format("%8d ", splitsByLevel[i]) + "\n");
+ }
+ strBuilder.append(String.format("READ LATCHES: %10d %10d\n", readLatchesAcquired.get(),
+ readLatchesReleased.get()));
+ strBuilder.append(String.format("WRITE LATCHES: %10d %10d\n", writeLatchesAcquired.get(),
+ writeLatchesReleased.get()));
+ strBuilder.append(String.format("PINS: %10d %10d\n", pins.get(), unpins.get()));
+
+ strBuilder.append(String.format("Num of Pages: %10d\n", numOfPages));
+
+ return strBuilder.toString();
+ }
+
+ public void printTree(IRTreeFrame leafFrame, IRTreeFrame interiorFrame, ISerializerDeserializer[] keySerdes)
+ throws Exception {
+ printTree(rootPage, null, false, leafFrame, interiorFrame, keySerdes);
+ }
+
+ public void printTree(int pageId, ICachedPage parent, boolean unpin, IRTreeFrame leafFrame,
+ IRTreeFrame interiorFrame, ISerializerDeserializer[] keySerdes) throws Exception {
+
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ incrementPins();
+ node.acquireReadLatch();
+ incrementReadLatchesAcquired();
+
+ try {
+ if (parent != null && unpin == true) {
+ parent.releaseReadLatch();
+ incrementReadLatchesReleased();
+ bufferCache.unpin(parent);
+ incrementUnpins();
+ }
+
+ interiorFrame.setPage(node);
+ int level = interiorFrame.getLevel();
+
+ System.out.format("%1d ", level);
+ System.out.format("%3d ", pageId);
+ for (int i = 0; i < currentLevel - level; i++)
+ System.out.format(" ");
+
+ String keyString;
+ if (interiorFrame.isLeaf()) {
+ leafFrame.setPage(node);
+ keyString = TreeIndexUtils.printFrameTuples(leafFrame, keySerdes);
+ } else {
+ keyString = TreeIndexUtils.printFrameTuples(interiorFrame, keySerdes);
+ }
+
+ System.out.format(keyString);
+ if (!interiorFrame.isLeaf()) {
+ ArrayList<Integer> children = ((RTreeNSMFrame) (interiorFrame)).getChildren(cmp);
+ for (int i = 0; i < children.size(); i++) {
+ printTree(children.get(i), node, i == children.size() - 1, leafFrame, interiorFrame, keySerdes);
+ }
+ } else {
+ node.releaseReadLatch();
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ }
+ } catch (Exception e) {
+ node.releaseReadLatch();
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ throw e;
+ }
+ }
+
+ @Override
+ public void create(int fileId) throws HyracksDataException {
+ treeLatch.writeLock().lock();
+ try {
+ if (created) {
+ return;
+ }
+
+ ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
+ ITreeIndexMetaDataFrame metaFrame = freePageManager.getMetaDataFrameFactory().createFrame();
+ freePageManager.init(metaFrame, rootPage);
+
+ // initialize root page
+ ICachedPage rootNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
+ incrementPins();
+
+ rootNode.acquireWriteLatch();
+ incrementWriteLatchesAcquired();
+ try {
+ leafFrame.setPage(rootNode);
+ leafFrame.initBuffer((byte) 0);
+ } finally {
+ rootNode.releaseWriteLatch();
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(rootNode);
+ incrementUnpins();
+ }
+ currentLevel = 0;
+
+ created = true;
+ } finally {
+ treeLatch.writeLock().unlock();
+ }
+ }
+
+ public void open(int fileId) {
+ this.fileId = fileId;
+ }
+
+ public void close() {
+ fileId = -1;
+ }
+
+ private RTreeOpContext createOpContext() {
+ return new RTreeOpContext((IRTreeLeafFrame) leafFrameFactory.createFrame(),
+ (IRTreeInteriorFrame) interiorFrameFactory.createFrame(), freePageManager.getMetaDataFrameFactory()
+ .createFrame(), 8);
+ }
+
+ private void insert(ITupleReference tuple, IIndexOpContext ictx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ RTreeOpContext ctx = (RTreeOpContext) ictx;
+ ctx.reset();
+ ctx.setTuple(tuple);
+ ctx.splitKey.reset();
+ ctx.splitKey.getLeftTuple().setFieldCount(cmp.getKeyFieldCount());
+ ctx.splitKey.getRightTuple().setFieldCount(cmp.getKeyFieldCount());
+
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i), tuple.getFieldData(j), tuple.getFieldStart(j), tuple.getFieldLength(j));
+ if (c > 0) {
+ throw new IllegalArgumentException("The low key point has larger coordinates than the high key point.");
+ }
+ }
+
+ ICachedPage leafNode = findLeaf(ctx);
+
+ int pageId = ctx.pathList.getLastPageId();
+ ctx.pathList.moveLast();
+ insertTuple(leafNode, pageId, ctx.getTuple(), ctx, true);
+
+ while (true) {
+ if (ctx.splitKey.getLeftPageBuffer() != null) {
+ updateParentForInsert(ctx);
+ } else {
+ break;
+ }
+ }
+
+ leafNode.releaseWriteLatch();
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(leafNode);
+ incrementUnpins();
+ }
+
+ public ICachedPage findLeaf(RTreeOpContext ctx) throws HyracksDataException {
+ int pageId = rootPage;
+ boolean writeLatched = false;
+ boolean readLatched = false;
+ boolean succeed = false;
+ ICachedPage node = null;
+ boolean isLeaf = false;
+ long pageLsn = 0, parentLsn = 0;
+
+ try {
+
+ while (true) {
+ if (!writeLatched) {
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ incrementPins();
+ ctx.interiorFrame.setPage(node);
+ isLeaf = ctx.interiorFrame.isLeaf();
+ if (isLeaf) {
+ node.acquireWriteLatch();
+ writeLatched = true;
+ incrementWriteLatchesAcquired();
+
+ if (!ctx.interiorFrame.isLeaf()) {
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ continue;
+ }
+ } else {
+ // Be optimistic and grab read latch first. We will swap
+ // it
+ // to write latch if we need to enlarge the best child
+ // tuple.
+ node.acquireReadLatch();
+ readLatched = true;
+ incrementReadLatchesAcquired();
+ }
+ }
+
+ if (pageId != rootPage && parentLsn < ctx.interiorFrame.getPageNsn()) {
+ // Concurrent split detected, go back to parent and
+ // re-choose
+ // the best child
+ if (writeLatched) {
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ } else {
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ }
+
+ pageId = ctx.pathList.getLastPageId();
+ if (pageId != rootPage) {
+ parentLsn = ctx.pathList.getPageLsn(ctx.pathList.size() - 2);
+ }
+ ctx.pathList.moveLast();
+ continue;
+ }
+
+ pageLsn = ctx.interiorFrame.getPageLsn();
+ ctx.pathList.add(pageId, pageLsn, -1);
+
+ if (!isLeaf) {
+ // findBestChild must be called *before* getBestChildPageId
+ ctx.interiorFrame.findBestChild(ctx.getTuple(), cmp);
+ int childPageId = ctx.interiorFrame.getBestChildPageId();
+
+ if (!writeLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ // TODO: do we need to un-pin and pin again?
+ bufferCache.unpin(node);
+ incrementUnpins();
+
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ incrementPins();
+ node.acquireWriteLatch();
+ writeLatched = true;
+ incrementWriteLatchesAcquired();
+ ctx.interiorFrame.setPage(node);
+
+ if (ctx.interiorFrame.getPageLsn() != pageLsn) {
+ // The page was changed while we unlocked it; thus,
+ // retry (re-choose best child)
+
+ ctx.pathList.moveLast();
+ continue;
+ }
+ }
+
+ // We don't need to reset the frameTuple because it is
+ // already pointing to the best child
+ ctx.interiorFrame.enlarge(ctx.getTuple(), cmp);
+
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+
+ pageId = childPageId;
+ parentLsn = pageLsn;
+ } else {
+ ctx.leafFrame.setPage(node);
+ succeed = true;
+ return node;
+ }
+ }
+ } finally {
+ if (!succeed) {
+ if (readLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ } else if (writeLatched) {
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ }
+ }
+ }
+ }
+
+ private void insertTuple(ICachedPage node, int pageId, ITupleReference tuple, RTreeOpContext ctx, boolean isLeaf)
+ throws HyracksDataException, TreeIndexException, PageAllocationException {
+ FrameOpSpaceStatus spaceStatus;
+ if (!isLeaf) {
+ spaceStatus = ctx.interiorFrame.hasSpaceInsert(tuple);
+ } else {
+ spaceStatus = ctx.leafFrame.hasSpaceInsert(tuple);
+ }
+
+ switch (spaceStatus) {
+ case SUFFICIENT_CONTIGUOUS_SPACE: {
+ if (!isLeaf) {
+ ctx.interiorFrame.insert(tuple, -1);
+ incrementGlobalNsn();
+ ctx.interiorFrame.setPageLsn(getGlobalNsn());
+ } else {
+ ctx.leafFrame.insert(tuple, -1);
+ incrementGlobalNsn();
+ ctx.leafFrame.setPageLsn(getGlobalNsn());
+ }
+ ctx.splitKey.reset();
+ break;
+ }
+
+ case SUFFICIENT_SPACE: {
+ if (!isLeaf) {
+ ctx.interiorFrame.compact();
+ ctx.interiorFrame.insert(tuple, -1);
+ incrementGlobalNsn();
+ ctx.interiorFrame.setPageLsn(getGlobalNsn());
+ } else {
+ ctx.leafFrame.compact();
+ ctx.leafFrame.insert(tuple, -1);
+ incrementGlobalNsn();
+ ctx.leafFrame.setPageLsn(getGlobalNsn());
+ }
+ ctx.splitKey.reset();
+ break;
+ }
+
+ case INSUFFICIENT_SPACE: {
+ int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
+ ICachedPage rightNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId), true);
+ incrementPins();
+ rightNode.acquireWriteLatch();
+ incrementWriteLatchesAcquired();
+
+ try {
+ IRTreeFrame rightFrame;
+ numOfPages++; // debug
+ if (!isLeaf) {
+ splitsByLevel[ctx.interiorFrame.getLevel()]++; // debug
+ rightFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
+ rightFrame.setPage(rightNode);
+ rightFrame.initBuffer((byte) ctx.interiorFrame.getLevel());
+ ctx.interiorFrame.split(rightFrame, tuple, ctx.splitKey);
+ ctx.interiorFrame.setRightPage(rightPageId);
+ rightFrame.setPageNsn(ctx.interiorFrame.getPageNsn());
+ incrementGlobalNsn();
+ long newNsn = getGlobalNsn();
+ rightFrame.setPageLsn(newNsn);
+ ctx.interiorFrame.setPageNsn(newNsn);
+ ctx.interiorFrame.setPageLsn(newNsn);
+ } else {
+ splitsByLevel[0]++; // debug
+ rightFrame = (IRTreeFrame) leafFrameFactory.createFrame();
+ rightFrame.setPage(rightNode);
+ rightFrame.initBuffer((byte) 0);
+ ctx.leafFrame.split(rightFrame, tuple, ctx.splitKey);
+ ctx.leafFrame.setRightPage(rightPageId);
+ rightFrame.setPageNsn(ctx.leafFrame.getPageNsn());
+ incrementGlobalNsn();
+ long newNsn = getGlobalNsn();
+ rightFrame.setPageLsn(newNsn);
+ ctx.leafFrame.setPageNsn(newNsn);
+ ctx.leafFrame.setPageLsn(newNsn);
+ }
+ ctx.splitKey.setPages(pageId, rightPageId);
+ if (pageId == rootPage) {
+ rootSplits++; // debug
+ splitsByLevel[currentLevel]++;
+ currentLevel++;
+
+ int newLeftId = freePageManager.getFreePage(ctx.metaFrame);
+ ICachedPage newLeftNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, newLeftId),
+ true);
+ incrementPins();
+ newLeftNode.acquireWriteLatch();
+ incrementWriteLatchesAcquired();
+ try {
+ // copy left child to new left child
+ System.arraycopy(node.getBuffer().array(), 0, newLeftNode.getBuffer().array(), 0,
+ newLeftNode.getBuffer().capacity());
+
+ // initialize new root (leftNode becomes new root)
+ ctx.interiorFrame.setPage(node);
+ ctx.interiorFrame.initBuffer((byte) (ctx.interiorFrame.getLevel() + 1));
+
+ ctx.splitKey.setLeftPage(newLeftId);
+
+ ctx.interiorFrame.insert(ctx.splitKey.getLeftTuple(), -1);
+ ctx.interiorFrame.insert(ctx.splitKey.getRightTuple(), -1);
+
+ incrementGlobalNsn();
+ long newNsn = getGlobalNsn();
+ ctx.interiorFrame.setPageLsn(newNsn);
+ ctx.interiorFrame.setPageNsn(newNsn);
+ } finally {
+ newLeftNode.releaseWriteLatch();
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(newLeftNode);
+ incrementUnpins();
+ }
+
+ ctx.splitKey.reset();
+ }
+ } finally {
+ rightNode.releaseWriteLatch();
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(rightNode);
+ incrementUnpins();
+ }
+ break;
+ }
+ }
+ }
+
+ public void updateParentForInsert(RTreeOpContext ctx) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ boolean writeLatched = false;
+ int parentId = ctx.pathList.getLastPageId();
+ ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
+ incrementPins();
+ parentNode.acquireWriteLatch();
+ writeLatched = true;
+ incrementWriteLatchesAcquired();
+ ctx.interiorFrame.setPage(parentNode);
+ boolean foundParent = true;
+
+ try {
+
+ if (ctx.interiorFrame.getPageLsn() != ctx.pathList.getLastPageLsn()) {
+ foundParent = false;
+ while (true) {
+ if (ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), cmp) != -1) {
+ // found the parent
+ foundParent = true;
+ break;
+ }
+ int rightPage = ctx.interiorFrame.getRightPage();
+ parentNode.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(parentNode);
+ incrementUnpins();
+
+ if (rightPage == -1) {
+ break;
+ }
+
+ parentId = rightPage;
+ parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
+ incrementPins();
+ parentNode.acquireWriteLatch();
+ writeLatched = true;
+ incrementWriteLatchesAcquired();
+ ctx.interiorFrame.setPage(parentNode);
+ }
+ }
+ if (foundParent) {
+ ctx.interiorFrame.adjustKey(ctx.splitKey.getLeftTuple(), -1, cmp);
+ insertTuple(parentNode, parentId, ctx.splitKey.getRightTuple(), ctx, ctx.interiorFrame.isLeaf());
+ ctx.pathList.moveLast();
+
+ parentNode.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(parentNode);
+ incrementUnpins();
+ return;
+ }
+
+ } finally {
+ if (writeLatched) {
+ parentNode.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(parentNode);
+ incrementUnpins();
+ }
+ }
+ // very rare situation when the there is a root split, do an
+ // exhaustive
+ // breadth-first traversal looking for the parent tuple
+
+ ctx.pathList.clear();
+ ctx.traverseList.clear();
+ findPath(ctx);
+ updateParentForInsert(ctx);
+ }
+
+ public void findPath(RTreeOpContext ctx) throws HyracksDataException {
+ boolean readLatched = false;
+ int pageId = rootPage;
+ int parentIndex = -1;
+ long parentLsn = 0;
+ long pageLsn;
+ int pageIndex;
+ ICachedPage node = null;
+ ctx.traverseList.add(pageId, -1, parentIndex);
+ try {
+ while (!ctx.traverseList.isLast()) {
+ pageId = ctx.traverseList.getFirstPageId();
+ parentIndex = ctx.traverseList.getFirstPageIndex();
+
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ incrementPins();
+ node.acquireReadLatch();
+ readLatched = true;
+ incrementReadLatchesAcquired();
+ ctx.interiorFrame.setPage(node);
+ pageLsn = ctx.interiorFrame.getPageLsn();
+ pageIndex = ctx.traverseList.first();
+ ctx.traverseList.setPageLsn(pageIndex, pageLsn);
+
+ ctx.traverseList.moveFirst();
+
+ if (pageId != rootPage && parentLsn < ctx.interiorFrame.getPageNsn()) {
+ int rightPage = ctx.interiorFrame.getRightPage();
+ if (rightPage != -1) {
+ ctx.traverseList.add(rightPage, -1, parentIndex);
+ }
+ }
+ parentLsn = pageLsn;
+
+ if (ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), ctx.traverseList, pageIndex, cmp) != -1) {
+ fillPath(ctx, pageIndex);
+ return;
+ }
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ }
+ } finally {
+ if (readLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ }
+ }
+ }
+
+ public void fillPath(RTreeOpContext ctx, int pageIndex) {
+ if (pageIndex != -1) {
+ fillPath(ctx, ctx.traverseList.getPageIndex(pageIndex));
+ ctx.pathList.add(ctx.traverseList.getPageId(pageIndex), ctx.traverseList.getPageLsn(pageIndex), -1);
+ }
+ }
+
+ public void delete(ITupleReference tuple, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ ctx.reset();
+ ctx.setTuple(tuple);
+ ctx.splitKey.reset();
+ ctx.splitKey.getLeftTuple().setFieldCount(cmp.getKeyFieldCount());
+
+ int tupleIndex = findTupleToDelete(ctx);
+
+ if (tupleIndex != -1) {
+ int pageId = ctx.pathList.getLastPageId();
+ ctx.pathList.moveLast();
+ deleteTuple(pageId, tupleIndex, ctx);
+
+ while (true) {
+ if (ctx.splitKey.getLeftPageBuffer() != null) {
+ updateParentForDelete(ctx);
+ } else {
+ break;
+ }
+ }
+
+ ctx.leafFrame.getPage().releaseWriteLatch();
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(ctx.leafFrame.getPage());
+ incrementUnpins();
+ }
+ }
+
+ public void updateParentForDelete(RTreeOpContext ctx) throws HyracksDataException {
+ boolean writeLatched = false;
+ int parentId = ctx.pathList.getLastPageId();
+ ICachedPage parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
+ incrementPins();
+ parentNode.acquireWriteLatch();
+ writeLatched = true;
+ incrementWriteLatchesAcquired();
+ ctx.interiorFrame.setPage(parentNode);
+ boolean foundParent = true;
+ int tupleIndex = -1;
+
+ try {
+ if (ctx.interiorFrame.getPageLsn() != ctx.pathList.getLastPageLsn()) {
+ foundParent = false;
+ while (true) {
+ tupleIndex = ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), cmp);
+ if (tupleIndex != -1) {
+ // found the parent
+ foundParent = true;
+ break;
+ }
+ int rightPage = ctx.interiorFrame.getRightPage();
+ parentNode.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(parentNode);
+ incrementUnpins();
+
+ if (rightPage == -1) {
+ break;
+ }
+
+ parentId = rightPage;
+ parentNode = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, parentId), false);
+ incrementPins();
+ parentNode.acquireWriteLatch();
+ writeLatched = true;
+ incrementWriteLatchesAcquired();
+ ctx.interiorFrame.setPage(parentNode);
+ }
+ }
+ if (foundParent) {
+ if (tupleIndex == -1) {
+ tupleIndex = ctx.interiorFrame.findTupleByPointer(ctx.splitKey.getLeftTuple(), cmp);
+ }
+ boolean recomputeMBR = ctx.interiorFrame.recomputeMBR(ctx.splitKey.getLeftTuple(), tupleIndex, cmp);
+
+ if (recomputeMBR) {
+ ctx.interiorFrame.adjustKey(ctx.splitKey.getLeftTuple(), tupleIndex, cmp);
+ ctx.pathList.moveLast();
+
+ incrementGlobalNsn();
+ ctx.interiorFrame.setPageLsn(getGlobalNsn());
+
+ ctx.splitKey.reset();
+ if (!ctx.pathList.isEmpty()) {
+ ctx.interiorFrame.computeMBR(ctx.splitKey);
+ ctx.splitKey.setLeftPage(parentId);
+ }
+ } else {
+ ctx.pathList.moveLast();
+ ctx.splitKey.reset();
+ }
+
+ parentNode.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(parentNode);
+ incrementUnpins();
+ return;
+ }
+ } finally {
+ if (writeLatched) {
+ parentNode.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(parentNode);
+ incrementUnpins();
+ }
+ }
+
+ // very rare situation when the there is a root split, do an exhaustive
+ // breadth-first traversal looking for the parent tuple
+
+ ctx.pathList.clear();
+ ctx.traverseList.clear();
+ findPath(ctx);
+ updateParentForDelete(ctx);
+ }
+
+ public int findTupleToDelete(RTreeOpContext ctx) throws HyracksDataException {
+ boolean writeLatched = false;
+ boolean readLatched = false;
+ boolean succeed = false;
+ ICachedPage node = null;
+ ctx.traverseList.add(rootPage, -1, -1);
+ ctx.pathList.add(rootPage, -1, ctx.traverseList.size() - 1);
+
+ try {
+ while (!ctx.pathList.isEmpty()) {
+ int pageId = ctx.pathList.getLastPageId();
+ long parentLsn = ctx.pathList.getLastPageLsn();
+ int pageIndex = ctx.pathList.getLastPageIndex();
+ ctx.pathList.moveLast();
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ incrementPins();
+ node.acquireReadLatch();
+ readLatched = true;
+ incrementReadLatchesAcquired();
+ ctx.interiorFrame.setPage(node);
+ boolean isLeaf = ctx.interiorFrame.isLeaf();
+ long pageLsn = ctx.interiorFrame.getPageLsn();
+ int parentIndex = ctx.traverseList.getPageIndex(pageIndex);
+ ctx.traverseList.setPageLsn(pageIndex, pageLsn);
+
+ if (pageId != rootPage && parentLsn < ctx.interiorFrame.getPageNsn()) {
+ // Concurrent split detected, we need to visit the right
+ // page
+ int rightPage = ctx.interiorFrame.getRightPage();
+ if (rightPage != -1) {
+ ctx.traverseList.add(rightPage, -1, parentIndex);
+ ctx.pathList.add(rightPage, parentLsn, ctx.traverseList.size() - 1);
+ }
+ }
+
+ if (!isLeaf) {
+ for (int i = 0; i < ctx.interiorFrame.getTupleCount(); i++) {
+ int childPageId = ctx.interiorFrame.getChildPageIdIfIntersect(ctx.tuple, i, cmp);
+ if (childPageId != -1) {
+ ctx.traverseList.add(childPageId, -1, pageIndex);
+ ctx.pathList.add(childPageId, pageLsn, ctx.traverseList.size() - 1);
+ }
+ }
+ } else {
+ ctx.leafFrame.setPage(node);
+ int tupleIndex = ctx.leafFrame.findTupleIndex(ctx.tuple, cmp);
+ if (tupleIndex != -1) {
+
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+
+ node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ incrementPins();
+ node.acquireWriteLatch();
+ writeLatched = true;
+ incrementWriteLatchesAcquired();
+ ctx.leafFrame.setPage(node);
+
+ if (ctx.leafFrame.getPageLsn() != pageLsn) {
+ // The page was changed while we unlocked it
+
+ tupleIndex = ctx.leafFrame.findTupleIndex(ctx.tuple, cmp);
+ if (tupleIndex == -1) {
+ ctx.traverseList.add(pageId, -1, parentIndex);
+ ctx.pathList.add(pageId, parentLsn, ctx.traverseList.size() - 1);
+
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ continue;
+ } else {
+ ctx.pathList.clear();
+ fillPath(ctx, pageIndex);
+ succeed = true;
+ return tupleIndex;
+ }
+ } else {
+ ctx.pathList.clear();
+ fillPath(ctx, pageIndex);
+ succeed = true;
+ return tupleIndex;
+ }
+ }
+ }
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ }
+ } finally {
+ if (!succeed) {
+ if (readLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ incrementReadLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ } else if (writeLatched) {
+ node.releaseWriteLatch();
+ writeLatched = false;
+ incrementWriteLatchesReleased();
+ bufferCache.unpin(node);
+ incrementUnpins();
+ }
+ }
+ }
+ return -1;
+ }
+
+ public void deleteTuple(int pageId, int tupleIndex, RTreeOpContext ctx) throws HyracksDataException {
+ ctx.leafFrame.delete(tupleIndex, cmp);
+ incrementGlobalNsn();
+ ctx.leafFrame.setPageLsn(getGlobalNsn());
+
+ // if the page is empty, just leave it there for future inserts
+ if (pageId != rootPage && ctx.leafFrame.getTupleCount() > 0) {
+ ctx.leafFrame.computeMBR(ctx.splitKey);
+ ctx.splitKey.setLeftPage(pageId);
+ }
+ }
+
+ private void search(ITreeIndexCursor cursor, ISearchPredicate searchPred, RTreeOpContext ctx) throws HyracksDataException, TreeIndexException {
+ ctx.reset();
+ ctx.cursor = cursor;
+
+ cursor.setBufferCache(bufferCache);
+ cursor.setFileId(fileId);
+ ctx.cursorInitialState.setRootPage(rootPage);
+ ctx.cursor.open(ctx.cursorInitialState, (SearchPredicate)searchPred);
+ }
+
+ public ITreeIndexFrameFactory getInteriorFrameFactory() {
+ return interiorFrameFactory;
+ }
+
+ public ITreeIndexFrameFactory getLeafFrameFactory() {
+ return leafFrameFactory;
+ }
+
+ public MultiComparator getCmp() {
+ return cmp;
+ }
+
+ public IFreePageManager getFreePageManager() {
+ return freePageManager;
+ }
+
+ private void update(ITupleReference tuple, RTreeOpContext ctx) {
+ throw new UnsupportedOperationException("RTree Update not implemented.");
+ }
+
+ public final class BulkLoadContext implements IIndexBulkLoadContext {
+
+ public ITreeIndexAccessor indexAccessor;
+
+ public BulkLoadContext(float fillFactor, IRTreeFrame leafFrame, IRTreeFrame interiorFrame,
+ ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
+ indexAccessor = createAccessor();
+ }
+ }
+
+ @Override
+ public IIndexBulkLoadContext beginBulkLoad(float fillFactor) throws HyracksDataException {
+ if (loaded) {
+ throw new HyracksDataException("Trying to bulk-load RTree but RTree has already been loaded.");
+ }
+
+ BulkLoadContext ctx = new BulkLoadContext(fillFactor, (IRTreeFrame) leafFrameFactory.createFrame(),
+ (IRTreeFrame) interiorFrameFactory.createFrame(), freePageManager.getMetaDataFrameFactory()
+ .createFrame());
+ return ctx;
+ }
+
+ @Override
+ public void bulkLoadAddTuple(ITupleReference tuple, IIndexBulkLoadContext ictx) throws HyracksDataException {
+ try {
+ ((BulkLoadContext) ictx).indexAccessor.insert(tuple);
+ } catch (Exception e) {
+ throw new HyracksDataException("BulkLoad Error");
+ }
+ }
+
+ @Override
+ public void endBulkLoad(IIndexBulkLoadContext ictx) throws HyracksDataException {
+ loaded = true;
+ }
+
+ private void diskOrderScan(ITreeIndexCursor icursor, RTreeOpContext ctx) throws HyracksDataException {
+ TreeDiskOrderScanCursor cursor = (TreeDiskOrderScanCursor) icursor;
+ ctx.reset();
+
+ int currentPageId = rootPage + 1;
+ int maxPageId = freePageManager.getMaxPage(ctx.metaFrame);
+
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
+ page.acquireReadLatch();
+ try {
+ cursor.setBufferCache(bufferCache);
+ cursor.setFileId(fileId);
+ cursor.setCurrentPageId(currentPageId);
+ cursor.setMaxPageId(maxPageId);
+ ctx.cursorInitialState.setPage(page);
+ cursor.open(ctx.cursorInitialState, diskOrderScanPredicate);
+ } catch (Exception e) {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ throw new HyracksDataException(e);
+ }
+ }
+
+ @Override
+ public int getRootPageId() {
+ return rootPage;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return fieldCount;
+ }
+
+ @Override
+ public IndexType getIndexType() {
+ return IndexType.RTREE;
+ }
+
+ @Override
+ public ITreeIndexAccessor createAccessor() {
+ return new RTreeAccessor(this);
}
+
+ private class RTreeAccessor implements ITreeIndexAccessor {
+ private RTree rtree;
+ private RTreeOpContext ctx;
+
+ public RTreeAccessor(RTree rtree) {
+ this.rtree = rtree;
+ this.ctx = rtree.createOpContext();
+ }
+
+ @Override
+ public void insert(ITupleReference tuple) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ ctx.reset(IndexOp.INSERT);
+ rtree.insert(tuple, ctx);
+ }
- public void incrementGlobalNsn() {
- globalNsn.incrementAndGet();
- }
+ @Override
+ public void update(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ ctx.reset(IndexOp.UPDATE);
+ rtree.update(tuple, ctx);
+ }
- public int getGlobalNsn() {
- return globalNsn.get();
- }
+ @Override
+ public void delete(ITupleReference tuple) throws HyracksDataException, TreeIndexException {
+ ctx.reset(IndexOp.DELETE);
+ rtree.delete(tuple, ctx);
+ }
- public void incrementReadLatchesAcquired() {
- readLatchesAcquired.incrementAndGet();
- }
+ @Override
+ public void search(ITreeIndexCursor cursor, ISearchPredicate searchPred) throws HyracksDataException,
+ TreeIndexException {
+ ctx.reset(IndexOp.SEARCH);
+ rtree.search(cursor, searchPred, ctx);
+ }
- public void incrementReadLatchesReleased() {
- readLatchesReleased.incrementAndGet();
- }
-
- public void incrementWriteLatchesAcquired() {
- writeLatchesAcquired.incrementAndGet();
- }
-
- public void incrementWriteLatchesReleased() {
- writeLatchesReleased.incrementAndGet();
- }
-
- public void incrementPins() {
- pins.incrementAndGet();
- }
-
- public void incrementUnpins() {
- unpins.incrementAndGet();
- }
-
- public String printStats() {
- StringBuilder strBuilder = new StringBuilder();
- strBuilder.append("\n");
- strBuilder.append("ROOTSPLITS: " + rootSplits + "\n");
- strBuilder.append("SPLITS BY LEVEL\n");
- for (int i = 0; i < currentLevel; i++) {
- strBuilder.append(String.format("%3d ", i)
- + String.format("%8d ", splitsByLevel[i]) + "\n");
- }
- strBuilder.append(String.format("READ LATCHES: %10d %10d\n",
- readLatchesAcquired.get(), readLatchesReleased.get()));
- strBuilder.append(String.format("WRITE LATCHES: %10d %10d\n",
- writeLatchesAcquired.get(), writeLatchesReleased.get()));
- strBuilder.append(String.format("PINS: %10d %10d\n",
- pins.get(), unpins.get()));
-
- strBuilder.append(String.format("Num of Pages: %10d\n",
- numOfPages));
-
- return strBuilder.toString();
- }
-
- public void printTree(IRTreeFrame leafFrame, IRTreeFrame interiorFrame,
- ISerializerDeserializer[] fields) throws Exception {
- printTree(rootPage, null, false, leafFrame, interiorFrame, fields);
- }
-
- public void printTree(int pageId, ICachedPage parent, boolean unpin,
- IRTreeFrame leafFrame, IRTreeFrame interiorFrame,
- ISerializerDeserializer[] fields) throws Exception {
-
- ICachedPage node = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
- node.acquireReadLatch();
- incrementReadLatchesAcquired();
-
- try {
- if (parent != null && unpin == true) {
- parent.releaseReadLatch();
- incrementReadLatchesReleased();
- bufferCache.unpin(parent);
- incrementUnpins();
- }
-
- interiorFrame.setPage(node);
- int level = interiorFrame.getLevel();
-
- System.out.format("%1d ", level);
- System.out.format("%3d ", pageId);
- for (int i = 0; i < currentLevel - level; i++)
- System.out.format(" ");
-
- String keyString;
- if (interiorFrame.isLeaf()) {
- leafFrame.setPage(node);
- keyString = leafFrame.printKeys(cmp, fields);
- } else {
- keyString = interiorFrame.printKeys(cmp, fields);
- }
-
- System.out.format(keyString);
- if (!interiorFrame.isLeaf()) {
- ArrayList<Integer> children = ((RTreeNSMFrame) (interiorFrame))
- .getChildren(cmp);
- for (int i = 0; i < children.size(); i++) {
- printTree(children.get(i), node, i == children.size() - 1,
- leafFrame, interiorFrame, fields);
- }
- } else {
- node.releaseReadLatch();
- incrementReadLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- }
- } catch (Exception e) {
- node.releaseReadLatch();
- incrementReadLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- throw e;
- }
- }
-
- @Override
- public void create(int fileId, ITreeIndexFrame leafFrame,
- ITreeIndexMetaDataFrame metaFrame) throws Exception {
- if (created)
- return;
-
- treeLatch.writeLock().lock();
- try {
- // check if another thread beat us to it
- if (created)
- return;
-
- freePageManager.init(metaFrame, rootPage);
-
- // initialize root page
- ICachedPage rootNode = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, rootPage), true);
- incrementPins();
-
- rootNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- try {
- leafFrame.setPage(rootNode);
- leafFrame.initBuffer((byte) 0);
- } finally {
- rootNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(rootNode);
- incrementUnpins();
- }
- currentLevel = 0;
-
- created = true;
- } finally {
- treeLatch.writeLock().unlock();
- }
- }
-
- public void open(int fileId) {
- this.fileId = fileId;
- }
-
- public void close() {
- fileId = -1;
- }
-
- @Override
- public RTreeOpContext createOpContext(IndexOp op,
- ITreeIndexFrame leafFrame, ITreeIndexFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame) {
- return new RTreeOpContext(op, (IRTreeLeafFrame) leafFrame,
- (IRTreeInteriorFrame) interiorFrame, metaFrame, 8);
- }
-
- @Override
- public void insert(ITupleReference tuple, IndexOpContext ictx)
- throws Exception {
- RTreeOpContext ctx = (RTreeOpContext) ictx;
- ctx.reset();
- ctx.setTuple(tuple);
- ctx.splitKey.reset();
- ctx.splitKey.getLeftTuple().setFieldCount(cmp.getKeyFieldCount());
- ctx.splitKey.getRightTuple().setFieldCount(cmp.getKeyFieldCount());
- ctx.interiorFrame.setPageTupleFieldCount(cmp.getKeyFieldCount());
- ctx.leafFrame.setPageTupleFieldCount(cmp.getFieldCount());
-
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(tuple.getFieldData(i),
- tuple.getFieldStart(i), tuple.getFieldLength(i),
- tuple.getFieldData(j), tuple.getFieldStart(j),
- tuple.getFieldLength(j));
- if (c > 0) {
- throw new IllegalArgumentException(
- "The low key point has larger coordinates than the high key point.");
- }
- }
-
- ICachedPage leafNode = findLeaf(ctx);
-
- int pageId = ctx.pathList.getLastPageId();
- ctx.pathList.moveLast();
- insertTuple(leafNode, pageId, ctx.getTuple(), ctx, true);
-
- while (true) {
- if (ctx.splitKey.getLeftPageBuffer() != null) {
- updateParentForInsert(ctx);
- } else {
- break;
- }
- }
-
- leafNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(leafNode);
- incrementUnpins();
- }
-
- public ICachedPage findLeaf(RTreeOpContext ctx) throws Exception {
- int pageId = rootPage;
- boolean writeLatched = false;
- ICachedPage node = null;
- boolean isLeaf = false;
- int pageLsn = 0, parentLsn = 0;
-
- while (true) {
- if (!writeLatched) {
- node = bufferCache
- .pin(BufferedFileHandle.getDiskPageId(fileId, pageId),
- false);
- incrementPins();
- ctx.interiorFrame.setPage(node);
- isLeaf = ctx.interiorFrame.isLeaf();
- if (isLeaf) {
- node.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- writeLatched = true;
-
- if (!ctx.interiorFrame.isLeaf()) {
- node.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- writeLatched = false;
- continue;
- }
- } else {
- // Be optimistic and grab read latch first. We will swap it
- // to write latch if we need to enlarge the best child
- // tuple.
- node.acquireReadLatch();
- incrementReadLatchesAcquired();
- }
- }
-
- if (pageId != rootPage
- && parentLsn < ctx.interiorFrame.getPageNsn()) {
- // Concurrent split detected, go back to parent and re-choose
- // the best child
- if (writeLatched) {
- node.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- writeLatched = false;
- } else {
- node.releaseReadLatch();
- incrementReadLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- }
-
- pageId = ctx.pathList.getLastPageId();
- if (pageId != rootPage) {
- parentLsn = ctx.pathList
- .getPageLsn(ctx.pathList.size() - 2);
- }
- ctx.pathList.moveLast();
- continue;
- }
-
- pageLsn = ctx.interiorFrame.getPageLsn();
- ctx.pathList.add(pageId, pageLsn, -1);
-
- if (!isLeaf) {
- // findBestChild must be called *before* getBestChildPageId
- ctx.interiorFrame.findBestChild(ctx.getTuple(), cmp);
- int childPageId = ctx.interiorFrame.getBestChildPageId(cmp);
-
- if (!writeLatched) {
- node.releaseReadLatch();
- incrementReadLatchesReleased();
- // TODO: do we need to un-pin and pin again?
- bufferCache.unpin(node);
- incrementUnpins();
-
- node = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId),
- false);
- incrementPins();
- node.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(node);
- writeLatched = true;
-
- if (ctx.interiorFrame.getPageLsn() != pageLsn) {
- // The page was changed while we unlocked it; thus,
- // retry (re-choose best child)
-
- ctx.pathList.moveLast();
- continue;
- }
- }
-
- // We don't need to reset the frameTuple because it is
- // already pointing to the best child
- ctx.interiorFrame.enlarge(ctx.getTuple(), cmp);
-
- node.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- writeLatched = false;
-
- pageId = childPageId;
- parentLsn = pageLsn;
- } else {
- ctx.leafFrame.setPage(node);
- return node;
- }
- }
- }
-
- private void insertTuple(ICachedPage node, int pageId,
- ITupleReference tuple, RTreeOpContext ctx, boolean isLeaf)
- throws Exception {
- FrameOpSpaceStatus spaceStatus;
- if (!isLeaf) {
- spaceStatus = ctx.interiorFrame.hasSpaceInsert(tuple, cmp);
- } else {
- spaceStatus = ctx.leafFrame.hasSpaceInsert(tuple, cmp);
- }
-
- switch (spaceStatus) {
- case SUFFICIENT_CONTIGUOUS_SPACE: {
- if (!isLeaf) {
- ctx.interiorFrame.insert(tuple, cmp, -1);
- incrementGlobalNsn();
- ctx.interiorFrame.setPageLsn(getGlobalNsn());
- } else {
- ctx.leafFrame.insert(tuple, cmp, -1);
- incrementGlobalNsn();
- ctx.leafFrame.setPageLsn(getGlobalNsn());
- }
- ctx.splitKey.reset();
- break;
- }
-
- case SUFFICIENT_SPACE: {
- if (!isLeaf) {
- ctx.interiorFrame.compact(cmp);
- ctx.interiorFrame.insert(tuple, cmp, -1);
- incrementGlobalNsn();
- ctx.interiorFrame.setPageLsn(getGlobalNsn());
- } else {
- ctx.leafFrame.compact(cmp);
- ctx.leafFrame.insert(tuple, cmp, -1);
- incrementGlobalNsn();
- ctx.leafFrame.setPageLsn(getGlobalNsn());
- }
- ctx.splitKey.reset();
- break;
- }
-
- case INSUFFICIENT_SPACE: {
- int rightPageId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage rightNode = bufferCache
- .pin(BufferedFileHandle.getDiskPageId(fileId, rightPageId),
- true);
- incrementPins();
- rightNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
-
- try {
- IRTreeFrame rightFrame;
- int ret;
- numOfPages++; // debug
- if (!isLeaf) {
- splitsByLevel[ctx.interiorFrame.getLevel()]++; // debug
- rightFrame = (IRTreeFrame) interiorFrameFactory
- .createFrame();
- rightFrame.setPage(rightNode);
- rightFrame.initBuffer((byte) ctx.interiorFrame.getLevel());
- rightFrame.setPageTupleFieldCount(cmp.getKeyFieldCount());
- ret = ctx.interiorFrame.split(rightFrame, tuple, cmp,
- ctx.splitKey);
- ctx.interiorFrame.setRightPage(rightPageId);
- rightFrame.setPageNsn(ctx.interiorFrame.getPageNsn());
- incrementGlobalNsn();
- int newNsn = getGlobalNsn();
- rightFrame.setPageLsn(newNsn);
- ctx.interiorFrame.setPageNsn(newNsn);
- ctx.interiorFrame.setPageLsn(newNsn);
- } else {
- splitsByLevel[0]++; // debug
- rightFrame = (IRTreeFrame) leafFrameFactory.createFrame();
- rightFrame.setPage(rightNode);
- rightFrame.initBuffer((byte) 0);
- rightFrame.setPageTupleFieldCount(cmp.getFieldCount());
- ret = ctx.leafFrame.split(rightFrame, tuple, cmp,
- ctx.splitKey);
- ctx.leafFrame.setRightPage(rightPageId);
- rightFrame.setPageNsn(ctx.leafFrame.getPageNsn());
- incrementGlobalNsn();
- int newNsn = getGlobalNsn();
- rightFrame.setPageLsn(newNsn);
- ctx.leafFrame.setPageNsn(newNsn);
- ctx.leafFrame.setPageLsn(newNsn);
- }
- if (ret != 0) {
- ctx.splitKey.reset();
- } else {
- ctx.splitKey.setPages(pageId, rightPageId);
- }
- if (pageId == rootPage) {
- rootSplits++; // debug
- splitsByLevel[currentLevel]++;
- currentLevel++;
-
- int newLeftId = freePageManager.getFreePage(ctx.metaFrame);
- ICachedPage newLeftNode = bufferCache
- .pin(BufferedFileHandle.getDiskPageId(fileId,
- newLeftId), true);
- incrementPins();
- newLeftNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- try {
- // copy left child to new left child
- System.arraycopy(node.getBuffer().array(), 0,
- newLeftNode.getBuffer().array(), 0, newLeftNode
- .getBuffer().capacity());
-
- // initialize new root (leftNode becomes new root)
- ctx.interiorFrame.setPage(node);
- ctx.interiorFrame.initBuffer((byte) (ctx.interiorFrame
- .getLevel() + 1));
-
- ctx.splitKey.setLeftPage(newLeftId);
-
- ctx.interiorFrame.insert(ctx.splitKey.getLeftTuple(),
- cmp, -1);
- ctx.interiorFrame.insert(ctx.splitKey.getRightTuple(),
- cmp, -1);
-
- incrementGlobalNsn();
- int newNsn = getGlobalNsn();
- ctx.interiorFrame.setPageLsn(newNsn);
- ctx.interiorFrame.setPageNsn(newNsn);
- } finally {
- newLeftNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(newLeftNode);
- incrementUnpins();
- }
-
- ctx.splitKey.reset();
- }
- } finally {
- rightNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(rightNode);
- incrementUnpins();
- }
- break;
- }
- }
- }
-
- public void updateParentForInsert(RTreeOpContext ctx) throws Exception {
- int parentId = ctx.pathList.getLastPageId();
- ICachedPage parentNode = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, parentId), false);
- incrementPins();
- parentNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(parentNode);
- boolean foundParent = true;
-
- if (ctx.interiorFrame.getPageLsn() != ctx.pathList.getLastPageLsn()) {
- foundParent = false;
- while (true) {
- if (ctx.interiorFrame.findTupleByPointer(
- ctx.splitKey.getLeftTuple(), cmp) != -1) {
- // found the parent
- foundParent = true;
- break;
- }
- int rightPage = ctx.interiorFrame.getRightPage();
- parentNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(parentNode);
- incrementUnpins();
-
- if (rightPage == -1) {
- break;
- }
-
- parentId = rightPage;
- parentNode = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, parentId),
- false);
- incrementPins();
- parentNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(parentNode);
- }
- }
- if (foundParent) {
- ctx.interiorFrame.adjustKey(ctx.splitKey.getLeftTuple(), -1, cmp);
- insertTuple(parentNode, parentId, ctx.splitKey.getRightTuple(),
- ctx, ctx.interiorFrame.isLeaf());
- ctx.pathList.moveLast();
-
- parentNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(parentNode);
- incrementUnpins();
- return;
- }
-
- // very rare situation when the there is a root split, do an exhaustive
- // breadth-first traversal looking for the parent tuple
-
- ctx.pathList.clear();
- ctx.traverseList.clear();
- findPath(ctx);
- updateParentForInsert(ctx);
- }
-
- public void findPath(RTreeOpContext ctx) throws Exception {
- int pageId = rootPage;
- int parentIndex = -1;
- int parentLsn = 0;
- int pageLsn, pageIndex;
- ctx.traverseList.add(pageId, -1, parentIndex);
- while (!ctx.traverseList.isLast()) {
- pageId = ctx.traverseList.getFirstPageId();
- parentIndex = ctx.traverseList.getFirstPageIndex();
-
- ICachedPage node = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
- node.acquireReadLatch();
- incrementReadLatchesAcquired();
- ctx.interiorFrame.setPage(node);
- pageLsn = ctx.interiorFrame.getPageLsn();
- pageIndex = ctx.traverseList.first();
- ctx.traverseList.setPageLsn(pageIndex, pageLsn);
-
- ctx.traverseList.moveFirst();
-
- if (pageId != rootPage
- && parentLsn < ctx.interiorFrame.getPageNsn()) {
- int rightPage = ctx.interiorFrame.getRightPage();
- if (rightPage != -1) {
- ctx.traverseList.add(rightPage, -1, parentIndex);
- }
- }
- parentLsn = pageLsn;
-
- if (ctx.interiorFrame.findTupleByPointer(
- ctx.splitKey.getLeftTuple(), ctx.traverseList, pageIndex,
- cmp) != -1) {
- fillPath(ctx, pageIndex);
-
- node.releaseReadLatch();
- incrementReadLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- return;
- }
- node.releaseReadLatch();
- incrementReadLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- }
- }
-
- public void fillPath(RTreeOpContext ctx, int pageIndex) throws Exception {
- if (pageIndex != -1) {
- fillPath(ctx, ctx.traverseList.getPageIndex(pageIndex));
- ctx.pathList.add(ctx.traverseList.getPageId(pageIndex),
- ctx.traverseList.getPageLsn(pageIndex), -1);
- }
- }
-
- @Override
- public void delete(ITupleReference tuple, IndexOpContext ictx)
- throws Exception {
- RTreeOpContext ctx = (RTreeOpContext) ictx;
- ctx.reset();
- ctx.setTuple(tuple);
- ctx.splitKey.reset();
- ctx.splitKey.getLeftTuple().setFieldCount(cmp.getKeyFieldCount());
- ctx.interiorFrame.setPageTupleFieldCount(cmp.getKeyFieldCount());
- ctx.leafFrame.setPageTupleFieldCount(cmp.getFieldCount());
-
- int tupleIndex = findTupleToDelete(ctx);
-
- if (tupleIndex != -1) {
- int pageId = ctx.pathList.getLastPageId();
- ctx.pathList.moveLast();
- deleteTuple(pageId, tupleIndex, ctx);
-
- while (true) {
- if (ctx.splitKey.getLeftPageBuffer() != null) {
- updateParentForDelete(ctx);
- } else {
- break;
- }
- }
-
- ctx.leafFrame.getPage().releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(ctx.leafFrame.getPage());
- incrementUnpins();
- }
- }
-
- public void updateParentForDelete(RTreeOpContext ctx) throws Exception {
- int parentId = ctx.pathList.getLastPageId();
- ICachedPage parentNode = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, parentId), false);
- incrementPins();
- parentNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(parentNode);
- boolean foundParent = true;
- int tupleIndex = -1;
-
- if (ctx.interiorFrame.getPageLsn() != ctx.pathList.getLastPageLsn()) {
- foundParent = false;
- while (true) {
- tupleIndex = ctx.interiorFrame.findTupleByPointer(
- ctx.splitKey.getLeftTuple(), cmp);
- if (tupleIndex != -1) {
- // found the parent
- foundParent = true;
- break;
- }
- int rightPage = ctx.interiorFrame.getRightPage();
- parentNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(parentNode);
- incrementUnpins();
-
- if (rightPage == -1) {
- break;
- }
-
- parentId = rightPage;
- parentNode = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, parentId),
- false);
- incrementPins();
- parentNode.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- ctx.interiorFrame.setPage(parentNode);
- }
- }
- if (foundParent) {
- if (tupleIndex == -1) {
- tupleIndex = ctx.interiorFrame.findTupleByPointer(
- ctx.splitKey.getLeftTuple(), cmp);
- }
- boolean recomputeMBR = ctx.interiorFrame.recomputeMBR(
- ctx.splitKey.getLeftTuple(), tupleIndex, cmp);
-
- if (recomputeMBR) {
- ctx.interiorFrame.adjustKey(ctx.splitKey.getLeftTuple(),
- tupleIndex, cmp);
- ctx.pathList.moveLast();
-
- incrementGlobalNsn();
- ctx.interiorFrame.setPageLsn(getGlobalNsn());
-
- ctx.splitKey.reset();
- if (!ctx.pathList.isEmpty()) {
- ctx.interiorFrame.computeMBR(ctx.splitKey, cmp);
- ctx.splitKey.setLeftPage(parentId);
- }
- } else {
- ctx.pathList.moveLast();
- ctx.splitKey.reset();
- }
-
- parentNode.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(parentNode);
- incrementUnpins();
- return;
- }
-
- // very rare situation when the there is a root split, do an exhaustive
- // breadth-first traversal looking for the parent tuple
-
- ctx.pathList.clear();
- ctx.traverseList.clear();
- findPath(ctx);
- updateParentForDelete(ctx);
- }
-
- public int findTupleToDelete(RTreeOpContext ctx) throws Exception {
-
- ctx.traverseList.add(rootPage, -1, -1);
- ctx.pathList.add(rootPage, -1, ctx.traverseList.size() - 1);
-
- while (!ctx.pathList.isEmpty()) {
- int pageId = ctx.pathList.getLastPageId();
- int parentLsn = ctx.pathList.getLastPageLsn();
- int pageIndex = ctx.pathList.getLastPageIndex();
- ctx.pathList.moveLast();
- ICachedPage node = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- incrementPins();
- node.acquireReadLatch();
- incrementReadLatchesAcquired();
- ctx.interiorFrame.setPage(node);
- boolean isLeaf = ctx.interiorFrame.isLeaf();
- int pageLsn = ctx.interiorFrame.getPageLsn();
- int parentIndex = ctx.traverseList.getPageIndex(pageIndex);
- ctx.traverseList.setPageLsn(pageIndex, pageLsn);
-
- if (pageId != rootPage
- && parentLsn < ctx.interiorFrame.getPageNsn()) {
- // Concurrent split detected, we need to visit the right page
- int rightPage = ctx.interiorFrame.getRightPage();
- if (rightPage != -1) {
- ctx.traverseList.add(rightPage, -1, parentIndex);
- ctx.pathList.add(rightPage, parentLsn,
- ctx.traverseList.size() - 1);
- }
- }
-
- if (!isLeaf) {
- for (int i = 0; i < ctx.interiorFrame.getTupleCount(); i++) {
- int childPageId = ctx.interiorFrame
- .getChildPageIdIfIntersect(ctx.tuple, i, cmp);
- if (childPageId != -1) {
- ctx.traverseList.add(childPageId, -1, pageIndex);
- ctx.pathList.add(childPageId, pageLsn,
- ctx.traverseList.size() - 1);
- }
- }
- } else {
- ctx.leafFrame.setPage(node);
- int tupleIndex = ctx.leafFrame.findTupleIndex(ctx.tuple, cmp);
- if (tupleIndex != -1) {
-
- node.releaseReadLatch();
- incrementReadLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
-
- node = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId),
- false);
- incrementPins();
- node.acquireWriteLatch();
- incrementWriteLatchesAcquired();
- ctx.leafFrame.setPage(node);
-
- if (ctx.leafFrame.getPageLsn() != pageLsn) {
- // The page was changed while we unlocked it
-
- tupleIndex = ctx.leafFrame.findTupleIndex(ctx.tuple,
- cmp);
- if (tupleIndex == -1) {
- ctx.traverseList.add(pageId, -1, parentIndex);
- ctx.pathList.add(pageId, parentLsn,
- ctx.traverseList.size() - 1);
-
- node.releaseWriteLatch();
- incrementWriteLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- continue;
- } else {
- ctx.pathList.clear();
- fillPath(ctx, pageIndex);
- return tupleIndex;
- }
- } else {
- ctx.pathList.clear();
- fillPath(ctx, pageIndex);
- return tupleIndex;
- }
- }
- }
- node.releaseReadLatch();
- incrementReadLatchesReleased();
- bufferCache.unpin(node);
- incrementUnpins();
- }
- return -1;
- }
-
- public void deleteTuple(int pageId, int tupleIndex, RTreeOpContext ctx)
- throws Exception {
- ctx.leafFrame.delete(tupleIndex, cmp);
- incrementGlobalNsn();
- ctx.leafFrame.setPageLsn(getGlobalNsn());
-
- // if the page is empty, just leave it there for future inserts
- if (pageId != rootPage && ctx.leafFrame.getTupleCount() > 0) {
- ctx.leafFrame.computeMBR(ctx.splitKey, cmp);
- ctx.splitKey.setLeftPage(pageId);
- }
- }
-
- public void search(ITreeIndexCursor cursor, SearchPredicate pred,
- RTreeOpContext ctx) throws Exception {
- ctx.reset();
- ctx.cursor = cursor;
-
- cursor.setBufferCache(bufferCache);
- cursor.setFileId(fileId);
- ctx.cursorInitialState.setRootPage(rootPage);
- ctx.cursor.open(ctx.cursorInitialState, pred);
- }
-
- public ITreeIndexFrameFactory getInteriorFrameFactory() {
- return interiorFrameFactory;
- }
-
- public ITreeIndexFrameFactory getLeafFrameFactory() {
- return leafFrameFactory;
- }
-
- public MultiComparator getCmp() {
- return cmp;
- }
-
- public IFreePageManager getFreePageManager() {
- return freePageManager;
- }
-
- @Override
- public void update(ITupleReference tuple, IndexOpContext ictx)
- throws Exception {
- throw new Exception("RTree Update not implemented.");
- }
-
- public final class BulkLoadContext implements IIndexBulkLoadContext {
-
- public RTreeOpContext insertOpCtx;
-
- public BulkLoadContext(float fillFactor, IRTreeFrame leafFrame,
- IRTreeFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame)
- throws HyracksDataException {
-
- insertOpCtx = createOpContext(IndexOp.INSERT, leafFrame,
- interiorFrame, metaFrame);
- }
- }
-
- @Override
- public IIndexBulkLoadContext beginBulkLoad(float fillFactor,
- ITreeIndexFrame leafFrame, ITreeIndexFrame interiorFrame,
- ITreeIndexMetaDataFrame metaFrame) throws HyracksDataException {
- if (loaded)
- throw new HyracksDataException(
- "Trying to bulk-load RTree but RTree has already been loaded.");
-
- BulkLoadContext ctx = new BulkLoadContext(fillFactor,
- (IRTreeFrame) leafFrame, (IRTreeFrame) interiorFrame, metaFrame);
- return ctx;
- }
-
- @Override
- public void bulkLoadAddTuple(IIndexBulkLoadContext ictx,
- ITupleReference tuple) throws HyracksDataException {
- try {
- insert(tuple, ((BulkLoadContext) ictx).insertOpCtx);
- } catch (Exception e) {
- throw new HyracksDataException("BulkLoad Error");
- }
- }
-
- @Override
- public void endBulkLoad(IIndexBulkLoadContext ictx)
- throws HyracksDataException {
- loaded = true;
- }
-
- @Override
- public void diskOrderScan(ITreeIndexCursor icursor,
- ITreeIndexFrame leafFrame, ITreeIndexMetaDataFrame metaFrame,
- IndexOpContext ictx) throws HyracksDataException {
- TreeDiskOrderScanCursor cursor = (TreeDiskOrderScanCursor) icursor;
- RTreeOpContext ctx = (RTreeOpContext) ictx;
- ctx.reset();
-
- int currentPageId = rootPage + 1;
- int maxPageId = freePageManager.getMaxPage(metaFrame);
-
- ICachedPage page = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, currentPageId), false);
- page.acquireReadLatch();
- cursor.setBufferCache(bufferCache);
- cursor.setFileId(fileId);
- cursor.setCurrentPageId(currentPageId);
- cursor.setMaxPageId(maxPageId);
- ctx.cursorInitialState.setPage(page);
- cursor.open(ctx.cursorInitialState, diskOrderScanPredicate);
- }
-
- @Override
- public int getRootPageId() {
- return rootPage;
- }
-
- @Override
- public int getFieldCount() {
- return cmp.getFieldCount();
- }
-
- @Override
- public IndexType getIndexType() {
- return IndexType.RTREE;
- }
+ @Override
+ public void diskOrderScan(ITreeIndexCursor cursor) throws HyracksDataException {
+ ctx.reset(IndexOp.DISKORDERSCAN);
+ rtree.diskOrderScan(cursor, ctx);
+ }
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
index fc57019..c258377 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeOpContext.java
@@ -16,46 +16,34 @@
package edu.uci.ics.hyracks.storage.am.rtree.impls;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOpContext;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
-public final class RTreeOpContext implements IndexOpContext {
- public final IndexOp op;
+public final class RTreeOpContext implements IIndexOpContext {
public final IRTreeInteriorFrame interiorFrame;
public final IRTreeLeafFrame leafFrame;
+ public IndexOp op;
public ITreeIndexCursor cursor;
public RTreeCursorInitialState cursorInitialState;
- public final ITreeIndexMetaDataFrame metaFrame;
- public final RTreeSplitKey splitKey;
+ public ITreeIndexMetaDataFrame metaFrame;
+ public RTreeSplitKey splitKey;
public ITupleReference tuple;
- public final PathList pathList; // used to record the pageIds and pageLsns
- // of the visited pages
- public final PathList traverseList; // used for traversing the tree
+ public PathList pathList; // used to record the pageIds and pageLsns
+ // of the visited pages
+ public PathList traverseList; // used for traversing the tree
private static final int initTraverseListSize = 100;
- public RTreeOpContext(IndexOp op, IRTreeLeafFrame leafFrame,
+ public RTreeOpContext(IRTreeLeafFrame leafFrame,
IRTreeInteriorFrame interiorFrame,
ITreeIndexMetaDataFrame metaFrame, int treeHeightHint) {
- this.op = op;
this.interiorFrame = interiorFrame;
this.leafFrame = leafFrame;
this.metaFrame = metaFrame;
pathList = new PathList(treeHeightHint, treeHeightHint);
- if (op != IndexOp.SEARCH && op != IndexOp.DISKORDERSCAN) {
- splitKey = new RTreeSplitKey(interiorFrame.getTupleWriter()
- .createTupleReference(), interiorFrame.getTupleWriter()
- .createTupleReference());
- traverseList = new PathList(initTraverseListSize,
- initTraverseListSize);
- } else {
- splitKey = null;
- traverseList = null;
- cursorInitialState = new RTreeCursorInitialState(pathList, 1);
- }
}
public ITupleReference getTuple() {
@@ -74,4 +62,26 @@
traverseList.clear();
}
}
+
+ @Override
+ public void reset(IndexOp newOp) {
+ if (op != null && newOp == op) {
+ return;
+ }
+ if (op != IndexOp.SEARCH && op != IndexOp.DISKORDERSCAN) {
+ if (splitKey == null) {
+ splitKey = new RTreeSplitKey(interiorFrame.getTupleWriter()
+ .createTupleReference(), interiorFrame.getTupleWriter()
+ .createTupleReference());
+ }
+ if (traverseList == null) {
+ traverseList = new PathList(initTraverseListSize,
+ initTraverseListSize);
+ }
+ }
+ if (cursorInitialState == null) {
+ cursorInitialState = new RTreeCursorInitialState(pathList, 1);
+ }
+ this.op = newOp;
+ }
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
index 9d37d86..e65f1d4 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/RTreeSearchCursor.java
@@ -30,189 +30,198 @@
public class RTreeSearchCursor implements ITreeIndexCursor {
- private int fileId = -1;
- private ICachedPage page = null;
- private IRTreeInteriorFrame interiorFrame = null;
- private IRTreeLeafFrame leafFrame = null;
- private IBufferCache bufferCache = null;
+ private int fileId = -1;
+ private ICachedPage page = null;
+ private IRTreeInteriorFrame interiorFrame = null;
+ private IRTreeLeafFrame leafFrame = null;
+ private IBufferCache bufferCache = null;
- private SearchPredicate pred;
- private PathList pathList;
- private int rootPage;
- ITupleReference searchKey;
+ private SearchPredicate pred;
+ private PathList pathList;
+ private int rootPage;
+ ITupleReference searchKey;
- private int tupleIndex = 0;
- private int tupleIndexInc = 0;
+ private int tupleIndex = 0;
+ private int tupleIndexInc = 0;
- private MultiComparator cmp;
+ private MultiComparator cmp;
- private ITreeIndexTupleReference frameTuple;
- private boolean readLatched = false;
+ private ITreeIndexTupleReference frameTuple;
+ private boolean readLatched = false;
- private int pin = 0;
- private int unpin = 0;
+ private int pin = 0;
+ private int unpin = 0;
- public RTreeSearchCursor(IRTreeInteriorFrame interiorFrame,
- IRTreeLeafFrame leafFrame) {
- this.interiorFrame = interiorFrame;
- this.leafFrame = leafFrame;
- this.frameTuple = leafFrame.createTupleReference();
- }
+ public RTreeSearchCursor(IRTreeInteriorFrame interiorFrame, IRTreeLeafFrame leafFrame) {
+ this.interiorFrame = interiorFrame;
+ this.leafFrame = leafFrame;
+ this.frameTuple = leafFrame.createTupleReference();
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (readLatched) {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ readLatched = false;
+ }
+ tupleIndex = 0;
+ tupleIndexInc = 0;
+ page = null;
+ pathList = null;
+ }
+
+ public ITupleReference getTuple() {
+ return frameTuple;
+ }
+
+ @Override
+ public ICachedPage getPage() {
+ return page;
+ }
+
+ public boolean fetchNextLeafPage() throws HyracksDataException {
+ boolean succeed = false;
+ if (readLatched) {
+ page.releaseReadLatch();
+ bufferCache.unpin(page);
+ unpin++;
+ readLatched = false;
+ }
+
+ while (!pathList.isEmpty()) {
+ int pageId = pathList.getLastPageId();
+ long parentLsn = pathList.getLastPageLsn();
+ pathList.moveLast();
+ ICachedPage node = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
+ pin++;
+ node.acquireReadLatch();
+ readLatched = true;
+ try {
+ interiorFrame.setPage(node);
+ boolean isLeaf = interiorFrame.isLeaf();
+ long pageLsn = interiorFrame.getPageLsn();
+
+ if (pageId != rootPage && parentLsn < interiorFrame.getPageNsn()) {
+ // Concurrent split detected, we need to visit the right
+ // page
+ int rightPage = interiorFrame.getRightPage();
+ if (rightPage != -1) {
+ pathList.add(rightPage, parentLsn, -1);
+ }
+ }
+
+ if (!isLeaf) {
+ for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
+ int childPageId = interiorFrame.getChildPageIdIfIntersect(searchKey, i, cmp);
+ if (childPageId != -1) {
+ pathList.add(childPageId, pageLsn, -1);
+ }
+ }
+ } else {
+ page = node;
+ leafFrame.setPage(page);
+ tupleIndex = 0;
+ succeed = true;
+ return true;
+ }
+ } finally {
+ if (!succeed) {
+ if (readLatched) {
+ node.releaseReadLatch();
+ readLatched = false;
+ bufferCache.unpin(node);
+ unpin++;
+ }
+ }
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasNext() throws Exception {
+ if (page == null) {
+ return false;
+ }
+
+ if (tupleIndex == leafFrame.getTupleCount()) {
+ if (!fetchNextLeafPage()) {
+ return false;
+ }
+ }
+
+ do {
+ for (int i = tupleIndex; i < leafFrame.getTupleCount(); i++) {
+ if (leafFrame.intersect(searchKey, i, cmp)) {
+ frameTuple.resetByTupleIndex(leafFrame, i);
+ tupleIndexInc = i + 1;
+ return true;
+ }
+ }
+ } while (fetchNextLeafPage());
+ return false;
+ }
+
+ @Override
+ public void next() throws Exception {
+ tupleIndex = tupleIndexInc;
+ }
+
+ @Override
+ public void open(ICursorInitialState initialState, ISearchPredicate searchPred) throws HyracksDataException {
+ // in case open is called multiple times without closing
+ if (this.page != null) {
+ this.page.releaseReadLatch();
+ readLatched = false;
+ bufferCache.unpin(this.page);
+ pathList.clear();
+ }
+
+ pathList = ((RTreeCursorInitialState) initialState).getPathList();
+ rootPage = ((RTreeCursorInitialState) initialState).getRootPage();
+
+ pred = (SearchPredicate) searchPred;
+ cmp = pred.getLowKeyComparator();
+ searchKey = pred.getSearchKey();
+
+ int maxFieldPos = cmp.getKeyFieldCount() / 2;
+ for (int i = 0; i < maxFieldPos; i++) {
+ int j = maxFieldPos + i;
+ int c = cmp.getComparators()[i].compare(searchKey.getFieldData(i), searchKey.getFieldStart(i),
+ searchKey.getFieldLength(i), searchKey.getFieldData(j), searchKey.getFieldStart(j),
+ searchKey.getFieldLength(j));
+ if (c > 0) {
+ throw new IllegalArgumentException("The low key point has larger coordinates than the high key point.");
+ }
+ }
+
+ pathList.add(this.rootPage, -1, -1);
+ tupleIndex = 0;
+ fetchNextLeafPage();
+ }
+
+ @Override
+ public void reset() {
+ try {
+ close();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public void setBufferCache(IBufferCache bufferCache) {
+ this.bufferCache = bufferCache;
+ }
+
+ @Override
+ public void setFileId(int fileId) {
+ this.fileId = fileId;
+ }
@Override
- public void close() throws Exception {
- if (readLatched) {
- page.releaseReadLatch();
- bufferCache.unpin(page);
- readLatched = false;
- }
- tupleIndex = 0;
- tupleIndexInc = 0;
- page = null;
- pathList = null;
- }
-
- public ITupleReference getTuple() {
- return frameTuple;
- }
-
- @Override
- public ICachedPage getPage() {
- return page;
- }
-
- public boolean fetchNextLeafPage() throws HyracksDataException {
- if (readLatched) {
- page.releaseReadLatch();
- bufferCache.unpin(page);
- unpin++;
- readLatched = false;
- }
- while (!pathList.isEmpty()) {
- int pageId = pathList.getLastPageId();
- int parentLsn = pathList.getLastPageLsn();
- pathList.moveLast();
- ICachedPage node = bufferCache.pin(
- BufferedFileHandle.getDiskPageId(fileId, pageId), false);
- pin++;
- node.acquireReadLatch();
- readLatched = true;
- interiorFrame.setPage(node);
- boolean isLeaf = interiorFrame.isLeaf();
- int pageLsn = interiorFrame.getPageLsn();
-
- if (pageId != rootPage && parentLsn < interiorFrame.getPageNsn()) {
- // Concurrent split detected, we need to visit the right page
- int rightPage = interiorFrame.getRightPage();
- if (rightPage != -1) {
- pathList.add(rightPage, parentLsn, -1);
- }
- }
-
- if (!isLeaf) {
- for (int i = 0; i < interiorFrame.getTupleCount(); i++) {
- int childPageId = interiorFrame.getChildPageIdIfIntersect(
- searchKey, i, cmp);
- if (childPageId != -1) {
- pathList.add(childPageId, pageLsn, -1);
- }
- }
- } else {
- page = node;
- leafFrame.setPage(page);
- tupleIndex = 0;
- return true;
- }
- node.releaseReadLatch();
- readLatched = false;
- bufferCache.unpin(node);
- unpin++;
- }
+ public boolean exclusiveLatchNodes() {
return false;
}
-
- @Override
- public boolean hasNext() throws Exception {
- if (page == null) {
- return false;
- }
-
- if (tupleIndex == leafFrame.getTupleCount()) {
- if (!fetchNextLeafPage()) {
- return false;
- }
- }
-
- do {
- for (int i = tupleIndex; i < leafFrame.getTupleCount(); i++) {
- if (leafFrame.intersect(searchKey, i, cmp)) {
- frameTuple.resetByTupleIndex(leafFrame, i);
- tupleIndexInc = i + 1;
- return true;
- }
- }
- } while (fetchNextLeafPage());
- return false;
- }
-
- @Override
- public void next() throws Exception {
- tupleIndex = tupleIndexInc;
- }
-
- @Override
- public void open(ICursorInitialState initialState,
- ISearchPredicate searchPred) throws Exception {
- // in case open is called multiple times without closing
- if (this.page != null) {
- this.page.releaseReadLatch();
- readLatched = false;
- bufferCache.unpin(this.page);
- pathList.clear();
- }
-
- pathList = ((RTreeCursorInitialState) initialState).getPathList();
- rootPage = ((RTreeCursorInitialState) initialState).getRootPage();
-
- pred = (SearchPredicate) searchPred;
- cmp = pred.getLowKeyComparator();
- searchKey = pred.getSearchKey();
-
- int maxFieldPos = cmp.getKeyFieldCount() / 2;
- for (int i = 0; i < maxFieldPos; i++) {
- int j = maxFieldPos + i;
- int c = cmp.getComparators()[i].compare(searchKey.getFieldData(i),
- searchKey.getFieldStart(i), searchKey.getFieldLength(i),
- searchKey.getFieldData(j), searchKey.getFieldStart(j),
- searchKey.getFieldLength(j));
- if (c > 0) {
- throw new IllegalArgumentException(
- "The low key point has larger coordinates than the high key point.");
- }
- }
-
- pathList.add(this.rootPage, -1, -1);
- frameTuple.setFieldCount(cmp.getFieldCount());
- tupleIndex = 0;
- fetchNextLeafPage();
- }
-
- @Override
- public void reset() {
- try {
- close();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- @Override
- public void setBufferCache(IBufferCache bufferCache) {
- this.bufferCache = bufferCache;
- }
-
- @Override
- public void setFileId(int fileId) {
- this.fileId = fileId;
- }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/Rectangle.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/Rectangle.java
index 019760b..cb9b160 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/Rectangle.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/Rectangle.java
@@ -16,7 +16,7 @@
package edu.uci.ics.hyracks.storage.am.rtree.impls;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
public class Rectangle {
private int dim;
@@ -49,26 +49,26 @@
high[i] = value;
}
- public void set(ITupleReference tuple, MultiComparator cmp) {
+ public void set(ITupleReference tuple, IPrimitiveValueProvider[] valueProviders) {
for (int i = 0; i < getDim(); i++) {
int j = i + getDim();
- setLow(i, cmp.getValueProviders()[i].getValue(
+ setLow(i, valueProviders[i].getValue(
tuple.getFieldData(i), tuple.getFieldStart(i)));
- setHigh(i, cmp.getValueProviders()[j].getValue(
+ setHigh(i, valueProviders[j].getValue(
tuple.getFieldData(j), tuple.getFieldStart(j)));
}
}
- public void enlarge(ITupleReference tupleToBeInserted, MultiComparator cmp) {
+ public void enlarge(ITupleReference tupleToBeInserted, IPrimitiveValueProvider[] valueProviders) {
for (int i = 0; i < getDim(); i++) {
int j = getDim() + i;
- double low = cmp.getValueProviders()[i].getValue(
+ double low = valueProviders[i].getValue(
tupleToBeInserted.getFieldData(i),
tupleToBeInserted.getFieldStart(i));
if (getLow(i) > low) {
setLow(i, low);
}
- double high = cmp.getValueProviders()[j].getValue(
+ double high = valueProviders[j].getValue(
tupleToBeInserted.getFieldData(j),
tupleToBeInserted.getFieldStart(j));
if (getHigh(i) < high) {
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/UnorderedSlotManager.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/UnorderedSlotManager.java
index 7badb8e..260cdea 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/UnorderedSlotManager.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/impls/UnorderedSlotManager.java
@@ -24,6 +24,7 @@
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMFrame;
public class UnorderedSlotManager extends AbstractSlotManager {
+
@Override
public int findTupleIndex(ITupleReference searchKey,
ITreeIndexTupleReference frameTuple, MultiComparator multiCmp,
@@ -58,7 +59,7 @@
break;
}
}
- int remainingFieldCount = multiCmp.getFieldCount()
+ int remainingFieldCount = frameTuple.getFieldCount()
- multiCmp.getKeyFieldCount();
for (int j = multiCmp.getKeyFieldCount(); j < multiCmp
.getKeyFieldCount() + remainingFieldCount; j++) {
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/tuples/RTreeTypeAwareTupleWriter.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/tuples/RTreeTypeAwareTupleWriter.java
index 6acac7e..898ebf8 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/tuples/RTreeTypeAwareTupleWriter.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/tuples/RTreeTypeAwareTupleWriter.java
@@ -17,42 +17,41 @@
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
public class RTreeTypeAwareTupleWriter extends TypeAwareTupleWriter {
- public RTreeTypeAwareTupleWriter(ITypeTrait[] typeTraits) {
- super(typeTraits);
- }
+ public RTreeTypeAwareTupleWriter(ITypeTraits[] typeTraits) {
+ super(typeTraits);
+ }
- public int writeTupleFields(ITreeIndexTupleReference[] refs,
- int startField, ByteBuffer targetBuf, int targetOff) {
- int runner = targetOff;
- int nullFlagsBytes = getNullFlagsBytes(refs.length);
- // write null indicator bits
- for (int i = 0; i < nullFlagsBytes; i++) {
- targetBuf.put(runner++, (byte) 0);
- }
+ public int writeTupleFields(ITreeIndexTupleReference[] refs, int startField, ByteBuffer targetBuf, int targetOff) {
+ int runner = targetOff;
+ int nullFlagsBytes = getNullFlagsBytes(refs.length);
+ // write null indicator bits
+ for (int i = 0; i < nullFlagsBytes; i++) {
+ targetBuf.put(runner++, (byte) 0);
+ }
- // write field slots for variable length fields
- // since the r-tree has fixed length keys, we don't actually need this?
- encDec.reset(targetBuf.array(), runner);
- for (int i = startField; i < startField + refs.length; i++) {
- if (typeTraits[i].getStaticallyKnownDataLength() == ITypeTrait.VARIABLE_LENGTH) {
- encDec.encode(refs[i].getFieldLength(i));
- }
- }
- runner = encDec.getPos();
+ // write field slots for variable length fields
+ // since the r-tree has fixed length keys, we don't actually need this?
+ encDec.reset(targetBuf.array(), runner);
+ for (int i = startField; i < startField + refs.length; i++) {
+ if (!typeTraits[i].isFixedLength()) {
+ encDec.encode(refs[i].getFieldLength(i));
+ }
+ }
+ runner = encDec.getPos();
- // write data
- for (int i = 0; i < refs.length; i++) {
- System.arraycopy(refs[i].getFieldData(i), refs[i].getFieldStart(i),
- targetBuf.array(), runner, refs[i].getFieldLength(i));
- runner += refs[i].getFieldLength(i);
- }
- return runner - targetOff;
+ // write data
+ for (int i = 0; i < refs.length; i++) {
+ System.arraycopy(refs[i].getFieldData(i), refs[i].getFieldStart(i), targetBuf.array(), runner,
+ refs[i].getFieldLength(i));
+ runner += refs[i].getFieldLength(i);
+ }
+ return runner - targetOff;
- }
+ }
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/tuples/RTreeTypeAwareTupleWriterFactory.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/tuples/RTreeTypeAwareTupleWriterFactory.java
index 7d88f2e..24b2b53 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/tuples/RTreeTypeAwareTupleWriterFactory.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/tuples/RTreeTypeAwareTupleWriterFactory.java
@@ -15,23 +15,21 @@
package edu.uci.ics.hyracks.storage.am.rtree.tuples;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-public class RTreeTypeAwareTupleWriterFactory implements
- ITreeIndexTupleWriterFactory {
+public class RTreeTypeAwareTupleWriterFactory implements ITreeIndexTupleWriterFactory {
- private static final long serialVersionUID = 1L;
- private ITypeTrait[] typeTraits;
+ private static final long serialVersionUID = 1L;
+ private ITypeTraits[] typeTraits;
- public RTreeTypeAwareTupleWriterFactory(ITypeTrait[] typeTraits) {
- this.typeTraits = typeTraits;
- }
+ public RTreeTypeAwareTupleWriterFactory(ITypeTraits[] typeTraits) {
+ this.typeTraits = typeTraits;
+ }
- @Override
- public ITreeIndexTupleWriter createTupleWriter() {
- return new RTreeTypeAwareTupleWriter(typeTraits);
- }
-
+ @Override
+ public ITreeIndexTupleWriter createTupleWriter() {
+ return new RTreeTypeAwareTupleWriter(typeTraits);
+ }
}
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
new file mode 100644
index 0000000..74e4840
--- /dev/null
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/util/RTreeUtils.java
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.rtree.util;
+
+import edu.uci.ics.hyracks.data.std.api.IPointableFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.data.PointablePrimitiveValueProviderFactory;
+
+public class RTreeUtils {
+ public static IPrimitiveValueProviderFactory[] createPrimitiveValueProviderFactories(int len, IPointableFactory pf) {
+ IPrimitiveValueProviderFactory[] pvpfs = new IPrimitiveValueProviderFactory[len];
+ for (int i = 0; i < len; ++i) {
+ pvpfs[i] = new PointablePrimitiveValueProviderFactory(pf);
+ }
+ return pvpfs;
+ }
+}
diff --git a/hyracks-storage-common/.classpath b/hyracks-storage-common/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-storage-common/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-storage-common/.project b/hyracks-storage-common/.project
deleted file mode 100644
index d990298..0000000
--- a/hyracks-storage-common/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-storage-common</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- <nature>org.eclipse.jdt.core.javanature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-storage-common/.settings/org.eclipse.jdt.core.prefs b/hyracks-storage-common/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 450f5c4..0000000
--- a/hyracks-storage-common/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:04 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-storage-common/.settings/org.maven.ide.eclipse.prefs b/hyracks-storage-common/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index a91dbc3..0000000
--- a/hyracks-storage-common/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Tue Aug 24 14:59:44 PDT 2010
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-storage-common/pom.xml b/hyracks-storage-common/pom.xml
index ff6887d..50eecd2 100644
--- a/hyracks-storage-common/pom.xml
+++ b/hyracks-storage-common/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,7 +27,7 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-api</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
index d7558ba..a5584e0 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/BufferCache.java
@@ -40,7 +40,7 @@
private static final int MAX_VICTIMIZATION_TRY_COUNT = 3;
private final int maxOpenFiles;
-
+
private final IIOManager ioManager;
private final int pageSize;
private final int numPages;
@@ -54,7 +54,8 @@
private boolean closed;
public BufferCache(IIOManager ioManager, ICacheMemoryAllocator allocator,
- IPageReplacementStrategy pageReplacementStrategy, IFileMapManager fileMapManager, int pageSize, int numPages, int maxOpenFiles) {
+ IPageReplacementStrategy pageReplacementStrategy, IFileMapManager fileMapManager, int pageSize,
+ int numPages, int maxOpenFiles) {
this.ioManager = ioManager;
this.pageSize = pageSize;
this.numPages = numPages;
@@ -91,21 +92,21 @@
if (closed) {
throw new HyracksDataException("pin called on a closed cache");
}
-
+
// check whether file has been created and opened
int fileId = BufferedFileHandle.getFileId(dpid);
BufferedFileHandle fInfo = fileInfoMap.get(fileId);
- if(fInfo == null) {
+ if (fInfo == null) {
throw new HyracksDataException("pin called on a fileId " + fileId + " that has not been created.");
- } else if(fInfo.getReferenceCount() <= 0) {
+ } else if (fInfo.getReferenceCount() <= 0) {
throw new HyracksDataException("pin called on a fileId " + fileId + " that has not been opened.");
}
}
-
- @Override
- public ICachedPage tryPin(long dpid) throws HyracksDataException {
+
+ @Override
+ public ICachedPage tryPin(long dpid) throws HyracksDataException {
pinSanityCheck(dpid);
-
+
CachedPage cPage = null;
int hash = hash(dpid);
CacheBucket bucket = pageMap[hash];
@@ -123,15 +124,21 @@
} finally {
bucket.bucketLock.unlock();
}
-
+
return cPage;
}
-
+
@Override
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
pinSanityCheck(dpid);
-
+
CachedPage cPage = findPage(dpid, newPage);
+ if (cPage == null) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(dumpState());
+ }
+ throw new HyracksDataException("Failed to pin page because all pages are pinned.");
+ }
if (!newPage) {
if (!cPage.valid) {
/*
@@ -547,7 +554,7 @@
}
@Override
- public void close() {
+ public void close() {
closed = true;
synchronized (cleanerThread) {
cleanerThread.shutdownStart = true;
@@ -559,19 +566,22 @@
e.printStackTrace();
}
}
- }
-
+ }
+
synchronized (fileInfoMap) {
try {
- for(Map.Entry<Integer, BufferedFileHandle> entry : fileInfoMap.entrySet()) {
- sweepAndFlush(entry.getKey());
- ioManager.close(entry.getValue().getFileHandle());
+ for (Map.Entry<Integer, BufferedFileHandle> entry : fileInfoMap.entrySet()) {
+ boolean fileHasBeenDeleted = entry.getValue().fileHasBeenDeleted();
+ sweepAndFlush(entry.getKey(), !fileHasBeenDeleted);
+ if (!fileHasBeenDeleted) {
+ ioManager.close(entry.getValue().getFileHandle());
+ }
}
- } catch(HyracksDataException e) {
+ } catch (HyracksDataException e) {
e.printStackTrace();
}
fileInfoMap.clear();
- }
+ }
}
@Override
@@ -593,28 +603,32 @@
BufferedFileHandle fInfo;
fInfo = fileInfoMap.get(fileId);
if (fInfo == null) {
-
+
// map is full, make room by removing cleaning up unreferenced files
boolean unreferencedFileFound = true;
- while(fileInfoMap.size() >= maxOpenFiles && unreferencedFileFound) {
- unreferencedFileFound = false;
- for(Map.Entry<Integer, BufferedFileHandle> entry : fileInfoMap.entrySet()) {
- if(entry.getValue().getReferenceCount() <= 0) {
+ while (fileInfoMap.size() >= maxOpenFiles && unreferencedFileFound) {
+ unreferencedFileFound = false;
+ for (Map.Entry<Integer, BufferedFileHandle> entry : fileInfoMap.entrySet()) {
+ if (entry.getValue().getReferenceCount() <= 0) {
int entryFileId = entry.getKey();
- sweepAndFlush(entryFileId);
+ boolean fileHasBeenDeleted = entry.getValue().fileHasBeenDeleted();
+ sweepAndFlush(entryFileId, !fileHasBeenDeleted);
+ if (!fileHasBeenDeleted) {
+ ioManager.close(entry.getValue().getFileHandle());
+ }
fileInfoMap.remove(entryFileId);
- ioManager.close(entry.getValue().getFileHandle());
unreferencedFileFound = true;
// for-each iterator is invalid because we changed fileInfoMap
break;
}
}
}
-
- if(fileInfoMap.size() >= maxOpenFiles) {
- throw new HyracksDataException("Could not open fileId " + fileId + ". Max number of files " + maxOpenFiles + " already opened and referenced.");
+
+ if (fileInfoMap.size() >= maxOpenFiles) {
+ throw new HyracksDataException("Could not open fileId " + fileId + ". Max number of files "
+ + maxOpenFiles + " already opened and referenced.");
}
-
+
// create, open, and map new file reference
FileReference fileRef = fileMapManager.lookupFileName(fileId);
FileHandle fh = ioManager.open(fileRef, IIOManager.FileReadWriteMode.READ_WRITE,
@@ -625,8 +639,8 @@
fInfo.incReferenceCount();
}
}
-
- private void sweepAndFlush(int fileId) throws HyracksDataException {
+
+ private void sweepAndFlush(int fileId, boolean flushDirtyPages) throws HyracksDataException {
for (int i = 0; i < pageMap.length; ++i) {
CacheBucket bucket = pageMap[i];
bucket.bucketLock.lock();
@@ -637,7 +651,7 @@
if (cPage == null) {
break;
}
- if (invalidateIfFileIdMatch(fileId, cPage)) {
+ if (invalidateIfFileIdMatch(fileId, cPage, flushDirtyPages)) {
prev.next = cPage.next;
cPage.next = null;
} else {
@@ -646,7 +660,7 @@
}
// Take care of the head of the chain.
if (bucket.cachedPage != null) {
- if (invalidateIfFileIdMatch(fileId, bucket.cachedPage)) {
+ if (invalidateIfFileIdMatch(fileId, bucket.cachedPage, flushDirtyPages)) {
CachedPage cPage = bucket.cachedPage;
bucket.cachedPage = bucket.cachedPage.next;
cPage.next = null;
@@ -658,14 +672,20 @@
}
}
- private boolean invalidateIfFileIdMatch(int fileId, CachedPage cPage) throws HyracksDataException {
+ private boolean invalidateIfFileIdMatch(int fileId, CachedPage cPage, boolean flushDirtyPages)
+ throws HyracksDataException {
if (BufferedFileHandle.getFileId(cPage.dpid) == fileId) {
+ int pinCount;
if (cPage.dirty.get()) {
- write(cPage);
+ if (flushDirtyPages) {
+ write(cPage);
+ }
cPage.dirty.set(false);
- cPage.pinCount.decrementAndGet();
+ pinCount = cPage.pinCount.decrementAndGet();
+ } else {
+ pinCount = cPage.pinCount.get();
}
- if (cPage.pinCount.get() != 0) {
+ if (pinCount != 0) {
throw new IllegalStateException("Page is pinned and file is being closed");
}
cPage.invalidate();
@@ -686,7 +706,7 @@
throw new HyracksDataException("Closing unopened file");
}
if (fInfo.decReferenceCount() < 0) {
- throw new HyracksDataException("Closed fileId: " + fileId + " more times than it was opened.");
+ throw new HyracksDataException("Closed fileId: " + fileId + " more times than it was opened.");
}
}
}
@@ -697,11 +717,22 @@
LOGGER.info("Deleting file: " + fileId + " in cache: " + this);
}
synchronized (fileInfoMap) {
- BufferedFileHandle fInfo = fileInfoMap.get(fileId);
- if (fInfo != null) {
- throw new HyracksDataException("Deleting open file");
+ BufferedFileHandle fInfo = null;
+ try {
+ fInfo = fileInfoMap.get(fileId);
+ if (fInfo != null && fInfo.getReferenceCount() > 0) {
+ throw new HyracksDataException("Deleting open file");
+ }
+ } finally {
+ fileMapManager.unregisterFile(fileId);
+ if (fInfo != null) {
+ // Mark the fInfo as deleted,
+ // such that when its pages are reclaimed in openFile(),
+ // the pages are not flushed to disk but only invalidates.
+ ioManager.close(fInfo.getFileHandle());
+ fInfo.markAsDeleted();
+ }
}
- fileMapManager.unregisterFile(fileId);
}
}
}
\ No newline at end of file
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/DebugBufferCache.java
new file mode 100644
index 0000000..dc00df0
--- /dev/null
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/buffercache/DebugBufferCache.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.common.buffercache;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+
+/**
+ * Implementation of an IBufferCache that counts the number of pins/unpins,
+ * latches/unlatches, and file create/delete/open/close called on it. It
+ * delegates the actual functionality to another IBufferCache set in the c'tor.
+ * The counters are updated in a thread-safe fashion using AtomicLong.
+ */
+public class DebugBufferCache implements IBufferCache {
+
+ // Actual BufferCache functionality is delegated to this bufferCache.
+ private final IBufferCache bufferCache;
+ private AtomicLong pinCount;
+ private AtomicLong unpinCount;
+ private AtomicLong readLatchCount;
+ private AtomicLong readUnlatchCount;
+ private AtomicLong writeLatchCount;
+ private AtomicLong writeUnlatchCount;
+ private AtomicLong createFileCount;
+ private AtomicLong deleteFileCount;
+ private AtomicLong openFileCount;
+ private AtomicLong closeFileCount;
+
+ public DebugBufferCache(IBufferCache bufferCache) {
+ this.bufferCache = bufferCache;
+ resetCounters();
+ }
+
+ @Override
+ public void createFile(FileReference fileRef) throws HyracksDataException {
+ bufferCache.createFile(fileRef);
+ createFileCount.addAndGet(1);
+ }
+
+ @Override
+ public void openFile(int fileId) throws HyracksDataException {
+ bufferCache.openFile(fileId);
+ openFileCount.addAndGet(1);
+ }
+
+ @Override
+ public void closeFile(int fileId) throws HyracksDataException {
+ bufferCache.closeFile(fileId);
+ closeFileCount.addAndGet(1);
+ }
+
+ @Override
+ public void deleteFile(int fileId) throws HyracksDataException {
+ bufferCache.deleteFile(fileId);
+ deleteFileCount.addAndGet(1);
+ }
+
+ @Override
+ public ICachedPage tryPin(long dpid) throws HyracksDataException {
+ return bufferCache.tryPin(dpid);
+ }
+
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
+ ICachedPage page = bufferCache.pin(dpid, newPage);
+ pinCount.addAndGet(1);
+ return page;
+ }
+
+ @Override
+ public void unpin(ICachedPage page) throws HyracksDataException {
+ bufferCache.unpin(page);
+ unpinCount.addAndGet(1);
+ }
+
+ @Override
+ public int getPageSize() {
+ return bufferCache.getPageSize();
+ }
+
+ @Override
+ public int getNumPages() {
+ return bufferCache.getNumPages();
+ }
+
+ @Override
+ public void close() {
+ bufferCache.close();
+ }
+
+ public void resetCounters() {
+ pinCount.set(0);
+ unpinCount.set(0);
+ readLatchCount.set(0);
+ readUnlatchCount.set(0);
+ writeLatchCount.set(0);
+ writeUnlatchCount.set(0);
+ createFileCount.set(0);
+ deleteFileCount.set(0);
+ openFileCount.set(0);
+ closeFileCount.set(0);
+ }
+
+ public long getPinCount() {
+ return pinCount.get();
+ }
+
+ public long getUnpinCount() {
+ return unpinCount.get();
+ }
+
+ public long getReadLatchCount() {
+ return readLatchCount.get();
+ }
+
+ public long getReadUnlatchCount() {
+ return readUnlatchCount.get();
+ }
+
+ public long getWriteLatchCount() {
+ return writeLatchCount.get();
+ }
+
+ public long getWriteUnlatchCount() {
+ return writeUnlatchCount.get();
+ }
+
+ public long getCreateFileCount() {
+ return createFileCount.get();
+ }
+
+ public long getDeleteFileCount() {
+ return deleteFileCount.get();
+ }
+
+ public long getOpenFileCount() {
+ return openFileCount.get();
+ }
+
+ public long getCloseFileCount() {
+ return closeFileCount.get();
+ }
+}
diff --git a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/BufferedFileHandle.java b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/BufferedFileHandle.java
index 3137f20..ac062d2 100644
--- a/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/BufferedFileHandle.java
+++ b/hyracks-storage-common/src/main/java/edu/uci/ics/hyracks/storage/common/file/BufferedFileHandle.java
@@ -37,6 +37,14 @@
return handle;
}
+ public void markAsDeleted() {
+ handle = null;
+ }
+
+ public boolean fileHasBeenDeleted() {
+ return handle == null;
+ }
+
public int incReferenceCount() {
return refCount.incrementAndGet();
}
diff --git a/hyracks-test-support/.classpath b/hyracks-test-support/.classpath
deleted file mode 100644
index 1f3c1ff..0000000
--- a/hyracks-test-support/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/classes" path="src/main/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-test-support/.project b/hyracks-test-support/.project
deleted file mode 100644
index b0c27d2..0000000
--- a/hyracks-test-support/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-test-support</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-test-support/.settings/org.eclipse.jdt.core.prefs b/hyracks-test-support/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 375e12e..0000000
--- a/hyracks-test-support/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:07 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-test-support/.settings/org.maven.ide.eclipse.prefs b/hyracks-test-support/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index f462ae6..0000000
--- a/hyracks-test-support/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Wed Jan 05 15:09:30 PST 2011
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-test-support/pom.xml b/hyracks-test-support/pom.xml
index e9a8a27..fdcf7d4 100644
--- a/hyracks-test-support/pom.xml
+++ b/hyracks-test-support/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-test-support</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -27,19 +27,26 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.hyracks</groupId>
+ <artifactId>hyracks-storage-am-invertedindex</artifactId>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestIndexRegistryProvider.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestIndexRegistryProvider.java
new file mode 100644
index 0000000..27d50f5
--- /dev/null
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestIndexRegistryProvider.java
@@ -0,0 +1,29 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.hyracks.test.support;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
+
+public class TestIndexRegistryProvider implements IIndexRegistryProvider<IIndex> {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IndexRegistry<IIndex> getRegistry(IHyracksTaskContext ctx) {
+ return TestStorageManagerComponentHolder.getIndexRegistry(ctx);
+ }
+}
\ No newline at end of file
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestNCApplicationContext.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestNCApplicationContext.java
index 295a428..1b95ccb 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestNCApplicationContext.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestNCApplicationContext.java
@@ -42,7 +42,7 @@
}
@Override
- public Serializable getDestributedState() {
+ public Serializable getDistributedState() {
return distributedState;
}
@@ -52,11 +52,6 @@
}
@Override
- public void setDistributedState(Serializable state) {
- distributedState = state;
- }
-
- @Override
public void setApplicationObject(Object object) {
this.appObject = object;
}
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
index c7bd09d..d006f42 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestStorageManagerComponentHolder.java
@@ -15,7 +15,7 @@
package edu.uci.ics.hyracks.test.support;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndex;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
import edu.uci.ics.hyracks.storage.common.buffercache.BufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
@@ -30,7 +30,7 @@
public class TestStorageManagerComponentHolder {
private static IBufferCache bufferCache;
private static IFileMapProvider fileMapProvider;
- private static IndexRegistry<ITreeIndex> treeIndexRegistry;
+ private static IndexRegistry<IIndex> indexRegistry;
private static int pageSize;
private static int numPages;
@@ -42,7 +42,7 @@
TestStorageManagerComponentHolder.maxOpenFiles = maxOpenFiles;
bufferCache = null;
fileMapProvider = null;
- treeIndexRegistry = null;
+ indexRegistry = null;
}
public synchronized static IBufferCache getBufferCache(IHyracksTaskContext ctx) {
@@ -63,10 +63,10 @@
return fileMapProvider;
}
- public synchronized static IndexRegistry<ITreeIndex> getTreeIndexRegistry(IHyracksTaskContext ctx) {
- if (treeIndexRegistry == null) {
- treeIndexRegistry = new IndexRegistry<ITreeIndex>();
+ public synchronized static IndexRegistry<IIndex> getIndexRegistry(IHyracksTaskContext ctx) {
+ if (indexRegistry == null) {
+ indexRegistry = new IndexRegistry<IIndex>();
}
- return treeIndexRegistry;
+ return indexRegistry;
}
}
\ No newline at end of file
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java
index 83b608f..e1743a1 100644
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java
+++ b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTaskContext.java
@@ -18,8 +18,9 @@
import edu.uci.ics.hyracks.api.context.IHyracksJobletContext;
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.ActivityId;
import edu.uci.ics.hyracks.api.dataflow.TaskAttemptId;
+import edu.uci.ics.hyracks.api.dataflow.TaskId;
+import edu.uci.ics.hyracks.api.dataflow.state.ITaskState;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.exceptions.HyracksException;
import edu.uci.ics.hyracks.api.io.FileReference;
@@ -91,19 +92,12 @@
}
@Override
- public Object lookupGlobalVariable(ActivityId producerActivity, int partition, String varName)
- throws HyracksDataException {
- return null;
+ public void setTaskState(ITaskState taskState) {
+
}
@Override
- public Object lookupLocalVariable(ActivityId producerActivity, int partition, String varName)
- throws HyracksDataException {
+ public ITaskState getTaskState(TaskId taskId) {
return null;
}
-
- @Override
- public void setVariable(String name, Object value) throws HyracksDataException {
-
- }
}
\ No newline at end of file
diff --git a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTreeIndexRegistryProvider.java b/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTreeIndexRegistryProvider.java
deleted file mode 100644
index 9925fe8..0000000
--- a/hyracks-test-support/src/main/java/edu/uci/ics/hyracks/test/support/TestTreeIndexRegistryProvider.java
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.hyracks.test.support;
-
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
-
-public class TestTreeIndexRegistryProvider implements IIndexRegistryProvider<ITreeIndex> {
- private static final long serialVersionUID = 1L;
-
- @Override
- public IndexRegistry<ITreeIndex> getRegistry(IHyracksTaskContext ctx) {
- return TestStorageManagerComponentHolder.getTreeIndexRegistry(ctx);
- }
-}
\ No newline at end of file
diff --git a/hyracks-tests/.project b/hyracks-tests/.project
deleted file mode 100644
index 198463d..0000000
--- a/hyracks-tests/.project
+++ /dev/null
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-tests</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-tests/.settings/org.maven.ide.eclipse.prefs b/hyracks-tests/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 99b89a6..0000000
--- a/hyracks-tests/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Jan 06 11:27:16 PST 2011
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/.classpath b/hyracks-tests/hyracks-storage-am-btree-test/.classpath
deleted file mode 100644
index f2cc5f7..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/.project b/hyracks-tests/hyracks-storage-am-btree-test/.project
deleted file mode 100644
index bc6bc56..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-storage-am-btree-test</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/.settings/org.eclipse.jdt.core.prefs b/hyracks-tests/hyracks-storage-am-btree-test/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 7cf8ad6..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,264 +0,0 @@
-#Fri May 20 19:34:07 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
-org.eclipse.jdt.core.formatter.align_type_members_on_columns=false
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_enum_constant=48
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_explicit_constructor_call=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_arguments_in_qualified_allocation_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_assignment=0
-org.eclipse.jdt.core.formatter.alignment_for_binary_expression=16
-org.eclipse.jdt.core.formatter.alignment_for_compact_if=16
-org.eclipse.jdt.core.formatter.alignment_for_conditional_expression=80
-org.eclipse.jdt.core.formatter.alignment_for_enum_constants=48
-org.eclipse.jdt.core.formatter.alignment_for_expressions_in_array_initializer=16
-org.eclipse.jdt.core.formatter.alignment_for_multiple_fields=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_parameters_in_method_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_selector_in_method_invocation=16
-org.eclipse.jdt.core.formatter.alignment_for_superclass_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_enum_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_superinterfaces_in_type_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_constructor_declaration=16
-org.eclipse.jdt.core.formatter.alignment_for_throws_clause_in_method_declaration=16
-org.eclipse.jdt.core.formatter.blank_lines_after_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_after_package=1
-org.eclipse.jdt.core.formatter.blank_lines_before_field=0
-org.eclipse.jdt.core.formatter.blank_lines_before_first_class_body_declaration=0
-org.eclipse.jdt.core.formatter.blank_lines_before_imports=1
-org.eclipse.jdt.core.formatter.blank_lines_before_member_type=1
-org.eclipse.jdt.core.formatter.blank_lines_before_method=1
-org.eclipse.jdt.core.formatter.blank_lines_before_new_chunk=1
-org.eclipse.jdt.core.formatter.blank_lines_before_package=0
-org.eclipse.jdt.core.formatter.blank_lines_between_import_groups=1
-org.eclipse.jdt.core.formatter.blank_lines_between_type_declarations=1
-org.eclipse.jdt.core.formatter.brace_position_for_annotation_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_anonymous_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_array_initializer=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_block_in_case=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_constructor_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_constant=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_enum_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_method_declaration=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_switch=end_of_line
-org.eclipse.jdt.core.formatter.brace_position_for_type_declaration=end_of_line
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_block_comment=false
-org.eclipse.jdt.core.formatter.comment.clear_blank_lines_in_javadoc_comment=false
-org.eclipse.jdt.core.formatter.comment.format_block_comments=true
-org.eclipse.jdt.core.formatter.comment.format_header=false
-org.eclipse.jdt.core.formatter.comment.format_html=true
-org.eclipse.jdt.core.formatter.comment.format_javadoc_comments=true
-org.eclipse.jdt.core.formatter.comment.format_line_comments=true
-org.eclipse.jdt.core.formatter.comment.format_source_code=true
-org.eclipse.jdt.core.formatter.comment.indent_parameter_description=true
-org.eclipse.jdt.core.formatter.comment.indent_root_tags=true
-org.eclipse.jdt.core.formatter.comment.insert_new_line_before_root_tags=insert
-org.eclipse.jdt.core.formatter.comment.insert_new_line_for_parameter=insert
-org.eclipse.jdt.core.formatter.comment.line_length=80
-org.eclipse.jdt.core.formatter.compact_else_if=true
-org.eclipse.jdt.core.formatter.continuation_indentation=2
-org.eclipse.jdt.core.formatter.continuation_indentation_for_array_initializer=2
-org.eclipse.jdt.core.formatter.format_guardian_clause_on_one_line=false
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_annotation_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_constant_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_enum_declaration_header=true
-org.eclipse.jdt.core.formatter.indent_body_declarations_compare_to_type_header=true
-org.eclipse.jdt.core.formatter.indent_breaks_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_empty_lines=false
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_block=true
-org.eclipse.jdt.core.formatter.indent_statements_compare_to_body=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_cases=true
-org.eclipse.jdt.core.formatter.indent_switchstatements_compare_to_switch=true
-org.eclipse.jdt.core.formatter.indentation.size=4
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_local_variable=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_member=insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_annotation_on_parameter=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_after_opening_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_at_end_of_file_if_missing=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_catch_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_closing_brace_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_else_in_if_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_finally_in_try_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_before_while_in_do_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_annotation_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_block=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_method_body=insert
-org.eclipse.jdt.core.formatter.insert_new_line_in_empty_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_after_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_after_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_at_in_annotation_type_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_angle_bracket_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_after_closing_paren_in_cast=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_case=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_colon_in_labeled_statement=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_allocation_expression=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_annotation=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_constructor_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_constant_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_enum_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_explicitconstructorcall_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_increments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_for_inits=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_declaration_throws=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_method_invocation_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_field_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_multiple_local_declarations=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_parameterized_type_reference=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_superinterfaces=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_arguments=insert
-org.eclipse.jdt.core.formatter.insert_space_after_comma_in_type_parameters=insert
-org.eclipse.jdt.core.formatter.insert_space_after_ellipsis=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_opening_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_after_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_after_semicolon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_after_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_and_in_type_parameter=insert
-org.eclipse.jdt.core.formatter.insert_space_before_assignment_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_at_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_binary_operator=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_cast=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_catch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_if=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_switch=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_synchronized=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_closing_paren_in_while=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_assert=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_case=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_default=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_colon_in_labeled_statement=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_constructor_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_constant_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_enum_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_explicitconstructorcall_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_increments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_for_inits=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_declaration_throws=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_method_invocation_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_field_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_multiple_local_declarations=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_superinterfaces=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_comma_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_ellipsis=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_parameterized_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_arguments=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_angle_bracket_in_type_parameters=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_annotation_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_anonymous_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_array_initializer=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_block=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_constructor_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_constant=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_enum_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_method_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_brace_in_type_declaration=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_bracket_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_catch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_for=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_if=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_parenthesized_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_switch=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_synchronized=insert
-org.eclipse.jdt.core.formatter.insert_space_before_opening_paren_in_while=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_return=insert
-org.eclipse.jdt.core.formatter.insert_space_before_parenthesized_expression_in_throw=insert
-org.eclipse.jdt.core.formatter.insert_space_before_postfix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_prefix_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_conditional=insert
-org.eclipse.jdt.core.formatter.insert_space_before_question_in_wildcard=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_semicolon_in_for=do not insert
-org.eclipse.jdt.core.formatter.insert_space_before_unary_operator=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_brackets_in_array_type_reference=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_braces_in_array_initializer=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_brackets_in_array_allocation_expression=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_annotation_type_member_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_constructor_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_enum_constant=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_declaration=do not insert
-org.eclipse.jdt.core.formatter.insert_space_between_empty_parens_in_method_invocation=do not insert
-org.eclipse.jdt.core.formatter.join_lines_in_comments=true
-org.eclipse.jdt.core.formatter.join_wrapped_lines=true
-org.eclipse.jdt.core.formatter.keep_else_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.keep_empty_array_initializer_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_imple_if_on_one_line=false
-org.eclipse.jdt.core.formatter.keep_then_statement_on_same_line=false
-org.eclipse.jdt.core.formatter.lineSplit=120
-org.eclipse.jdt.core.formatter.never_indent_block_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.never_indent_line_comments_on_first_column=false
-org.eclipse.jdt.core.formatter.number_of_blank_lines_at_beginning_of_method_body=0
-org.eclipse.jdt.core.formatter.number_of_empty_lines_to_preserve=1
-org.eclipse.jdt.core.formatter.put_empty_statement_on_new_line=true
-org.eclipse.jdt.core.formatter.tabulation.char=space
-org.eclipse.jdt.core.formatter.tabulation.size=4
-org.eclipse.jdt.core.formatter.use_tabs_only_for_leading_indentations=false
-org.eclipse.jdt.core.formatter.wrap_before_binary_operator=true
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/.settings/org.maven.ide.eclipse.prefs b/hyracks-tests/hyracks-storage-am-btree-test/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 99b89a6..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Jan 06 11:27:16 PST 2011
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/pom.xml b/hyracks-tests/hyracks-storage-am-btree-test/pom.xml
index 94089f0..68ab309 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/pom.xml
+++ b/hyracks-tests/hyracks-storage-am-btree-test/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree-test</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -34,20 +34,20 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-btree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-test-support</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>test</scope>
</dependency>
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/AbstractBTreeTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/AbstractBTreeTest.java
deleted file mode 100644
index 56ae6e9..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/AbstractBTreeTest.java
+++ /dev/null
@@ -1,28 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import java.io.File;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-import java.util.logging.Logger;
-
-import org.junit.AfterClass;
-
-public abstract class AbstractBTreeTest {
-
- protected static final Logger LOGGER = Logger.getLogger(AbstractBTreeTest.class.getName());
-
- protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
- protected final static String sep = System.getProperty("file.separator");
- protected final static String fileName = tmpDir + sep + simpleDateFormat.format(new Date());
-
- protected void print(String str) {
- System.out.print(str);
- }
-
- @AfterClass
- public static void cleanup() throws Exception {
- File f = new File(fileName);
- f.deleteOnExit();
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
new file mode 100644
index 0000000..5bb86b9
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeExamplesTest.java
@@ -0,0 +1,624 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+import java.util.logging.Level;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeUtils;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+
+@SuppressWarnings("rawtypes")
+public class BTreeExamplesTest extends AbstractBTreeTest {
+
+ /**
+ * Fixed-Length Key,Value Example.
+ *
+ * Create a BTree with one fixed-length key field and one fixed-length value
+ * field. Fill BTree with random values using insertions (not bulk load).
+ * Perform scans and range search.
+ */
+ @Test
+ public void fixedLengthKeyValueExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Fixed-Length Key,Value Example.");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+
+ BTree btree = BTreeUtils
+ .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+ int numInserts = 10000;
+ for (int i = 0; i < numInserts; i++) {
+ int f0 = rnd.nextInt() % numInserts;
+ int f1 = 5;
+ TupleUtils.createIntegerTuple(tb, tuple, f0, f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i + " : " + f0 + " " + f1);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(btree, indexAccessor, fieldSerdes);
+ diskOrderScan(btree, indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(keyFieldCount);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -1000);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(keyFieldCount);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 1000);
+
+ rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ btree.close();
+ }
+
+ /**
+ * Composite Key Example (Non-Unique B-Tree).
+ *
+ * Create a BTree with two fixed-length key fields and one fixed-length
+ * value field. Fill BTree with random values using insertions (not bulk
+ * load) Perform scans and range search.
+ */
+ @Test
+ public void twoFixedLengthKeysOneFixedLengthValueExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Composite Key Test");
+ }
+
+ // Declare fields.
+ int fieldCount = 3;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // declare keys
+ int keyFieldCount = 2;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+
+ BTree btree = BTreeUtils
+ .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+ int numInserts = 10000;
+ for (int i = 0; i < 10000; i++) {
+ int f0 = rnd.nextInt() % 2000;
+ int f1 = rnd.nextInt() % 1000;
+ int f2 = 5;
+ TupleUtils.createIntegerTuple(tb, tuple, f0, f1, f2);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i + " : " + f0 + " " + f1 + " " + f2);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(btree, indexAccessor, fieldSerdes);
+ diskOrderScan(btree, indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, -3);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 3);
+
+ // Prefix-Range search in [-3, 3]
+ rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ btree.close();
+ }
+
+ /**
+ * Variable-Length Example. Create a BTree with one variable-length key
+ * field and one variable-length value field. Fill BTree with random values
+ * using insertions (not bulk load) Perform ordered scans and range search.
+ */
+ @Test
+ public void varLenKeyValueExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Variable-Length Key,Value Example");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
+
+ BTree btree = BTreeUtils
+ .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+
+ long start = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+ // Max string length to be generated.
+ int maxLength = 10;
+ int numInserts = 10000;
+ for (int i = 0; i < 10000; i++) {
+ String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + f0 + " " + f1);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(numInserts + " inserts in " + (end - start) + "ms");
+ }
+
+ orderedScan(btree, indexAccessor, fieldSerdes);
+ diskOrderScan(btree, indexAccessor, fieldSerdes);
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createTuple(lowKeyTb, lowKey, fieldSerdes, "cbf");
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createTuple(highKeyTb, highKey, fieldSerdes, "cc7");
+
+ rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ btree.close();
+ }
+
+ /**
+ * Deletion Example.
+ *
+ * Create a BTree with one variable-length key field and one variable-length
+ * value field. Fill B-tree with random values using insertions, then delete
+ * entries one-by-one. Repeat procedure a few times on same BTree.
+ */
+ @Test
+ public void deleteExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deletion Example");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
+
+ BTree btree = BTreeUtils
+ .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+ // Max string length to be generated.
+ int runs = 3;
+ for (int run = 0; run < runs; run++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deletion example run: " + (run + 1) + "/" + runs);
+ LOGGER.info("Inserting into tree...");
+ }
+ int maxLength = 10;
+ int ins = 10000;
+ String[] f0s = new String[ins];
+ String[] f1s = new String[ins];
+ int insDone = 0;
+ int[] insDoneCmp = new int[ins];
+ for (int i = 0; i < ins; i++) {
+ String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
+ f0s[i] = f0;
+ f1s[i] = f1;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ insDone++;
+ } catch (TreeIndexException e) {
+ }
+ insDoneCmp[i] = insDone;
+ }
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Deleting from tree...");
+ }
+ int delDone = 0;
+ for (int i = 0; i < ins; i++) {
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0s[i], f1s[i]);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Deleting " + i);
+ }
+ }
+ try {
+ indexAccessor.delete(tuple);
+ delDone++;
+ } catch (TreeIndexException e) {
+ }
+ if (insDoneCmp[i] != delDone) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION EXAMPLE.");
+ LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
+ }
+ break;
+ }
+ }
+ if (insDone != delDone) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
+ }
+ break;
+ }
+ }
+ btree.close();
+ }
+
+ /**
+ * Update example.
+ *
+ * Create a BTree with one variable-length key field and one variable-length
+ * value field. Fill B-tree with random values using insertions, then update
+ * entries one-by-one. Repeat procedure a few times on same BTree.
+ */
+ @Test
+ public void updateExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Update example");
+ }
+
+ // Declare fields.
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = UTF8StringPointable.TYPE_TRAITS;
+ typeTraits[1] = UTF8StringPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Declare keys.
+ int keyFieldCount = 1;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
+
+ BTree btree = BTreeUtils
+ .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Inserting into tree...");
+ }
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ int maxLength = 10;
+ int ins = 10000;
+ String[] keys = new String[10000];
+ for (int i = 0; i < ins; i++) {
+ String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, f0, f1);
+ keys[i] = f0;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("Inserting " + i);
+ }
+ }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ // Print before doing any updates.
+ orderedScan(btree, indexAccessor, fieldSerdes);
+
+ int runs = 3;
+ for (int run = 0; run < runs; run++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Update test run: " + (run + 1) + "/" + runs);
+ LOGGER.info("Updating BTree");
+ }
+ for (int i = 0; i < ins; i++) {
+ // Generate a new random value for f1.
+ String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
+ TupleUtils.createTuple(tb, tuple, fieldSerdes, keys[i], f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("UPDATING " + i);
+ }
+ }
+ try {
+ indexAccessor.update(tuple);
+ } catch (TreeIndexException e) {
+ }
+ }
+ // Do another scan after a round of updates.
+ orderedScan(btree, indexAccessor, fieldSerdes);
+ }
+ btree.close();
+ }
+
+ /**
+ * Bulk load example.
+ *
+ * Load a tree with 100,000 tuples. BTree has a composite key to "simulate"
+ * non-unique index creation.
+ *
+ */
+ @Test
+ public void bulkLoadExample() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bulk load example");
+ }
+ // Declare fields.
+ int fieldCount = 3;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ // Declare field serdes.
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // declare keys
+ int keyFieldCount = 2;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+
+ BTree btree = BTreeUtils
+ .createBTree(bufferCache, btreeFileId, typeTraits, cmps, BTreeLeafFrameType.REGULAR_NSM);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+
+ // Load sorted records.
+ int ins = 100000;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Bulk loading " + ins + " tuples");
+ }
+ long start = System.currentTimeMillis();
+ IIndexBulkLoadContext bulkLoadCtx = btree.beginBulkLoad(0.7f);
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ for (int i = 0; i < ins; i++) {
+ TupleUtils.createIntegerTuple(tb, tuple, i, i, 5);
+ btree.bulkLoadAddTuple(tuple, bulkLoadCtx);
+ }
+ btree.endBulkLoad(bulkLoadCtx);
+ long end = System.currentTimeMillis();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(ins + " tuples loaded in " + (end - start) + "ms");
+ }
+
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+
+ // Build low key.
+ ArrayTupleBuilder lowKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(lowKeyTb, lowKey, 44444);
+
+ // Build high key.
+ ArrayTupleBuilder highKeyTb = new ArrayTupleBuilder(1);
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ TupleUtils.createIntegerTuple(highKeyTb, highKey, 44500);
+
+ // Prefix-Range search in [44444, 44500]
+ rangeSearch(btree, indexAccessor, fieldSerdes, lowKey, highKey);
+
+ btree.close();
+ }
+
+ private void orderedScan(BTree btree, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes)
+ throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Ordered Scan:");
+ }
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
+ ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame, false);
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ indexAccessor.search(scanCursor, nullPred);
+ try {
+ while (scanCursor.hasNext()) {
+ scanCursor.next();
+ ITupleReference frameTuple = scanCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ scanCursor.close();
+ }
+ }
+
+ private void diskOrderScan(BTree btree, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes)
+ throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Disk-Order Scan:");
+ }
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
+ TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
+ indexAccessor.diskOrderScan(diskOrderCursor);
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference frameTuple = diskOrderCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ diskOrderCursor.close();
+ }
+ }
+
+ private void rangeSearch(BTree btree, ITreeIndexAccessor indexAccessor, ISerializerDeserializer[] fieldSerdes,
+ ITupleReference lowKey, ITupleReference highKey) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ String lowKeyString = TupleUtils.printTuple(lowKey, fieldSerdes);
+ String highKeyString = TupleUtils.printTuple(highKey, fieldSerdes);
+ LOGGER.info("Range-Search in: [" + lowKeyString + ", " + highKeyString + "]");
+ }
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
+ MultiComparator lowKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), lowKey);
+ MultiComparator highKeySearchCmp = BTreeUtils.getSearchMultiComparator(btree.getMultiComparator(), highKey);
+ ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame, false);
+ RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, lowKeySearchCmp,
+ highKeySearchCmp);
+ indexAccessor.search(rangeCursor, rangePred);
+ try {
+ while (rangeCursor.hasNext()) {
+ rangeCursor.next();
+ ITupleReference frameTuple = rangeCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, fieldSerdes);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } finally {
+ rangeCursor.close();
+ }
+ }
+
+ public static String randomString(int length, Random random) {
+ String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < s.length() && i < length; i++) {
+ strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
+ }
+ return strBuilder.toString();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeFieldPrefixNSMTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeFieldPrefixNSMTest.java
deleted file mode 100644
index 4402e22..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeFieldPrefixNSMTest.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.api.IPrefixSlotManager;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeException;
-import edu.uci.ics.hyracks.storage.am.btree.impls.FieldPrefixSlotManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
-import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-
-public class BTreeFieldPrefixNSMTest extends AbstractBTreeTest {
-
- private static final int PAGE_SIZE = 32768; // 32K
- private static final int NUM_PAGES = 40;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- private ITupleReference createTuple(IHyracksTaskContext ctx, int f0,
- int f1, int f2, boolean print) throws HyracksDataException {
- if (print)
- LOGGER.info("CREATING: " + f0 + " " + f1 + " " + f2);
-
- ByteBuffer buf = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(3);
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = {
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx
- .getFrameSize(), recDesc);
- accessor.reset(buf);
- FrameTupleReference tuple = new FrameTupleReference();
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f2, dos);
- tb.addFieldEndOffset();
-
- appender.reset(buf, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb
- .getSize());
-
- tuple.reset(accessor, 0);
-
- return tuple;
- }
-
- @Test
- public void test01() throws Exception {
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare fields
- int fieldCount = 3;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(4);
- typeTraits[2] = new TypeTrait(4);
-
- // declare keys
- int keyFieldCount = 3;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- cmps[1] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- cmps[2] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
-
- // just for printing
- ISerializerDeserializer[] sers = {
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(
- fileId, 0), false);
- try {
-
- IPrefixSlotManager slotManager = new FieldPrefixSlotManager();
- ITreeIndexTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
- BTreeFieldPrefixNSMLeafFrame frame = new BTreeFieldPrefixNSMLeafFrame(
- tupleWriter);
- frame.setPage(page);
- frame.initBuffer((byte) 0);
- slotManager.setFrame(frame);
- frame.setPrefixTupleCount(0);
-
- String before = new String();
- String after = new String();
-
- int compactFreq = 5;
- int compressFreq = 5;
- int smallMax = 10;
- int numRecords = 1000;
-
- int[][] savedFields = new int[numRecords][3];
-
- // insert records with random calls to compact and compress
- for (int i = 0; i < numRecords; i++) {
-
- if ((i + 1) % 100 == 0)
- LOGGER.info("INSERTING " + (i + 1) + " / " + numRecords);
-
- int a = rnd.nextInt() % smallMax;
- int b = rnd.nextInt() % smallMax;
- int c = i;
-
- ITupleReference tuple = createTuple(ctx, a, b, c, false);
- try {
- int targetTupleIndex = frame.findTupleIndex(tuple, cmp);
- frame.insert(tuple, cmp, targetTupleIndex);
- } catch (BTreeException e) {
- e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- savedFields[i][0] = a;
- savedFields[i][1] = b;
- savedFields[i][2] = c;
-
- if (rnd.nextInt() % compactFreq == 0) {
- before = frame.printKeys(cmp, sers);
- frame.compact(cmp);
- after = frame.printKeys(cmp, sers);
- Assert.assertEquals(before, after);
- }
-
- if (rnd.nextInt() % compressFreq == 0) {
- before = frame.printKeys(cmp, sers);
- frame.compress(cmp);
- after = frame.printKeys(cmp, sers);
- Assert.assertEquals(before, after);
- }
-
- }
-
- // delete records with random calls to compact and compress
- for (int i = 0; i < numRecords; i++) {
-
- if ((i + 1) % 100 == 0)
- LOGGER.info("DELETING " + (i + 1) + " / " + numRecords);
-
- ITupleReference tuple = createTuple(ctx,
- savedFields[i][0], savedFields[i][1],
- savedFields[i][2], false);
- try {
- frame.delete(tuple, cmp, true);
- } catch (Exception e) {
- }
-
- if (rnd.nextInt() % compactFreq == 0) {
- before = frame.printKeys(cmp, sers);
- frame.compact(cmp);
- after = frame.printKeys(cmp, sers);
- Assert.assertEquals(before, after);
- }
-
- if (rnd.nextInt() % compressFreq == 0) {
- before = frame.printKeys(cmp, sers);
- frame.compress(cmp);
- after = frame.printKeys(cmp, sers);
- Assert.assertEquals(before, after);
- }
- }
-
- } finally {
- bufferCache.unpin(page);
- }
-
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
deleted file mode 100644
index 3ef0cc2..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
+++ /dev/null
@@ -1,162 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.common.utility.TreeIndexBufferCacheWarmup;
-import edu.uci.ics.hyracks.storage.am.common.utility.TreeIndexStats;
-import edu.uci.ics.hyracks.storage.am.common.utility.TreeIndexStatsGatherer;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-
-public class BTreeStatsTest extends AbstractBTreeTest {
-
- // private static final int PAGE_SIZE = 256;
- // private static final int NUM_PAGES = 10;
- // private static final int PAGE_SIZE = 32768;
- private static final int PAGE_SIZE = 4096;
- private static final int NUM_PAGES = 1000;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- @Test
- public void test01() throws Exception {
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare fields
- int fieldCount = 2;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(4);
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
-
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- long start = System.currentTimeMillis();
-
- LOGGER.info("INSERTING INTO TREE");
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- BTreeOpContext insertOpCtx = btree.createOpContext(IndexOp.INSERT, leafFrame, interiorFrame, metaFrame);
-
- // 10000
- for (int i = 0; i < 100000; i++) {
-
- int f0 = rnd.nextInt() % 100000;
- int f1 = 5;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (i % 10000 == 0) {
- long end = System.currentTimeMillis();
- LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
- }
-
- try {
- btree.insert(tuple, insertOpCtx);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
- btree.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
- LOGGER.info(stats.toString());
-
- TreeIndexBufferCacheWarmup bufferCacheWarmup = new TreeIndexBufferCacheWarmup(bufferCache, freePageManager,
- fileId);
- bufferCacheWarmup.warmup(leafFrame, metaFrame, new int[] { 1, 2 }, new int[] { 2, 5 });
-
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
-}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTest.java
deleted file mode 100644
index a9debd5..0000000
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTest.java
+++ /dev/null
@@ -1,1190 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.hyracks.storage.am.btree;
-
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
-import java.util.Random;
-
-import org.junit.Test;
-
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
-import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
-import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.tuples.SimpleTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-
-public class BTreeTest extends AbstractBTreeTest {
-
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- // FIXED-LENGTH KEY TEST
- // create a B-tree with one fixed-length "key" field and one fixed-length
- // "value" field
- // fill B-tree with random values using insertions (not bulk load)
- // perform ordered scan and range search
- @Test
- public void test01() throws Exception {
-
- LOGGER.info("FIXED-LENGTH KEY TEST");
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare fields
- int fieldCount = 2;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(4);
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
-
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- long start = System.currentTimeMillis();
-
- LOGGER.info("INSERTING INTO TREE");
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- BTreeOpContext insertOpCtx = btree.createOpContext(IndexOp.INSERT, leafFrame, interiorFrame, metaFrame);
-
- // 10000
- for (int i = 0; i < 10000; i++) {
-
- int f0 = rnd.nextInt() % 10000;
- int f1 = 5;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (i % 1000 == 0) {
- long end = System.currentTimeMillis();
- LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
- }
-
- try {
- btree.insert(tuple, insertOpCtx);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- // btree.printTree(leafFrame, interiorFrame);
-
- int maxPage = btree.getFreePageManager().getMaxPage(metaFrame);
- LOGGER.info("MAXPAGE: " + maxPage);
- LOGGER.info(btree.printStats());
-
- long end = System.currentTimeMillis();
- long duration = end - start;
- LOGGER.info("DURATION: " + duration);
-
- // ordered scan
-
- LOGGER.info("ORDERED SCAN:");
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- BTreeOpContext searchOpCtx = btree.createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
- btree.search(scanCursor, nullPred, searchOpCtx);
- try {
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- LOGGER.info(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
-
- // disk-order scan
- LOGGER.info("DISK-ORDER SCAN:");
- TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
- BTreeOpContext diskOrderScanOpCtx = btree.createOpContext(IndexOp.DISKORDERSCAN, leafFrame, null, null);
- btree.diskOrderScan(diskOrderCursor, leafFrame, metaFrame, diskOrderScanOpCtx);
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference frameTuple = diskOrderCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- LOGGER.info(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- diskOrderCursor.close();
- }
-
- // range search in [-1000, 1000]
- LOGGER.info("RANGE SEARCH:");
-
- ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame);
-
- // build low and high keys
- ArrayTupleBuilder ktb = new ArrayTupleBuilder(cmp.getKeyFieldCount());
- DataOutput kdos = ktb.getDataOutput();
-
- ISerializerDeserializer[] keyDescSers = { IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor keyDesc = new RecordDescriptor(keyDescSers);
- IFrameTupleAccessor keyAccessor = new FrameTupleAccessor(ctx.getFrameSize(), keyDesc);
- keyAccessor.reset(frame);
-
- appender.reset(frame, true);
-
- // build and append low key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(-1000, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // build and append high key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(1000, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // create tuplereferences for search keys
- FrameTupleReference lowKey = new FrameTupleReference();
- lowKey.reset(keyAccessor, 0);
-
- FrameTupleReference highKey = new FrameTupleReference();
- highKey.reset(keyAccessor, 1);
-
- IBinaryComparator[] searchCmps = new IBinaryComparator[1];
- searchCmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- MultiComparator searchCmp = new MultiComparator(typeTraits, searchCmps);
-
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, searchCmp, searchCmp);
- btree.search(rangeCursor, rangePred, searchOpCtx);
-
- try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- LOGGER.info(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- rangeCursor.close();
- }
-
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
-
- // COMPOSITE KEY TEST (NON-UNIQUE B-TREE)
- // create a B-tree with one two fixed-length "key" fields and one
- // fixed-length "value" field
- // fill B-tree with random values using insertions (not bulk load)
- // perform ordered scan and range search
- @Test
- public void test02() throws Exception {
-
- LOGGER.info("COMPOSITE KEY TEST");
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare fields
- int fieldCount = 3;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(4);
- typeTraits[2] = new TypeTrait(4);
-
- // declare keys
- int keyFieldCount = 2;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- cmps[1] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
-
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- // SimpleTupleWriterFactory tupleWriterFactory = new
- // SimpleTupleWriterFactory();
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- long start = System.currentTimeMillis();
-
- LOGGER.info("INSERTING INTO TREE");
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- BTreeOpContext insertOpCtx = btree.createOpContext(IndexOp.INSERT, leafFrame, interiorFrame, metaFrame);
-
- for (int i = 0; i < 10000; i++) {
- int f0 = rnd.nextInt() % 2000;
- int f1 = rnd.nextInt() % 1000;
- int f2 = 5;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f2, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (i % 1000 == 0) {
- LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1);
- }
-
- try {
- btree.insert(tuple, insertOpCtx);
- } catch (Exception e) {
- }
- }
- // btree.printTree(leafFrame, interiorFrame);
-
- long end = System.currentTimeMillis();
- long duration = end - start;
- LOGGER.info("DURATION: " + duration);
-
- // try a simple index scan
- LOGGER.info("ORDERED SCAN:");
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- BTreeOpContext searchOpCtx = btree.createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
- btree.search(scanCursor, nullPred, searchOpCtx);
-
- try {
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- LOGGER.info(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
-
- // range search in [(-3),(3)]
- LOGGER.info("RANGE SEARCH:");
- ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame);
-
- // build low and high keys
- ArrayTupleBuilder ktb = new ArrayTupleBuilder(cmp.getKeyFieldCount());
- DataOutput kdos = ktb.getDataOutput();
-
- ISerializerDeserializer[] keyDescSers = { IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor keyDesc = new RecordDescriptor(keyDescSers);
- IFrameTupleAccessor keyAccessor = new FrameTupleAccessor(ctx.getFrameSize(), keyDesc);
- keyAccessor.reset(frame);
-
- appender.reset(frame, true);
-
- // build and append low key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(-3, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // build and append high key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(3, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // create tuplereferences for search keys
- FrameTupleReference lowKey = new FrameTupleReference();
- lowKey.reset(keyAccessor, 0);
-
- FrameTupleReference highKey = new FrameTupleReference();
- highKey.reset(keyAccessor, 1);
-
- IBinaryComparator[] searchCmps = new IBinaryComparator[1];
- searchCmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- MultiComparator searchCmp = new MultiComparator(typeTraits, searchCmps); // use
- // only
- // a
- // single
- // comparator
- // for
- // searching
-
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, searchCmp, searchCmp);
- btree.search(rangeCursor, rangePred, searchOpCtx);
-
- try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- rangeCursor.close();
- }
-
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
-
- // VARIABLE-LENGTH TEST
- // create a B-tree with one variable-length "key" field and one
- // variable-length "value" field
- // fill B-tree with random values using insertions (not bulk load)
- // perform ordered scan and range search
- @Test
- public void test03() throws Exception {
-
- LOGGER.info("VARIABLE-LENGTH KEY TEST");
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare fields
- int fieldCount = 2;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- typeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
-
- SimpleTupleWriterFactory tupleWriterFactory = new SimpleTupleWriterFactory();
- // TypeAwareTupleWriterFactory tupleWriterFactory = new
- // TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- BTreeOpContext insertOpCtx = btree.createOpContext(IndexOp.INSERT, leafFrame, interiorFrame, metaFrame);
- int maxLength = 10; // max string length to be generated
- for (int i = 0; i < 10000; i++) {
-
- String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
-
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (i % 1000 == 0) {
- LOGGER.info("INSERTING " + i);
- }
-
- try {
- btree.insert(tuple, insertOpCtx);
- } catch (Exception e) {
- }
- }
- // btree.printTree();
-
- LOGGER.info("DONE INSERTING");
-
- // ordered scan
- LOGGER.info("ORDERED SCAN:");
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- BTreeOpContext searchOpCtx = btree.createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
- btree.search(scanCursor, nullPred, searchOpCtx);
-
- try {
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- LOGGER.info(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
-
- // range search in ["cbf", cc7"]
- LOGGER.info("RANGE SEARCH:");
-
- ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame);
-
- // build low and high keys
- ArrayTupleBuilder ktb = new ArrayTupleBuilder(cmp.getKeyFieldCount());
- DataOutput kdos = ktb.getDataOutput();
-
- ISerializerDeserializer[] keyDescSers = { UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyDesc = new RecordDescriptor(keyDescSers);
- IFrameTupleAccessor keyAccessor = new FrameTupleAccessor(ctx.getFrameSize(), keyDesc);
- keyAccessor.reset(frame);
-
- appender.reset(frame, true);
-
- // build and append low key
- ktb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("cbf", kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // build and append high key
- ktb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("cc7", kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // create tuplereferences for search keys
- FrameTupleReference lowKey = new FrameTupleReference();
- lowKey.reset(keyAccessor, 0);
-
- FrameTupleReference highKey = new FrameTupleReference();
- highKey.reset(keyAccessor, 1);
-
- IBinaryComparator[] searchCmps = new IBinaryComparator[1];
- searchCmps[0] = UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- MultiComparator searchCmp = new MultiComparator(typeTraits, searchCmps);
-
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, searchCmp, searchCmp);
- btree.search(rangeCursor, rangePred, searchOpCtx);
-
- try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- LOGGER.info(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- rangeCursor.close();
- }
-
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
-
- // DELETION TEST
- // create a B-tree with one variable-length "key" field and one
- // variable-length "value" field
- // fill B-tree with random values using insertions, then delete entries
- // one-by-one
- // repeat procedure a few times on same B-tree
- @Test
- public void test04() throws Exception {
-
- LOGGER.info("DELETION TEST");
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare fields
- int fieldCount = 2;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- typeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
-
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
-
- // SimpleTupleWriterFactory tupleWriterFactory = new
- // SimpleTupleWriterFactory();
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- BTreeOpContext insertOpCtx = btree.createOpContext(IndexOp.INSERT, leafFrame, interiorFrame, metaFrame);
- BTreeOpContext deleteOpCtx = btree.createOpContext(IndexOp.DELETE, leafFrame, interiorFrame, metaFrame);
-
- int runs = 3;
- for (int run = 0; run < runs; run++) {
-
- LOGGER.info("DELETION TEST RUN: " + (run + 1) + "/" + runs);
-
- LOGGER.info("INSERTING INTO BTREE");
- int maxLength = 10;
- int ins = 10000;
- String[] f0s = new String[ins];
- String[] f1s = new String[ins];
- int insDone = 0;
- int[] insDoneCmp = new int[ins];
- for (int i = 0; i < ins; i++) {
- String f0 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
- String f1 = randomString(Math.abs(rnd.nextInt()) % maxLength + 1, rnd);
-
- f0s[i] = f0;
- f1s[i] = f1;
-
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (i % 1000 == 0) {
- LOGGER.info("INSERTING " + i);
- }
-
- try {
- btree.insert(tuple, insertOpCtx);
- insDone++;
- } catch (TreeIndexException e) {
- // e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- insDoneCmp[i] = insDone;
- }
- // btree.printTree();
- // btree.printStats();
-
- LOGGER.info("DELETING FROM BTREE");
- int delDone = 0;
- for (int i = 0; i < ins; i++) {
-
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(f0s[i], dos);
- tb.addFieldEndOffset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(f1s[i], dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- if (i % 1000 == 0) {
- LOGGER.info("DELETING " + i);
- }
-
- try {
- btree.delete(tuple, deleteOpCtx);
- delDone++;
- } catch (TreeIndexException e) {
- // e.printStackTrace();
- } catch (Exception e) {
- e.printStackTrace();
- }
-
- if (insDoneCmp[i] != delDone) {
- LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION TEST");
- LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
- break;
- }
- }
- // btree.printTree(leafFrame, interiorFrame);
-
- if (insDone != delDone) {
- LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
- break;
- }
- }
-
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
-
- // BULK LOAD TEST
- // insert 100,000 records in bulk
- // B-tree has a composite key to "simulate" non-unique index creation
- // do range search
- @Test
- public void test05() throws Exception {
-
- LOGGER.info("BULK LOAD TEST");
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare fields
- int fieldCount = 3;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(4);
- typeTraits[2] = new TypeTrait(4);
-
- // declare keys
- int keyFieldCount = 2;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- cmps[1] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
-
- // SimpleTupleWriterFactory tupleWriterFactory = new
- // SimpleTupleWriterFactory();
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
- ITreeIndexFrame interiorFrame = interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- IIndexBulkLoadContext bulkLoadCtx = btree.beginBulkLoad(0.7f, leafFrame, interiorFrame, metaFrame);
-
- // generate sorted records
- int ins = 100000;
- LOGGER.info("BULK LOADING " + ins + " RECORDS");
- long start = System.currentTimeMillis();
- for (int i = 0; i < ins; i++) {
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(i, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(i, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(5, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- btree.bulkLoadAddTuple(bulkLoadCtx, tuple);
- }
-
- btree.endBulkLoad(bulkLoadCtx);
-
- // btree.printTree(leafFrame, interiorFrame);
-
- long end = System.currentTimeMillis();
- long duration = end - start;
- LOGGER.info("DURATION: " + duration);
-
- // range search
- LOGGER.info("RANGE SEARCH:");
- ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame);
-
- // build low and high keys
- ArrayTupleBuilder ktb = new ArrayTupleBuilder(1);
- DataOutput kdos = ktb.getDataOutput();
-
- ISerializerDeserializer[] keyDescSers = { IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor keyDesc = new RecordDescriptor(keyDescSers);
- IFrameTupleAccessor keyAccessor = new FrameTupleAccessor(ctx.getFrameSize(), keyDesc);
- keyAccessor.reset(frame);
-
- appender.reset(frame, true);
-
- // build and append low key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(44444, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // build and append high key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(44500, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // create tuplereferences for search keys
- FrameTupleReference lowKey = new FrameTupleReference();
- lowKey.reset(keyAccessor, 0);
-
- FrameTupleReference highKey = new FrameTupleReference();
- highKey.reset(keyAccessor, 1);
-
- IBinaryComparator[] searchCmps = new IBinaryComparator[1];
- searchCmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- MultiComparator searchCmp = new MultiComparator(typeTraits, searchCmps);
-
- // TODO: check when searching backwards
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, searchCmp, searchCmp);
- BTreeOpContext searchOpCtx = btree.createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
- btree.search(rangeCursor, rangePred, searchOpCtx);
-
- try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- LOGGER.info(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- rangeCursor.close();
- }
-
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
-
- // TIME-INTERVAL INTERSECTION DEMO FOR EVENT PEOPLE
- // demo for Arjun to show easy support of intersection queries on
- // time-intervals
- @Test
- public void test06() throws Exception {
-
- LOGGER.info("TIME-INTERVAL INTERSECTION DEMO");
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare fields
- int fieldCount = 3;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(4);
- typeTraits[2] = new TypeTrait(4);
-
- // declare keys
- int keyFieldCount = 2;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- cmps[1] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
-
- // SimpleTupleWriterFactory tupleWriterFactory = new
- // SimpleTupleWriterFactory();
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
-
- Random rnd = new Random();
- rnd.setSeed(50);
-
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
-
- ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
- accessor.reset(frame);
- FrameTupleReference tuple = new FrameTupleReference();
-
- long start = System.currentTimeMillis();
-
- int intervalCount = 10;
- int[][] intervals = new int[intervalCount][2];
-
- intervals[0][0] = 10;
- intervals[0][1] = 20;
-
- intervals[1][0] = 11;
- intervals[1][1] = 20;
-
- intervals[2][0] = 12;
- intervals[2][1] = 20;
-
- intervals[3][0] = 13;
- intervals[3][1] = 20;
-
- intervals[4][0] = 14;
- intervals[4][1] = 20;
-
- intervals[5][0] = 20;
- intervals[5][1] = 30;
-
- intervals[6][0] = 20;
- intervals[6][1] = 31;
-
- intervals[7][0] = 20;
- intervals[7][1] = 32;
-
- intervals[8][0] = 20;
- intervals[8][1] = 33;
-
- intervals[9][0] = 20;
- intervals[9][1] = 35;
-
- BTreeOpContext insertOpCtx = btree.createOpContext(IndexOp.INSERT, leafFrame, interiorFrame, metaFrame);
-
- // int exceptionCount = 0;
- for (int i = 0; i < intervalCount; i++) {
- int f0 = intervals[i][0];
- int f1 = intervals[i][1];
- int f2 = rnd.nextInt() % 100;
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(f2, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- LOGGER.info("INSERTING " + i);
-
- try {
- btree.insert(tuple, insertOpCtx);
- } catch (Exception e) {
- }
- }
- // btree.printTree(leafFrame, interiorFrame);
- // btree.printStats();
-
- long end = System.currentTimeMillis();
- long duration = end - start;
- LOGGER.info("DURATION: " + duration);
-
- // try a simple index scan
-
- LOGGER.info("ORDERED SCAN:");
- ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame);
- RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
- BTreeOpContext searchOpCtx = btree.createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
- btree.search(scanCursor, nullPred, searchOpCtx);
-
- try {
- while (scanCursor.hasNext()) {
- scanCursor.next();
- ITupleReference frameTuple = scanCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- scanCursor.close();
- }
-
- // try a range search
- LOGGER.info("RANGE SEARCH:");
- ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame);
-
- // build low and high keys
- ArrayTupleBuilder ktb = new ArrayTupleBuilder(cmp.getKeyFieldCount());
- DataOutput kdos = ktb.getDataOutput();
-
- ISerializerDeserializer[] keyDescSers = { IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor keyDesc = new RecordDescriptor(keyDescSers);
- IFrameTupleAccessor keyAccessor = new FrameTupleAccessor(ctx.getFrameSize(), keyDesc);
- keyAccessor.reset(frame);
-
- appender.reset(frame, true);
-
- // build and append low key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(12, kdos);
- ktb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(12, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // build and append high key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(19, kdos);
- ktb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(19, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb.getSize());
-
- // create tuplereferences for search keys
- FrameTupleReference lowKey = new FrameTupleReference();
- lowKey.reset(keyAccessor, 0);
-
- FrameTupleReference highKey = new FrameTupleReference();
- highKey.reset(keyAccessor, 1);
-
- IBinaryComparator[] searchCmps = new IBinaryComparator[2];
- searchCmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- searchCmps[1] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
- MultiComparator searchCmp = new MultiComparator(typeTraits, searchCmps);
-
- RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, searchCmp, searchCmp);
- btree.search(rangeCursor, rangePred, searchOpCtx);
-
- try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- LOGGER.info(rec);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- rangeCursor.close();
- }
-
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
-
- public static String randomString(int length, Random random) {
- String s = Long.toHexString(Double.doubleToLongBits(random.nextDouble()));
- StringBuilder strBuilder = new StringBuilder();
- for (int i = 0; i < s.length() && i < length; i++) {
- strBuilder.append(s.charAt(Math.abs(random.nextInt()) % s.length()));
- }
- return strBuilder.toString();
- }
-}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTestDriver.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTestDriver.java
new file mode 100644
index 0000000..1daa273
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTestDriver.java
@@ -0,0 +1,133 @@
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.logging.Level;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
+
+@SuppressWarnings("rawtypes")
+public abstract class BTreeTestDriver extends AbstractBTreeTest {
+
+ protected static final int numTuplesToInsert = 10000;
+
+ protected abstract void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception;
+ protected abstract String getTestOpName();
+
+ @Test
+ public void oneIntKeyAndValue() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With One Int Key And Value.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+ // Range search in [-1000, 1000]
+ ITupleReference lowKey = TupleUtils.createIntegerTuple(-1000);
+ ITupleReference highKey = TupleUtils.createIntegerTuple(1000);
+
+ runTest(fieldSerdes, 1, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, null, null);
+ runTest(fieldSerdes, 1, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, null, null);
+ }
+
+ @Test
+ public void twoIntKeys() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With Two Int Keys.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // Range search in [50 0, 50 500]
+ ITupleReference lowKey = TupleUtils.createIntegerTuple(50, 0);
+ ITupleReference highKey = TupleUtils.createIntegerTuple(50, 500);
+
+ // Prefix range search in [50, 50]
+ ITupleReference prefixLowKey = TupleUtils.createIntegerTuple(50);
+ ITupleReference prefixHighKey = TupleUtils.createIntegerTuple(50);
+
+ runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
+
+ @Test
+ public void twoIntKeysAndValues() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With Two Int Keys And Values.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ // Range search in [50 100, 100 100]
+ ITupleReference lowKey = TupleUtils.createIntegerTuple(-100, -100);
+ ITupleReference highKey = TupleUtils.createIntegerTuple(100, 100);
+
+ // Prefix range search in [50, 50]
+ ITupleReference prefixLowKey = TupleUtils.createIntegerTuple(50);
+ ITupleReference prefixHighKey = TupleUtils.createIntegerTuple(50);
+
+ runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
+
+ @Test
+ public void oneStringKeyAndValue() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With One String Key And Value.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Range search in ["cbf", cc7"]
+ ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
+ ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7");
+
+ runTest(fieldSerdes, 1, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, null, null);
+ runTest(fieldSerdes, 1, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, null, null);
+ }
+
+ @Test
+ public void twoStringKeys() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With Two String Keys.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Range search in ["cbf", "ddd", cc7", "eee"]
+ ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf", "ddd");
+ ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7", "eee");
+
+ // Prefix range search in ["cbf", cc7"]
+ ITupleReference prefixLowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
+ ITupleReference prefixHighKey = TupleUtils.createTuple(fieldSerdes, "cc7");
+
+ runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
+
+ @Test
+ public void twoStringKeysAndValues() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("BTree " + getTestOpName() + " Test With Two String Keys And Values.");
+ }
+
+ ISerializerDeserializer[] fieldSerdes = { UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE };
+
+ // Range search in ["cbf", "ddd", cc7", "eee"]
+ ITupleReference lowKey = TupleUtils.createTuple(fieldSerdes, "cbf", "ddd");
+ ITupleReference highKey = TupleUtils.createTuple(fieldSerdes, "cc7", "eee");
+
+ // Prefix range search in ["cbf", cc7"]
+ ITupleReference prefixLowKey = TupleUtils.createTuple(fieldSerdes, "cbf");
+ ITupleReference prefixHighKey = TupleUtils.createTuple(fieldSerdes, "cc7");
+
+ runTest(fieldSerdes, 2, BTreeLeafFrameType.REGULAR_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ runTest(fieldSerdes, 2, BTreeLeafFrameType.FIELD_PREFIX_COMPRESSED_NSM, lowKey, highKey, prefixLowKey, prefixHighKey);
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BulkLoadTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BulkLoadTest.java
new file mode 100644
index 0000000..49d08a3
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BulkLoadTest.java
@@ -0,0 +1,53 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
+
+@SuppressWarnings("rawtypes")
+public class BulkLoadTest extends BTreeTestDriver {
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
+ BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
+
+ // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ BTreeTestUtils.bulkLoadIntTuples(testCtx, numTuplesToInsert, rnd);
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ BTreeTestUtils.bulkLoadStringTuples(testCtx, numTuplesToInsert, rnd);
+ }
+
+ BTreeTestUtils.checkPointSearches(testCtx);
+ BTreeTestUtils.checkOrderedScan(testCtx);
+ BTreeTestUtils.checkDiskOrderScan(testCtx);
+ BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
+ }
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "BulkLoad";
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/DeleteTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/DeleteTest.java
new file mode 100644
index 0000000..2157adb
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/DeleteTest.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
+
+@SuppressWarnings("rawtypes")
+public class DeleteTest extends BTreeTestDriver {
+
+ private static final int numInsertRounds = 3;
+ private static final int numDeleteRounds = 3;
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
+ BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
+ for (int i = 0; i < numInsertRounds; i++) {
+
+ // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ BTreeTestUtils.insertIntTuples(testCtx, numTuplesToInsert, rnd);
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ BTreeTestUtils.insertStringTuples(testCtx, numTuplesToInsert, rnd);
+ }
+
+ int numTuplesPerDeleteRound = (int)Math.ceil((float)testCtx.checkTuples.size() / (float)numDeleteRounds);
+ for(int j = 0; j < numDeleteRounds; j++) {
+ BTreeTestUtils.deleteTuples(testCtx, numTuplesPerDeleteRound, rnd);
+
+ BTreeTestUtils.checkPointSearches(testCtx);
+ BTreeTestUtils.checkOrderedScan(testCtx);
+ BTreeTestUtils.checkDiskOrderScan(testCtx);
+ BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
+ }
+ }
+ }
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Delete";
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
new file mode 100644
index 0000000..e8aa00a
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/FieldPrefixNSMTest.java
@@ -0,0 +1,232 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.io.DataOutput;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.logging.Level;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriter;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexUtils;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+
+public class FieldPrefixNSMTest extends AbstractBTreeTest {
+
+ private static final int PAGE_SIZE = 32768; // 32K
+ private static final int NUM_PAGES = 40;
+ private static final int MAX_OPEN_FILES = 10;
+ private static final int HYRACKS_FRAME_SIZE = 128;
+
+ private ITupleReference createTuple(IHyracksTaskContext ctx, int f0, int f1, int f2, boolean print)
+ throws HyracksDataException {
+ if (print) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("CREATING: " + f0 + " " + f1 + " " + f2);
+ }
+ }
+
+ ByteBuffer buf = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(3);
+ DataOutput dos = tb.getDataOutput();
+
+ ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(buf);
+ FrameTupleReference tuple = new FrameTupleReference();
+
+ tb.reset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(f2, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(buf, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ return tuple;
+ }
+
+ @Test
+ public void test01() throws Exception {
+
+ // declare fields
+ int fieldCount = 3;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+
+ // declare keys
+ int keyFieldCount = 3;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ cmps[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ cmps[2] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ MultiComparator cmp = new MultiComparator(cmps);
+
+ // just for printing
+ ISerializerDeserializer[] fieldSerdes = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(btreeFileId, 0), false);
+ try {
+
+ ITreeIndexTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
+ BTreeFieldPrefixNSMLeafFrame frame = new BTreeFieldPrefixNSMLeafFrame(tupleWriter);
+ frame.setPage(page);
+ frame.initBuffer((byte) 0);
+ frame.setMultiComparator(cmp);
+ frame.setPrefixTupleCount(0);
+
+ String before = new String();
+ String after = new String();
+
+ int compactFreq = 5;
+ int compressFreq = 5;
+ int smallMax = 10;
+ int numRecords = 1000;
+
+ int[][] savedFields = new int[numRecords][3];
+
+ // insert records with random calls to compact and compress
+ for (int i = 0; i < numRecords; i++) {
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % 100 == 0) {
+ LOGGER.info("INSERTING " + (i + 1) + " / " + numRecords);
+ }
+ }
+
+ int a = rnd.nextInt() % smallMax;
+ int b = rnd.nextInt() % smallMax;
+ int c = i;
+
+ ITupleReference tuple = createTuple(ctx, a, b, c, false);
+ try {
+ int targetTupleIndex = frame.findInsertTupleIndex(tuple);
+ frame.insert(tuple, targetTupleIndex);
+ } catch (BTreeException e) {
+ e.printStackTrace();
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ savedFields[i][0] = a;
+ savedFields[i][1] = b;
+ savedFields[i][2] = c;
+
+ if (rnd.nextInt() % compactFreq == 0) {
+ before = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
+ frame.compact();
+ after = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
+ Assert.assertEquals(before, after);
+ }
+
+ if (rnd.nextInt() % compressFreq == 0) {
+ before = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
+ frame.compress();
+ after = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
+ Assert.assertEquals(before, after);
+ }
+
+ }
+
+ // delete records with random calls to compact and compress
+ for (int i = 0; i < numRecords; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % 100 == 0) {
+ LOGGER.info("DELETING " + (i + 1) + " / " + numRecords);
+ }
+ }
+
+ ITupleReference tuple = createTuple(ctx, savedFields[i][0], savedFields[i][1], savedFields[i][2], false);
+ try {
+ int tupleIndex = frame.findDeleteTupleIndex(tuple);
+ frame.delete(tuple, tupleIndex);
+ } catch (Exception e) {
+ }
+
+ if (rnd.nextInt() % compactFreq == 0) {
+ before = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
+ frame.compact();
+ after = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
+ Assert.assertEquals(before, after);
+ }
+
+ if (rnd.nextInt() % compressFreq == 0) {
+ before = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
+ frame.compress();
+ after = TreeIndexUtils.printFrameTuples(frame, fieldSerdes);
+ Assert.assertEquals(before, after);
+ }
+ }
+
+ } finally {
+ bufferCache.unpin(page);
+ }
+ }
+
+ public int getPageSize() {
+ return PAGE_SIZE;
+ }
+
+ public int getNumPages() {
+ return NUM_PAGES;
+ }
+
+ public int getHyracksFrameSize() {
+ return HYRACKS_FRAME_SIZE;
+ }
+
+ public int getMaxOpenFiles() {
+ return MAX_OPEN_FILES;
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/InsertTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/InsertTest.java
new file mode 100644
index 0000000..fa3f895
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/InsertTest.java
@@ -0,0 +1,65 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
+
+/**
+ * Tests the BTree insert operation with strings and integer fields using
+ * various numbers of key and payload fields.
+ *
+ * Each tests first fills a BTree with randomly generated tuples.
+ * We compare the following operations against expected results:
+ * 1. Point searches for all tuples.
+ * 2. Ordered scan.
+ * 3. Disk-order scan.
+ * 4. Range search (and prefix search for composite keys).
+ *
+ */
+@SuppressWarnings("rawtypes")
+public class InsertTest extends BTreeTestDriver {
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
+ BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
+ // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ BTreeTestUtils.insertIntTuples(testCtx, numTuplesToInsert, rnd);
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ BTreeTestUtils.insertStringTuples(testCtx, numTuplesToInsert, rnd);
+ }
+
+ BTreeTestUtils.checkPointSearches(testCtx);
+ BTreeTestUtils.checkOrderedScan(testCtx);
+ BTreeTestUtils.checkDiskOrderScan(testCtx);
+
+ BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
+ }
+ testCtx.btree.close();
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Insert";
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
index 7dada06..4031e7f 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
@@ -18,593 +18,432 @@
import java.io.ByteArrayInputStream;
import java.io.DataInput;
import java.io.DataInputStream;
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Random;
import java.util.TreeSet;
+import java.util.logging.Level;
import org.junit.Before;
import org.junit.Test;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeException;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
-import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
+import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
public class RangeSearchCursorTest extends AbstractBTreeTest {
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
+ // Declare fields
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
- // declare fields
- int fieldCount = 2;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(
- typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(
- tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(
- tupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ Random rnd = new Random(50);
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame)leafFrameFactory.createFrame();
- IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame)interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
+ @Before
+ public void setUp() throws HyracksDataException {
+ super.setUp();
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ }
- IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
- ByteBuffer frame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ @Test
+ public void uniqueIndexTest() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("TESTING RANGE SEARCH CURSOR ON UNIQUE INDEX");
+ }
- ISerializerDeserializer[] recDescSers = {
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(),
- recDesc);
- FrameTupleReference tuple = new FrameTupleReference();
+ // declare keys
+ int keyFieldCount = 1;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- Random rnd = new Random(50);
+ MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
- @Before
- public void setUp() {
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(4);
- accessor.reset(frame);
- }
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- @Test
- public void uniqueIndexTest() throws Exception {
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
+ IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- LOGGER.info("TESTING RANGE SEARCH CURSOR ON UNIQUE INDEX");
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
+ BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
- // declare keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory,
- leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
+ // generate keys
+ int numKeys = 50;
+ int maxKey = 1000;
+ TreeSet<Integer> uniqueKeys = new TreeSet<Integer>();
+ ArrayList<Integer> keys = new ArrayList<Integer>();
+ while (uniqueKeys.size() < numKeys) {
+ int key = rnd.nextInt() % maxKey;
+ uniqueKeys.add(key);
+ }
+ for (Integer i : uniqueKeys) {
+ keys.add(i);
+ }
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
+ // insert keys into btree
+ for (int i = 0; i < keys.size(); i++) {
- BTreeOpContext insertOpCtx = btree.createOpContext(IndexOp.INSERT,
- leafFrame, interiorFrame, metaFrame);
+ TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
- // generate keys
- int numKeys = 50;
- int maxKey = 1000;
- TreeSet<Integer> uniqueKeys = new TreeSet<Integer>();
- ArrayList<Integer> keys = new ArrayList<Integer>();
- while (uniqueKeys.size() < numKeys) {
- int key = rnd.nextInt() % maxKey;
- uniqueKeys.add(key);
- }
- for (Integer i : uniqueKeys) {
- keys.add(i);
- }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (BTreeException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
- // insert keys into btree
- for (int i = 0; i < keys.size(); i++) {
+ // btree.printTree(leafFrame, interiorFrame, recDescSers);
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(keys.get(i)
- .intValue(), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(i, dos);
- tb.addFieldEndOffset();
+ int minSearchKey = -100;
+ int maxSearchKey = 100;
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb
- .getSize());
+ // forward searches
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
- tuple.reset(accessor, 0);
+ // backward searches
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, false, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
- try {
- btree.insert(tuple, insertOpCtx);
- } catch (BTreeException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+ btree.close();
+ }
- // btree.printTree(leafFrame, interiorFrame, recDescSers);
+ @Test
+ public void nonUniqueIndexTest() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE INDEX");
+ }
- int minSearchKey = -100;
- int maxSearchKey = 100;
+ // declare keys
+ int keyFieldCount = 2;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- // forward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, true, true, false);
+ MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
- // backward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, true, true, false);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
+ IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- @Test
- public void nonUniqueIndexTest() throws Exception {
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
- LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE INDEX");
+ BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
- // declare keys
- int keyFieldCount = 2;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- cmps[1] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
+ // generate keys
+ int numKeys = 50;
+ int maxKey = 10;
+ ArrayList<Integer> keys = new ArrayList<Integer>();
+ for (int i = 0; i < numKeys; i++) {
+ int k = rnd.nextInt() % maxKey;
+ keys.add(k);
+ }
+ Collections.sort(keys);
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory,
- leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
+ // insert keys into btree
+ for (int i = 0; i < keys.size(); i++) {
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
+ TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
- BTreeOpContext insertOpCtx = btree.createOpContext(IndexOp.INSERT,
- leafFrame, interiorFrame, metaFrame);
+ try {
+ indexAccessor.insert(tuple);
+ } catch (BTreeException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
- // generate keys
- int numKeys = 50;
- int maxKey = 10;
- ArrayList<Integer> keys = new ArrayList<Integer>();
- for (int i = 0; i < numKeys; i++) {
- int k = rnd.nextInt() % maxKey;
- keys.add(k);
- }
- Collections.sort(keys);
+ // btree.printTree(leafFrame, interiorFrame, recDescSers);
- // insert keys into btree
- for (int i = 0; i < keys.size(); i++) {
+ int minSearchKey = -100;
+ int maxSearchKey = 100;
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(keys.get(i)
- .intValue(), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(i, dos);
- tb.addFieldEndOffset();
+ // forward searches
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb
- .getSize());
+ // backward searches
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, false, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
- tuple.reset(accessor, 0);
+ btree.close();
+ }
- try {
- btree.insert(tuple, insertOpCtx);
- } catch (BTreeException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+ @Test
+ public void nonUniqueFieldPrefixIndexTest() throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE FIELD-PREFIX COMPRESSED INDEX");
+ }
- // btree.printTree(leafFrame, interiorFrame, recDescSers);
+ // declare keys
+ int keyFieldCount = 2;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+ cmpFactories[1] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
- int minSearchKey = -100;
- int maxSearchKey = 100;
+ MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
- // forward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, true, true, false);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
- // backward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, true, true, false);
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
+ IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
- @Test
- public void nonUniqueFieldPrefixIndexTest() throws Exception {
+ BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
- LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE FIELD-PREFIX COMPRESSED INDEX");
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
- ITreeIndexFrameFactory leafFrameFactory = new BTreeFieldPrefixNSMLeafFrameFactory(
- tupleWriterFactory);
- IBTreeLeafFrame leafFrame = (IBTreeLeafFrame)leafFrameFactory.createFrame();
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
+ // generate keys
+ int numKeys = 50;
+ int maxKey = 10;
+ ArrayList<Integer> keys = new ArrayList<Integer>();
+ for (int i = 0; i < numKeys; i++) {
+ int k = rnd.nextInt() % maxKey;
+ keys.add(k);
+ }
+ Collections.sort(keys);
- // declare keys
- int keyFieldCount = 2;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- cmps[1] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
-
- MultiComparator cmp = new MultiComparator(typeTraits, cmps);
-
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory,
- leafFrameFactory, cmp);
- btree.create(fileId, leafFrame, metaFrame);
- btree.open(fileId);
-
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
-
- BTreeOpContext insertOpCtx = btree.createOpContext(IndexOp.INSERT,
- leafFrame, interiorFrame, metaFrame);
-
- // generate keys
- int numKeys = 50;
- int maxKey = 10;
- ArrayList<Integer> keys = new ArrayList<Integer>();
- for (int i = 0; i < numKeys; i++) {
- int k = rnd.nextInt() % maxKey;
- keys.add(k);
- }
- Collections.sort(keys);
-
- // insert keys into btree
- for (int i = 0; i < keys.size(); i++) {
-
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(keys.get(i)
- .intValue(), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(i, dos);
- tb.addFieldEndOffset();
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb
- .getSize());
-
- tuple.reset(accessor, 0);
-
- try {
- btree.insert(tuple, insertOpCtx);
- } catch (BTreeException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- // btree.printTree(leafFrame, interiorFrame, recDescSers);
-
- int minSearchKey = -100;
- int maxSearchKey = 100;
-
- // forward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, true, true, true, false);
+ // insert keys into btree
+ for (int i = 0; i < keys.size(); i++) {
- // backward searches
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, true, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, false, true, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, true, false, false);
- performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
- maxSearchKey, false, true, true, false);
+ TupleUtils.createIntegerTuple(tupleBuilder, tuple, keys.get(i), i);
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
- btree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (BTreeException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
- public RangePredicate createRangePredicate(int lk, int hk,
- boolean isForward, boolean lowKeyInclusive,
- boolean highKeyInclusive, MultiComparator cmp,
- ITypeTrait[] typeTraits) throws HyracksDataException {
- // build low and high keys
- ArrayTupleBuilder ktb = new ArrayTupleBuilder(1);
- DataOutput kdos = ktb.getDataOutput();
+ // btree.printTree(leafFrame, interiorFrame, recDescSers);
- ISerializerDeserializer[] keyDescSers = { IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor keyDesc = new RecordDescriptor(keyDescSers);
- IFrameTupleAccessor keyAccessor = new FrameTupleAccessor(ctx
- .getFrameSize(), keyDesc);
- keyAccessor.reset(frame);
+ int minSearchKey = -100;
+ int maxSearchKey = 100;
- appender.reset(frame, true);
+ // forward searches
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, false, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, false, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, true, true, true, false);
- // build and append low key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(lk, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb
- .getSize());
+ // backward searches
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, false, true, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, false, false);
+ performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey, maxSearchKey, false, true, true, false);
- // build and append high key
- ktb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(hk, kdos);
- ktb.addFieldEndOffset();
- appender.append(ktb.getFieldEndOffsets(), ktb.getByteArray(), 0, ktb
- .getSize());
+ btree.close();
+ }
- // create tuplereferences for search keys
- FrameTupleReference lowKey = new FrameTupleReference();
- lowKey.reset(keyAccessor, 0);
+ public RangePredicate createRangePredicate(int lk, int hk, boolean isForward, boolean lowKeyInclusive,
+ boolean highKeyInclusive, MultiComparator cmp) throws HyracksDataException {
- FrameTupleReference highKey = new FrameTupleReference();
- highKey.reset(keyAccessor, 1);
+ // create tuplereferences for search keys
+ ITupleReference lowKey = TupleUtils.createIntegerTuple(lk);
+ ITupleReference highKey = TupleUtils.createIntegerTuple(hk);
- IBinaryComparator[] searchCmps = new IBinaryComparator[1];
- searchCmps[0] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- MultiComparator searchCmp = new MultiComparator(typeTraits, searchCmps);
+ IBinaryComparator[] searchCmps = new IBinaryComparator[1];
+ searchCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ MultiComparator searchCmp = new MultiComparator(searchCmps);
- RangePredicate rangePred = new RangePredicate(isForward, lowKey,
- highKey, lowKeyInclusive, highKeyInclusive, searchCmp,
- searchCmp);
- return rangePred;
- }
+ RangePredicate rangePred = new RangePredicate(isForward, lowKey, highKey, lowKeyInclusive, highKeyInclusive,
+ searchCmp, searchCmp);
+ return rangePred;
+ }
- public void getExpectedResults(ArrayList<Integer> expectedResults,
- ArrayList<Integer> keys, int lk, int hk, boolean isForward,
- boolean lowKeyInclusive, boolean highKeyInclusive) {
+ public void getExpectedResults(ArrayList<Integer> expectedResults, ArrayList<Integer> keys, int lk, int hk,
+ boolean isForward, boolean lowKeyInclusive, boolean highKeyInclusive) {
- // special cases
- if (lk == hk && (!lowKeyInclusive || !highKeyInclusive))
- return;
- if (lk > hk)
- return;
+ // special cases
+ if (lk == hk && (!lowKeyInclusive || !highKeyInclusive))
+ return;
+ if (lk > hk)
+ return;
- if (isForward) {
- for (int i = 0; i < keys.size(); i++) {
- if ((lk == keys.get(i) && lowKeyInclusive)
- || (hk == keys.get(i) && highKeyInclusive)) {
- expectedResults.add(keys.get(i));
- continue;
- }
+ if (isForward) {
+ for (int i = 0; i < keys.size(); i++) {
+ if ((lk == keys.get(i) && lowKeyInclusive) || (hk == keys.get(i) && highKeyInclusive)) {
+ expectedResults.add(keys.get(i));
+ continue;
+ }
- if (lk < keys.get(i) && hk > keys.get(i)) {
- expectedResults.add(keys.get(i));
- continue;
- }
- }
- } else {
- for (int i = keys.size() - 1; i >= 0; i--) {
- if ((lk == keys.get(i) && lowKeyInclusive)
- || (hk == keys.get(i) && highKeyInclusive)) {
- expectedResults.add(keys.get(i));
- continue;
- }
+ if (lk < keys.get(i) && hk > keys.get(i)) {
+ expectedResults.add(keys.get(i));
+ continue;
+ }
+ }
+ } else {
+ for (int i = keys.size() - 1; i >= 0; i--) {
+ if ((lk == keys.get(i) && lowKeyInclusive) || (hk == keys.get(i) && highKeyInclusive)) {
+ expectedResults.add(keys.get(i));
+ continue;
+ }
- if (lk < keys.get(i) && hk > keys.get(i)) {
- expectedResults.add(keys.get(i));
- continue;
- }
- }
- }
- }
+ if (lk < keys.get(i) && hk > keys.get(i)) {
+ expectedResults.add(keys.get(i));
+ continue;
+ }
+ }
+ }
+ }
- public boolean performSearches(ArrayList<Integer> keys, BTree btree,
- IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame,
- int minKey, int maxKey, boolean isForward, boolean lowKeyInclusive,
- boolean highKeyInclusive, boolean printExpectedResults)
- throws Exception {
+ public boolean performSearches(ArrayList<Integer> keys, BTree btree, IBTreeLeafFrame leafFrame,
+ IBTreeInteriorFrame interiorFrame, int minKey, int maxKey, boolean isForward, boolean lowKeyInclusive,
+ boolean highKeyInclusive, boolean printExpectedResults) throws Exception {
- ArrayList<Integer> results = new ArrayList<Integer>();
- ArrayList<Integer> expectedResults = new ArrayList<Integer>();
+ ArrayList<Integer> results = new ArrayList<Integer>();
+ ArrayList<Integer> expectedResults = new ArrayList<Integer>();
- for (int i = minKey; i < maxKey; i++) {
- for (int j = minKey; j < maxKey; j++) {
+ for (int i = minKey; i < maxKey; i++) {
+ for (int j = minKey; j < maxKey; j++) {
- results.clear();
- expectedResults.clear();
+ results.clear();
+ expectedResults.clear();
- int lowKey = i;
- int highKey = j;
+ int lowKey = i;
+ int highKey = j;
- ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame);
- RangePredicate rangePred = createRangePredicate(lowKey,
- highKey, isForward, lowKeyInclusive, highKeyInclusive,
- btree.getMultiComparator(), btree.getMultiComparator()
- .getTypeTraits());
- BTreeOpContext searchOpCtx = btree.createOpContext(
- IndexOp.SEARCH, leafFrame, interiorFrame, null);
- btree.search(rangeCursor, rangePred, searchOpCtx);
+ ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame, false);
+ RangePredicate rangePred = createRangePredicate(lowKey, highKey, isForward, lowKeyInclusive,
+ highKeyInclusive, btree.getMultiComparator());
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+ indexAccessor.search(rangeCursor, rangePred);
- try {
- while (rangeCursor.hasNext()) {
- rangeCursor.next();
- ITupleReference frameTuple = rangeCursor.getTuple();
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- frameTuple.getFieldData(0), frameTuple
- .getFieldStart(0), frameTuple
- .getFieldLength(0));
- DataInput dataIn = new DataInputStream(inStream);
- Integer res = IntegerSerializerDeserializer.INSTANCE
- .deserialize(dataIn);
- results.add(res);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- rangeCursor.close();
- }
+ try {
+ while (rangeCursor.hasNext()) {
+ rangeCursor.next();
+ ITupleReference frameTuple = rangeCursor.getTuple();
+ ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(0),
+ frameTuple.getFieldStart(0), frameTuple.getFieldLength(0));
+ DataInput dataIn = new DataInputStream(inStream);
+ Integer res = IntegerSerializerDeserializer.INSTANCE.deserialize(dataIn);
+ results.add(res);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ rangeCursor.close();
+ }
- getExpectedResults(expectedResults, keys, lowKey, highKey,
- isForward, lowKeyInclusive, highKeyInclusive);
+ getExpectedResults(expectedResults, keys, lowKey, highKey, isForward, lowKeyInclusive, highKeyInclusive);
- if (printExpectedResults) {
- if (expectedResults.size() > 0) {
- char l, u;
+ if (printExpectedResults) {
+ if (expectedResults.size() > 0) {
+ char l, u;
- if (lowKeyInclusive)
- l = '[';
- else
- l = '(';
+ if (lowKeyInclusive)
+ l = '[';
+ else
+ l = '(';
- if (highKeyInclusive)
- u = ']';
- else
- u = ')';
+ if (highKeyInclusive)
+ u = ']';
+ else
+ u = ')';
- LOGGER.info("RANGE: " + l + " " + lowKey + " , "
- + highKey + " " + u);
- StringBuilder strBuilder = new StringBuilder();
- for (Integer r : expectedResults) {
- strBuilder.append(r + " ");
- }
- LOGGER.info(strBuilder.toString());
- }
- }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("RANGE: " + l + " " + lowKey + " , " + highKey + " " + u);
+ }
+ StringBuilder strBuilder = new StringBuilder();
+ for (Integer r : expectedResults) {
+ strBuilder.append(r + " ");
+ }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(strBuilder.toString());
+ }
+ }
+ }
- if (results.size() == expectedResults.size()) {
- for (int k = 0; k < results.size(); k++) {
- if (!results.get(k).equals(expectedResults.get(k))) {
- LOGGER.info("DIFFERENT RESULTS AT: i=" + i
- + " j=" + j + " k=" + k);
- LOGGER.info(results.get(k) + " "
- + expectedResults.get(k));
- return false;
- }
- }
- } else {
- LOGGER.info("UNEQUAL NUMBER OF RESULTS AT: i=" + i
- + " j=" + j);
- LOGGER.info("RESULTS: " + results.size());
- LOGGER.info("EXPECTED RESULTS: "
- + expectedResults.size());
- return false;
- }
- }
- }
+ if (results.size() == expectedResults.size()) {
+ for (int k = 0; k < results.size(); k++) {
+ if (!results.get(k).equals(expectedResults.get(k))) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("DIFFERENT RESULTS AT: i=" + i + " j=" + j + " k=" + k);
+ LOGGER.info(results.get(k) + " " + expectedResults.get(k));
+ }
+ return false;
+ }
+ }
+ } else {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("UNEQUAL NUMBER OF RESULTS AT: i=" + i + " j=" + j);
+ LOGGER.info("RESULTS: " + results.size());
+ LOGGER.info("EXPECTED RESULTS: " + expectedResults.size());
+ }
+ return false;
+ }
+ }
+ }
- return true;
- }
+ return true;
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StatsTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StatsTest.java
new file mode 100644
index 0000000..1ae80d6
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StatsTest.java
@@ -0,0 +1,169 @@
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.Random;
+import java.util.logging.Level;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexBufferCacheWarmup;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStats;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class StatsTest extends AbstractBTreeTest {
+
+ // private static final int PAGE_SIZE = 256;
+ // private static final int NUM_PAGES = 10;
+ // private static final int PAGE_SIZE = 32768;
+ private static final int PAGE_SIZE = 4096;
+ private static final int NUM_PAGES = 1000;
+ private static final int MAX_OPEN_FILES = 10;
+ private static final int HYRACKS_FRAME_SIZE = 128;
+ private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+
+ @Test
+ public void test01() throws Exception {
+
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
+ bufferCache.openFile(fileId);
+
+ // declare fields
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+
+ // declare keys
+ int keyFieldCount = 1;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
+
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
+ IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) interiorFrameFactory.createFrame();
+ ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
+
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
+
+ BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ btree.create(fileId);
+ btree.open(fileId);
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ long start = System.currentTimeMillis();
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("INSERTING INTO TREE");
+ }
+
+ ByteBuffer frame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
+
+ ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(frame);
+ FrameTupleReference tuple = new FrameTupleReference();
+
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+ // 10000
+ for (int i = 0; i < 100000; i++) {
+
+ int f0 = rnd.nextInt() % 100000;
+ int f1 = 5;
+
+ tb.reset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(f0, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(f1, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 10000 == 0) {
+ long end = System.currentTimeMillis();
+ LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
+ }
+ }
+
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
+ btree.getRootPageId());
+ TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("\n" + stats.toString());
+ }
+
+ TreeIndexBufferCacheWarmup bufferCacheWarmup = new TreeIndexBufferCacheWarmup(bufferCache, freePageManager,
+ fileId);
+ bufferCacheWarmup.warmup(leafFrame, metaFrame, new int[] { 1, 2 }, new int[] { 2, 5 });
+
+ btree.close();
+ bufferCache.closeFile(fileId);
+ bufferCache.close();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
index 39bedac..ac4133d 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
@@ -19,26 +19,25 @@
import java.util.LinkedList;
import java.util.List;
import java.util.Random;
+import java.util.logging.Level;
import org.junit.Test;
-import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.storage.common.sync.LatchType;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
public class StorageManagerTest extends AbstractBTreeTest {
private static final int PAGE_SIZE = 256;
private static final int NUM_PAGES = 10;
private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(32768);
+ private static final int HYRACKS_FRAME_SIZE = 32768;
public class PinnedLatchedPage {
public final ICachedPage page;
@@ -87,7 +86,9 @@
private void pinRandomPage() {
int pageId = Math.abs(rnd.nextInt() % maxPages);
- LOGGER.info(workerId + " PINNING PAGE: " + pageId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " PINNING PAGE: " + pageId);
+ }
try {
ICachedPage page = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, pageId), false);
@@ -101,14 +102,18 @@
break;
case FTA_READONLY: {
- LOGGER.info(workerId + " S LATCHING: " + pageId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " S LATCHING: " + pageId);
+ }
page.acquireReadLatch();
latch = LatchType.LATCH_S;
}
break;
case FTA_WRITEONLY: {
- LOGGER.info(workerId + " X LATCHING: " + pageId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " X LATCHING: " + pageId);
+ }
page.acquireWriteLatch();
latch = LatchType.LATCH_X;
}
@@ -116,11 +121,15 @@
case FTA_MIXED: {
if (rnd.nextInt() % 2 == 0) {
- LOGGER.info(workerId + " S LATCHING: " + pageId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " S LATCHING: " + pageId);
+ }
page.acquireReadLatch();
latch = LatchType.LATCH_S;
} else {
- LOGGER.info(workerId + " X LATCHING: " + pageId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " X LATCHING: " + pageId);
+ }
page.acquireWriteLatch();
latch = LatchType.LATCH_X;
}
@@ -143,14 +152,20 @@
if (plPage.latch != null) {
if (plPage.latch == LatchType.LATCH_S) {
- LOGGER.info(workerId + " S UNLATCHING: " + plPage.pageId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " S UNLATCHING: " + plPage.pageId);
+ }
plPage.page.releaseReadLatch();
} else {
- LOGGER.info(workerId + " X UNLATCHING: " + plPage.pageId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " X UNLATCHING: " + plPage.pageId);
+ }
plPage.page.releaseWriteLatch();
}
}
- LOGGER.info(workerId + " UNPINNING PAGE: " + plPage.pageId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " UNPINNING PAGE: " + plPage.pageId);
+ }
bufferCache.unpin(plPage.page);
pinnedPages.remove(index);
@@ -160,7 +175,9 @@
}
private void openFile() {
- LOGGER.info(workerId + " OPENING FILE: " + fileId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " OPENING FILE: " + fileId);
+ }
try {
bufferCache.openFile(fileId);
fileIsOpen = true;
@@ -170,7 +187,9 @@
}
private void closeFile() {
- LOGGER.info(workerId + " CLOSING FILE: " + fileId);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " CLOSING FILE: " + fileId);
+ }
try {
bufferCache.closeFile(fileId);
fileIsOpen = false;
@@ -187,7 +206,9 @@
while (loopCount < maxLoopCount) {
loopCount++;
- LOGGER.info(workerId + " LOOP: " + loopCount + "/" + maxLoopCount);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(workerId + " LOOP: " + loopCount + "/" + maxLoopCount);
+ }
if (fileIsOpen) {
@@ -255,4 +276,20 @@
bufferCache.close();
}
+
+ public int getPageSize() {
+ return PAGE_SIZE;
+ }
+
+ public int getNumPages() {
+ return NUM_PAGES;
+ }
+
+ public int getHyracksFrameSize() {
+ return HYRACKS_FRAME_SIZE;
+ }
+
+ public int getMaxOpenFiles() {
+ return MAX_OPEN_FILES;
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateSearchTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateSearchTest.java
new file mode 100644
index 0000000..5d43ae1
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateSearchTest.java
@@ -0,0 +1,152 @@
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import java.util.Random;
+import java.util.logging.Level;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.btree.util.AbstractBTreeTest;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.common.util.IndexUtils;
+
+public class UpdateSearchTest extends AbstractBTreeTest {
+
+ // Update scan test on fixed-length tuples.
+ @Test
+ public void test01() throws Exception {
+ // declare fields
+ int fieldCount = 2;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+
+ // declare keys
+ int keyFieldCount = 1;
+ IBinaryComparatorFactory[] cmpFactories = new IBinaryComparatorFactory[keyFieldCount];
+ cmpFactories[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY);
+
+ MultiComparator cmp = IndexUtils.createMultiComparator(cmpFactories);
+
+ ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
+
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
+ BTree btree = new BTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ long start = System.currentTimeMillis();
+
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("INSERTING INTO TREE");
+ }
+
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference insertTuple = new ArrayTupleReference();
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+
+ int numInserts = 10000;
+ for (int i = 0; i < 10000; i++) {
+ int f0 = rnd.nextInt() % 10000;
+ int f1 = 5;
+ TupleUtils.createIntegerTuple(tb, insertTuple, f0, f1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 10000 == 0) {
+ long end = System.currentTimeMillis();
+ LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
+ }
+ }
+
+ try {
+ indexAccessor.insert(insertTuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ long end = System.currentTimeMillis();
+ long duration = end - start;
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("DURATION: " + duration);
+ }
+
+ // Update scan.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("UPDATE SCAN:");
+ }
+ // Set the cursor to X latch nodes.
+ ITreeIndexCursor updateScanCursor = new BTreeRangeSearchCursor(leafFrame, true);
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ indexAccessor.search(updateScanCursor, nullPred);
+ try {
+ while (updateScanCursor.hasNext()) {
+ updateScanCursor.next();
+ ITupleReference tuple = updateScanCursor.getTuple();
+ // Change the value field.
+ IntegerSerializerDeserializer.putInt(10, tuple.getFieldData(1), tuple.getFieldStart(1));
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ updateScanCursor.close();
+ }
+
+ // Ordered scan to verify the values.
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("ORDERED SCAN:");
+ }
+ // Set the cursor to X latch nodes.
+ ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame, true);
+ indexAccessor.search(scanCursor, nullPred);
+ try {
+ while (scanCursor.hasNext()) {
+ scanCursor.next();
+ ITupleReference tuple = scanCursor.getTuple();
+ String rec = TupleUtils.printTuple(tuple, recDescSers);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ scanCursor.close();
+ }
+ btree.close();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateTest.java
new file mode 100644
index 0000000..b40539f
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/UpdateTest.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestContext;
+import edu.uci.ics.hyracks.storage.am.btree.util.BTreeTestUtils;
+
+@SuppressWarnings("rawtypes")
+public class UpdateTest extends BTreeTestDriver {
+ private static final int numUpdateRounds = 3;
+
+ @Override
+ protected void runTest(ISerializerDeserializer[] fieldSerdes, int numKeys, BTreeLeafFrameType leafType, ITupleReference lowKey, ITupleReference highKey, ITupleReference prefixLowKey, ITupleReference prefixHighKey) throws Exception {
+ // This is a noop because we can only update non-key fields.
+ if (fieldSerdes.length == numKeys) {
+ return;
+ }
+
+ BTreeTestContext testCtx = BTreeTestUtils.createBTreeTestContext(bufferCache, btreeFileId, fieldSerdes, numKeys, leafType);
+
+ // We assume all fieldSerdes are of the same type. Check the first one to determine which field types to generate.
+ if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
+ BTreeTestUtils.insertIntTuples(testCtx, numTuplesToInsert, rnd);
+ } else if (fieldSerdes[0] instanceof UTF8StringSerializerDeserializer) {
+ BTreeTestUtils.insertStringTuples(testCtx, numTuplesToInsert, rnd);
+ }
+
+ int numTuplesPerDeleteRound = (int)Math.ceil((float)testCtx.checkTuples.size() / (float)numUpdateRounds);
+ for(int j = 0; j < numUpdateRounds; j++) {
+ BTreeTestUtils.updateTuples(testCtx, numTuplesPerDeleteRound, rnd);
+
+ BTreeTestUtils.checkPointSearches(testCtx);
+ BTreeTestUtils.checkOrderedScan(testCtx);
+ BTreeTestUtils.checkDiskOrderScan(testCtx);
+ BTreeTestUtils.checkRangeSearch(testCtx, lowKey, highKey, true, true);
+ if (prefixLowKey != null && prefixHighKey != null) {
+ BTreeTestUtils.checkRangeSearch(testCtx, prefixLowKey, prefixHighKey, true, true);
+ }
+ }
+ }
+
+ @Override
+ protected String getTestOpName() {
+ return "Update";
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
new file mode 100644
index 0000000..9630a1b
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/AbstractBTreeTest.java
@@ -0,0 +1,76 @@
+package edu.uci.ics.hyracks.storage.am.btree.util;
+
+import java.io.File;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Random;
+import java.util.logging.Logger;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public abstract class AbstractBTreeTest {
+ protected static final Logger LOGGER = Logger.getLogger(AbstractBTreeTest.class.getName());
+ public static final long RANDOM_SEED = 50;
+
+ private static final int PAGE_SIZE = 256;
+ private static final int NUM_PAGES = 10;
+ private static final int MAX_OPEN_FILES = 10;
+ private static final int HYRACKS_FRAME_SIZE = 128;
+
+ protected IHyracksTaskContext ctx;
+ protected IBufferCache bufferCache;
+ protected int btreeFileId;
+
+ protected final Random rnd = new Random();
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected String fileName;
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ fileName = tmpDir + sep + simpleDateFormat.format(new Date());
+ ctx = TestUtils.create(getHyracksFrameSize());
+ TestStorageManagerComponentHolder.init(getPageSize(), getNumPages(), getMaxOpenFiles());
+ bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ btreeFileId = fmp.lookupFileId(file);
+ bufferCache.openFile(btreeFileId);
+ rnd.setSeed(RANDOM_SEED);
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.close();
+ File f = new File(fileName);
+ f.deleteOnExit();
+ }
+
+ public int getPageSize() {
+ return PAGE_SIZE;
+ }
+
+ public int getNumPages() {
+ return NUM_PAGES;
+ }
+
+ public int getHyracksFrameSize() {
+ return HYRACKS_FRAME_SIZE;
+ }
+
+ public int getMaxOpenFiles() {
+ return MAX_OPEN_FILES;
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
new file mode 100644
index 0000000..f1b03c1
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestContext.java
@@ -0,0 +1,63 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.util;
+
+import java.util.TreeSet;
+
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+@SuppressWarnings("rawtypes")
+public final class BTreeTestContext {
+ public final ISerializerDeserializer[] fieldSerdes;
+ public final IBufferCache bufferCache;
+ public final BTree btree;
+ public final IBTreeLeafFrame leafFrame;
+ public final IBTreeInteriorFrame interiorFrame;
+ public final ITreeIndexMetaDataFrame metaFrame;
+ public final ArrayTupleBuilder tupleBuilder;
+ public final ArrayTupleReference tuple = new ArrayTupleReference();
+ public final TreeSet<CheckTuple> checkTuples = new TreeSet<CheckTuple>();
+ public final ITreeIndexAccessor indexAccessor;
+
+ public BTreeTestContext(IBufferCache bufferCache, ISerializerDeserializer[] fieldSerdes, BTree btree,
+ IBTreeLeafFrame leafFrame, IBTreeInteriorFrame interiorFrame, ITreeIndexMetaDataFrame metaFrame,
+ ITreeIndexAccessor indexAccessor) {
+ this.bufferCache = bufferCache;
+ this.fieldSerdes = fieldSerdes;
+ this.btree = btree;
+ this.leafFrame = leafFrame;
+ this.interiorFrame = interiorFrame;
+ this.metaFrame = metaFrame;
+ this.indexAccessor = indexAccessor;
+ this.tupleBuilder = new ArrayTupleBuilder(fieldSerdes.length);
+ }
+
+ public int getFieldCount() {
+ return fieldSerdes.length;
+ }
+
+ public int getKeyFieldCount() {
+ return btree.getMultiComparator().getKeyFieldCount();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestUtils.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestUtils.java
new file mode 100644
index 0000000..b5186b0
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/BTreeTestUtils.java
@@ -0,0 +1,505 @@
+package edu.uci.ics.hyracks.storage.am.btree.util;
+
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.util.Iterator;
+import java.util.NavigableSet;
+import java.util.Random;
+import java.util.TreeSet;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.SerdeUtils;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.exceptions.BTreeDuplicateKeyException;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeLeafFrameType;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
+import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
+import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+
+@SuppressWarnings("rawtypes")
+public class BTreeTestUtils {
+ private static final Logger LOGGER = Logger.getLogger(BTreeTestUtils.class.getName());
+
+ public static BTreeTestContext createBTreeTestContext(IBufferCache bufferCache, int btreeFileId, ISerializerDeserializer[] fieldSerdes, int numKeyFields, BTreeLeafFrameType leafType) throws Exception {
+ ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes, fieldSerdes.length);
+ IBinaryComparator[] cmps = SerdeUtils.serdesToComparators(fieldSerdes, numKeyFields);
+
+ BTree btree = BTreeUtils.createBTree(bufferCache, btreeFileId, typeTraits, cmps, leafType);
+ btree.create(btreeFileId);
+ btree.open(btreeFileId);
+ ITreeIndexAccessor indexAccessor = btree.createAccessor();
+
+ IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) btree.getLeafFrameFactory().createFrame();
+ IBTreeInteriorFrame interiorFrame = (IBTreeInteriorFrame) btree.getInteriorFrameFactory().createFrame();
+ ITreeIndexMetaDataFrame metaFrame = btree.getFreePageManager().getMetaDataFrameFactory().createFrame();
+ BTreeTestContext testCtx = new BTreeTestContext(bufferCache, fieldSerdes, btree, leafFrame, interiorFrame, metaFrame, indexAccessor);
+ return testCtx;
+ }
+
+ private static void compareActualAndExpected(ITupleReference actual, CheckTuple expected, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
+ for (int i = 0; i < fieldSerdes.length; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(
+ actual.getFieldData(i), actual.getFieldStart(i),
+ actual.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Object actualObj = fieldSerdes[i].deserialize(dataIn);
+ if (!actualObj.equals(expected.get(i))) {
+ fail("Actual and expected fields do not match.\nExpected: " + expected.get(i) + "\nActual : " + actualObj);
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private static CheckTuple createCheckTupleFromTuple(ITupleReference tuple, ISerializerDeserializer[] fieldSerdes, int numKeys) throws HyracksDataException {
+ CheckTuple checkTuple = new CheckTuple(fieldSerdes.length, numKeys);
+ int fieldCount = Math.min(fieldSerdes.length, tuple.getFieldCount());
+ for (int i = 0; i < fieldCount; i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(
+ tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Comparable fieldObj = (Comparable)fieldSerdes[i].deserialize(dataIn);
+ checkTuple.add(fieldObj);
+ }
+ return checkTuple;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void createTupleFromCheckTuple(CheckTuple checkTuple, ArrayTupleBuilder tupleBuilder, ArrayTupleReference tuple, ISerializerDeserializer[] fieldSerdes) throws HyracksDataException {
+ int fieldCount = tupleBuilder.getFieldEndOffsets().length;
+ DataOutput dos = tupleBuilder.getDataOutput();
+ tupleBuilder.reset();
+ for (int i = 0; i < fieldCount; i++) {
+ fieldSerdes[i].serialize(checkTuple.get(i), dos);
+ tupleBuilder.addFieldEndOffset();
+ }
+ tuple.reset(tupleBuilder.getFieldEndOffsets(), tupleBuilder.getByteArray());
+ }
+
+ public static void checkOrderedScan(BTreeTestContext testCtx) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Ordered Scan.");
+ }
+ ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(testCtx.leafFrame, false);
+ RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
+ testCtx.indexAccessor.search(scanCursor, nullPred);
+ Iterator<CheckTuple> checkIter = testCtx.checkTuples.iterator();
+ int actualCount = 0;
+ try {
+ while (scanCursor.hasNext()) {
+ if (!checkIter.hasNext()) {
+ fail("Ordered scan returned more answers than expected.\nExpected: " + testCtx.checkTuples.size());
+ }
+ scanCursor.next();
+ CheckTuple expectedTuple = checkIter.next();
+ ITupleReference tuple = scanCursor.getTuple();
+ compareActualAndExpected(tuple, expectedTuple, testCtx.fieldSerdes);
+ actualCount++;
+ }
+ if (actualCount < testCtx.checkTuples.size()) {
+ fail("Ordered scan returned fewer answers than expected.\nExpected: " + testCtx.checkTuples.size() + "\nActual : " + actualCount);
+ }
+ } finally {
+ scanCursor.close();
+ }
+ }
+
+ public static void checkDiskOrderScan(BTreeTestContext testCtx) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Disk-Order Scan.");
+ }
+ ITreeIndexCursor diskOrderCursor = new TreeDiskOrderScanCursor(testCtx.leafFrame);
+ testCtx.indexAccessor.diskOrderScan(diskOrderCursor);
+ int actualCount = 0;
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference tuple = diskOrderCursor.getTuple();
+ CheckTuple checkTuple = createCheckTupleFromTuple(tuple, testCtx.fieldSerdes, testCtx.btree.getMultiComparator().getKeyFieldCount());
+ if (!testCtx.checkTuples.contains(checkTuple)) {
+ fail("Disk-order scan returned unexpected answer: " + checkTuple.toString());
+ }
+ actualCount++;
+ }
+ if (actualCount < testCtx.checkTuples.size()) {
+ fail("Disk-order scan returned fewer answers than expected.\nExpected: " + testCtx.checkTuples.size() + "\nActual : " + actualCount);
+ }
+ if (actualCount > testCtx.checkTuples.size()) {
+ fail("Disk-order scan returned more answers than expected.\nExpected: " + testCtx.checkTuples.size() + "\nActual : " + actualCount);
+ }
+ } finally {
+ diskOrderCursor.close();
+ }
+ }
+
+ public static void checkRangeSearch(BTreeTestContext testCtx, ITupleReference lowKey, ITupleReference highKey, boolean lowKeyInclusive, boolean highKeyInclusive) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Range Search.");
+ }
+ MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), lowKey);
+ MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), highKey);
+ ITreeIndexCursor searchCursor = new BTreeRangeSearchCursor(testCtx.leafFrame, false);
+ RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, lowKeyInclusive, highKeyInclusive, lowKeyCmp, highKeyCmp);
+ testCtx.indexAccessor.search(searchCursor, rangePred);
+ // Get the subset of elements from the expected set within given key range.
+ CheckTuple lowKeyCheck = createCheckTupleFromTuple(lowKey, testCtx.fieldSerdes, lowKeyCmp.getKeyFieldCount());
+ CheckTuple highKeyCheck = createCheckTupleFromTuple(highKey, testCtx.fieldSerdes, highKeyCmp.getKeyFieldCount());
+ NavigableSet<CheckTuple> expectedSubset = null;
+ if (lowKeyCmp.getKeyFieldCount() < testCtx.btree.getMultiComparator().getKeyFieldCount() ||
+ highKeyCmp.getKeyFieldCount() < testCtx.btree.getMultiComparator().getKeyFieldCount()) {
+ // Searching on a key prefix (low key or high key or both).
+ expectedSubset = getPrefixExpectedSubset(testCtx.checkTuples, lowKeyCheck, highKeyCheck);
+ } else {
+ // Searching on all key fields.
+ expectedSubset = testCtx.checkTuples.subSet(lowKeyCheck, lowKeyInclusive, highKeyCheck, highKeyInclusive);
+ }
+ Iterator<CheckTuple> checkIter = expectedSubset.iterator();
+ int actualCount = 0;
+ try {
+ while (searchCursor.hasNext()) {
+ if (!checkIter.hasNext()) {
+ fail("Range search returned more answers than expected.\nExpected: " + expectedSubset.size());
+ }
+ searchCursor.next();
+ CheckTuple expectedTuple = checkIter.next();
+ ITupleReference tuple = searchCursor.getTuple();
+ compareActualAndExpected(tuple, expectedTuple, testCtx.fieldSerdes);
+ actualCount++;
+ }
+ if (actualCount < expectedSubset.size()) {
+ fail("Range search returned fewer answers than expected.\nExpected: " + expectedSubset.size() + "\nActual : " + actualCount);
+ }
+ } finally {
+ searchCursor.close();
+ }
+ }
+
+ public static void checkPointSearches(BTreeTestContext testCtx) throws Exception {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Testing Point Searches On All Expected Keys.");
+ }
+ ITreeIndexCursor searchCursor = new BTreeRangeSearchCursor(testCtx.leafFrame, false);
+
+ ArrayTupleBuilder lowKeyBuilder = new ArrayTupleBuilder(testCtx.btree.getMultiComparator().getKeyFieldCount());
+ ArrayTupleReference lowKey = new ArrayTupleReference();
+ ArrayTupleBuilder highKeyBuilder = new ArrayTupleBuilder(testCtx.btree.getMultiComparator().getKeyFieldCount());
+ ArrayTupleReference highKey = new ArrayTupleReference();
+ RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, null, null);
+
+ // Iterate through expected tuples, and perform a point search in the BTree to verify the tuple can be reached.
+ for (CheckTuple checkTuple : testCtx.checkTuples) {
+ createTupleFromCheckTuple(checkTuple, lowKeyBuilder, lowKey, testCtx.fieldSerdes);
+ createTupleFromCheckTuple(checkTuple, highKeyBuilder, highKey, testCtx.fieldSerdes);
+ MultiComparator lowKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), lowKey);
+ MultiComparator highKeyCmp = BTreeUtils.getSearchMultiComparator(testCtx.btree.getMultiComparator(), highKey);
+
+ rangePred.setLowKey(lowKey, true);
+ rangePred.setHighKey(highKey, true);
+ rangePred.setLowKeyComparator(lowKeyCmp);
+ rangePred.setHighKeyComparator(highKeyCmp);
+
+ testCtx.indexAccessor.search(searchCursor, rangePred);
+
+ try {
+ // We expect exactly one answer.
+ if (searchCursor.hasNext()) {
+ searchCursor.next();
+ ITupleReference tuple = searchCursor.getTuple();
+ compareActualAndExpected(tuple, checkTuple, testCtx.fieldSerdes);
+ }
+ if (searchCursor.hasNext()) {
+ fail("Point search returned more than one answer.");
+ }
+ } finally {
+ searchCursor.close();
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ // Create a new TreeSet containing the elements satisfying the prefix search.
+ // Implementing prefix search by changing compareTo() in CheckTuple does not work.
+ public static TreeSet<CheckTuple> getPrefixExpectedSubset(TreeSet<CheckTuple> checkTuples, CheckTuple lowKey, CheckTuple highKey) {
+ TreeSet<CheckTuple> expectedSubset = new TreeSet<CheckTuple>();
+ Iterator<CheckTuple> iter = checkTuples.iterator();
+ while(iter.hasNext()) {
+ CheckTuple t = iter.next();
+ boolean geLowKey = true;
+ boolean leHighKey = true;
+ for (int i = 0; i < lowKey.getNumKeys(); i++) {
+ if (t.get(i).compareTo(lowKey.get(i)) < 0) {
+ geLowKey = false;
+ break;
+ }
+ }
+ for (int i = 0; i < highKey.getNumKeys(); i++) {
+ if (t.get(i).compareTo(highKey.get(i)) > 0) {
+ leHighKey = false;
+ break;
+ }
+ }
+ if (geLowKey && leHighKey) {
+ expectedSubset.add(t);
+ }
+ }
+ return expectedSubset;
+ }
+
+ public static void insertIntTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = testCtx.getFieldCount();
+ int numKeyFields = testCtx.getKeyFieldCount();
+ int[] tupleValues = new int[testCtx.getFieldCount()];
+ // Scale range of values according to number of keys.
+ // For example, for 2 keys we want the square root of numTuples, for 3 keys the cube root of numTuples, etc.
+ int maxValue = (int)Math.ceil(Math.pow(numTuples, 1.0/(double)numKeyFields));
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ tupleValues[j] = rnd.nextInt() % maxValue;
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ tupleValues[j] = j;
+ }
+ TupleUtils.createIntegerTuple(testCtx.tupleBuilder, testCtx.tuple, tupleValues);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ try {
+ testCtx.indexAccessor.insert(testCtx.tuple);
+ // Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
+ CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(fieldCount, numKeyFields);
+ for(int v : tupleValues) {
+ checkTuple.add(v);
+ }
+ testCtx.checkTuples.add(checkTuple);
+ } catch (BTreeDuplicateKeyException e) {
+ // Ignore duplicate key insertions.
+ }
+ }
+ }
+
+ public static void insertStringTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = testCtx.getFieldCount();
+ int numKeyFields = testCtx.getKeyFieldCount();
+ Object[] tupleValues = new Object[fieldCount];
+ for (int i = 0; i < numTuples; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Inserting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ int length = (Math.abs(rnd.nextInt()) % 10) + 1;
+ tupleValues[j] = getRandomString(length, rnd);
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ tupleValues[j] = getRandomString(5, rnd);
+ }
+ TupleUtils.createTuple(testCtx.tupleBuilder, testCtx.tuple, testCtx.fieldSerdes, tupleValues);
+ try {
+ testCtx.indexAccessor.insert(testCtx.tuple);
+ // Set expected values. Do this only after insertion succeeds because we ignore duplicate keys.
+ CheckTuple<String> checkTuple = new CheckTuple<String>(fieldCount, numKeyFields);
+ for(Object v : tupleValues) {
+ checkTuple.add((String)v);
+ }
+ testCtx.checkTuples.add(checkTuple);
+ } catch (BTreeDuplicateKeyException e) {
+ // Ignore duplicate key insertions.
+ }
+ }
+ }
+
+ public static void bulkLoadIntTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = testCtx.getFieldCount();
+ int numKeyFields = testCtx.getKeyFieldCount();
+ int[] tupleValues = new int[testCtx.getFieldCount()];
+ int maxValue = (int)Math.ceil(Math.pow(numTuples, 1.0/(double)numKeyFields));
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ tupleValues[j] = rnd.nextInt() % maxValue;
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ tupleValues[j] = j;
+ }
+
+ // Set expected values. We also use these as the pre-sorted stream for bulk loading.
+ CheckTuple<Integer> checkTuple = new CheckTuple<Integer>(fieldCount, numKeyFields);
+ for(int v : tupleValues) {
+ checkTuple.add(v);
+ }
+ testCtx.checkTuples.add(checkTuple);
+ }
+
+ bulkLoadCheckTuples(testCtx, numTuples);
+ }
+
+ public static void bulkLoadStringTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = testCtx.getFieldCount();
+ int numKeyFields = testCtx.getKeyFieldCount();
+ String[] tupleValues = new String[fieldCount];
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ for (int j = 0; j < numKeyFields; j++) {
+ int length = (Math.abs(rnd.nextInt()) % 10) + 1;
+ tupleValues[j] = getRandomString(length, rnd);
+ }
+ // Set values.
+ for (int j = numKeyFields; j < fieldCount; j++) {
+ tupleValues[j] = getRandomString(5, rnd);
+ }
+ // Set expected values. We also use these as the pre-sorted stream for bulk loading.
+ CheckTuple<String> checkTuple = new CheckTuple<String>(fieldCount, numKeyFields);
+ for(String v : tupleValues) {
+ checkTuple.add(v);
+ }
+ testCtx.checkTuples.add(checkTuple);
+ }
+
+ bulkLoadCheckTuples(testCtx, numTuples);
+ }
+
+ private static void bulkLoadCheckTuples(BTreeTestContext testCtx, int numTuples) throws HyracksDataException, TreeIndexException, PageAllocationException {
+ int fieldCount = testCtx.getFieldCount();
+ ArrayTupleBuilder tupleBuilder = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference tuple = new ArrayTupleReference();
+ // Perform bulk load.
+ IIndexBulkLoadContext bulkLoadCtx = testCtx.btree.beginBulkLoad(0.7f);
+ int c = 1;
+ for (CheckTuple checkTuple : testCtx.checkTuples) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (c % (numTuples / 10) == 0) {
+ LOGGER.info("Bulk Loading Tuple " + c + "/" + numTuples);
+ }
+ }
+ createTupleFromCheckTuple(checkTuple, tupleBuilder, tuple, testCtx.fieldSerdes);
+ testCtx.btree.bulkLoadAddTuple(tuple, bulkLoadCtx);
+ c++;
+ }
+ testCtx.btree.endBulkLoad(bulkLoadCtx);
+ }
+
+ public static void deleteTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
+ ArrayTupleBuilder deleteTupleBuilder = new ArrayTupleBuilder(testCtx.btree.getMultiComparator().getKeyFieldCount());
+ ArrayTupleReference deleteTuple = new ArrayTupleReference();
+ int numCheckTuples = testCtx.checkTuples.size();
+ // Copy CheckTuple references into array, so we can randomly pick from there.
+ CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
+ int idx = 0;
+ for (CheckTuple checkTuple : testCtx.checkTuples) {
+ checkTuples[idx++] = checkTuple;
+ }
+ for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Deleting Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
+ CheckTuple checkTuple = checkTuples[checkTupleIdx];
+ createTupleFromCheckTuple(checkTuple, deleteTupleBuilder, deleteTuple, testCtx.fieldSerdes);
+ testCtx.indexAccessor.delete(deleteTuple);
+
+ // Remove check tuple from expected results.
+ testCtx.checkTuples.remove(checkTuple);
+
+ // Swap with last "valid" CheckTuple.
+ CheckTuple tmp = checkTuples[numCheckTuples - 1];
+ checkTuples[numCheckTuples - 1] = checkTuple;
+ checkTuples[checkTupleIdx] = tmp;
+ numCheckTuples--;
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public static void updateTuples(BTreeTestContext testCtx, int numTuples, Random rnd) throws Exception {
+ int fieldCount = testCtx.btree.getFieldCount();
+ int keyFieldCount = testCtx.btree.getMultiComparator().getKeyFieldCount();
+ // This is a noop because we can only update non-key fields.
+ if (fieldCount == keyFieldCount) {
+ return;
+ }
+ ArrayTupleBuilder updateTupleBuilder = new ArrayTupleBuilder(fieldCount);
+ ArrayTupleReference updateTuple = new ArrayTupleReference();
+ int numCheckTuples = testCtx.checkTuples.size();
+ // Copy CheckTuple references into array, so we can randomly pick from there.
+ CheckTuple[] checkTuples = new CheckTuple[numCheckTuples];
+ int idx = 0;
+ for (CheckTuple checkTuple : testCtx.checkTuples) {
+ checkTuples[idx++] = checkTuple;
+ }
+ for (int i = 0; i < numTuples && numCheckTuples > 0; i++) {
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if ((i + 1) % (numTuples / Math.min(10, numTuples)) == 0) {
+ LOGGER.info("Updating Tuple " + (i + 1) + "/" + numTuples);
+ }
+ }
+ int checkTupleIdx = Math.abs(rnd.nextInt() % numCheckTuples);
+ CheckTuple checkTuple = checkTuples[checkTupleIdx];
+ // Update check tuple's non-key fields.
+ for (int j = keyFieldCount; j < fieldCount; j++) {
+ Comparable newValue = getRandomUpdateValue(testCtx.fieldSerdes[j], rnd);
+ checkTuple.set(j, newValue);
+ }
+
+ createTupleFromCheckTuple(checkTuple, updateTupleBuilder, updateTuple, testCtx.fieldSerdes);
+ testCtx.indexAccessor.update(updateTuple);
+
+ // Swap with last "valid" CheckTuple.
+ CheckTuple tmp = checkTuples[numCheckTuples - 1];
+ checkTuples[numCheckTuples - 1] = checkTuple;
+ checkTuples[checkTupleIdx] = tmp;
+ numCheckTuples--;
+ }
+ }
+
+ private static Comparable getRandomUpdateValue(ISerializerDeserializer serde, Random rnd) {
+ if (serde instanceof IntegerSerializerDeserializer) {
+ return Integer.valueOf(rnd.nextInt());
+ } else if (serde instanceof UTF8StringSerializerDeserializer) {
+ return getRandomString(10, rnd);
+ }
+ return null;
+ }
+
+ public static String getRandomString(int length, Random rnd) {
+ String s = Long.toHexString(Double.doubleToLongBits(rnd.nextDouble()));
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < s.length() && i < length; i++) {
+ strBuilder.append(s.charAt(Math.abs(rnd.nextInt()) % s.length()));
+ }
+ return strBuilder.toString();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/CheckTuple.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/CheckTuple.java
new file mode 100644
index 0000000..f945ab9
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/util/CheckTuple.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.btree.util;
+
+@SuppressWarnings({"rawtypes", "unchecked"})
+public class CheckTuple<T extends Comparable<T>> implements Comparable<T> {
+ private final int numKeys;
+ private final Comparable[] tuple;
+ private int pos;
+
+ public CheckTuple(int numFields, int numKeys) {
+ this.numKeys = numKeys;
+ this.tuple = new Comparable[numFields];
+ pos = 0;
+ }
+
+ public void add(T e) {
+ tuple[pos++] = e;
+ }
+
+ @Override
+ public int compareTo(T o) {
+ CheckTuple<T> other = (CheckTuple<T>)o;
+ for (int i = 0; i < numKeys; i++) {
+ int cmp = tuple[i].compareTo(other.get(i));
+ if (cmp != 0) {
+ return cmp;
+ }
+ }
+ return 0;
+ }
+
+ public T get(int idx) {
+ return (T)tuple[idx];
+ }
+
+ public void set(int idx, T e) {
+ tuple[idx] = e;
+ }
+
+ public int size() {
+ return tuple.length;
+ }
+
+ public int getNumKeys() {
+ return numKeys;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < tuple.length; i++) {
+ strBuilder.append(tuple[i].toString());
+ if (i != tuple.length-1) {
+ strBuilder.append(" ");
+ }
+ }
+ return strBuilder.toString();
+ }
+}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/.classpath b/hyracks-tests/hyracks-storage-am-invertedindex-test/.classpath
deleted file mode 100644
index f2cc5f7..0000000
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/.project b/hyracks-tests/hyracks-storage-am-invertedindex-test/.project
deleted file mode 100644
index f60b2f9..0000000
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-storage-am-invertedindex-test</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/.settings/org.eclipse.jdt.core.prefs b/hyracks-tests/hyracks-storage-am-invertedindex-test/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 375e12e..0000000
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:07 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/.settings/org.maven.ide.eclipse.prefs b/hyracks-tests/hyracks-storage-am-invertedindex-test/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 99b89a6..0000000
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Jan 06 11:27:16 PST 2011
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/pom.xml b/hyracks-tests/hyracks-storage-am-invertedindex-test/pom.xml
index c098506..50bea01 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/pom.xml
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-invertedindex-test</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -19,6 +19,7 @@
<configuration>
<source>1.6</source>
<target>1.6</target>
+ <encoding>UTF-8</encoding>
</configuration>
</plugin>
</plugins>
@@ -27,20 +28,20 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-invertedindex</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-test-support</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>test</scope>
</dependency>
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
index ea97511..7b6bd9f 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex;
import java.io.DataOutput;
@@ -13,17 +28,17 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
@@ -43,6 +58,7 @@
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.IBinaryTokenizer;
import edu.uci.ics.hyracks.storage.am.invertedindex.tokenizers.ITokenFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.util.InvertedIndexUtils;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
@@ -64,17 +80,17 @@
protected FileReference btreeFile = new FileReference(new File(btreeFileName));
protected int btreeFileId;
- // declare btree fields
- protected int fieldCount = 5;
- protected ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
+ // declare token type traits
+ protected ITypeTraits[] tokenTypeTraits = new ITypeTraits[] { UTF8StringPointable.TYPE_TRAITS };
+ protected ITypeTraits[] btreeTypeTraits = InvertedIndexUtils.getBTreeTypeTraits(tokenTypeTraits);
// declare btree keys
protected int btreeKeyFieldCount = 1;
protected IBinaryComparator[] btreeBinCmps = new IBinaryComparator[btreeKeyFieldCount];
- protected MultiComparator btreeCmp = new MultiComparator(typeTraits, btreeBinCmps);
+ protected MultiComparator btreeCmp = new MultiComparator(btreeBinCmps);
// btree frame factories
- protected TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ protected TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(btreeTypeTraits);
protected ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
protected ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
protected ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
@@ -93,11 +109,11 @@
protected int invListsFileId;
protected int invListFields = 1;
- protected ITypeTrait[] invListTypeTraits = new ITypeTrait[invListFields];
+ protected ITypeTraits[] invListTypeTraits = new ITypeTraits[invListFields];
protected int invListKeys = 1;
protected IBinaryComparator[] invListBinCmps = new IBinaryComparator[invListKeys];
- protected MultiComparator invListCmp = new MultiComparator(invListTypeTraits, invListBinCmps);
+ protected MultiComparator invListCmp = new MultiComparator(invListBinCmps);
protected InvertedIndex invIndex;
@@ -153,23 +169,13 @@
btreeFileId = fmp.lookupFileId(btreeFile);
bufferCache.openFile(btreeFileId);
- // token (key)
- typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- // startPageId
- typeTraits[1] = new TypeTrait(4);
- // endPageId
- typeTraits[2] = new TypeTrait(4);
- // startOff
- typeTraits[3] = new TypeTrait(4);
- // numElements
- typeTraits[4] = new TypeTrait(4);
-
- btreeBinCmps[0] = UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+ btreeBinCmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
- btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, btreeCmp);
- btree.create(btreeFileId, leafFrame, metaFrame);
+ btree = new BTree(bufferCache, btreeTypeTraits.length, btreeCmp, freePageManager, interiorFrameFactory,
+ leafFrameFactory);
+ btree.create(btreeFileId);
btree.open(btreeFileId);
// --- INVERTED INDEX ---
@@ -178,10 +184,10 @@
invListsFileId = fmp.lookupFileId(invListsFile);
bufferCache.openFile(invListsFileId);
- invListTypeTraits[0] = new TypeTrait(4);
- invListBinCmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+ invListTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ invListBinCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
- invIndex = new InvertedIndex(bufferCache, btree, invListCmp);
+ invIndex = new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmp);
invIndex.open(invListsFileId);
rnd.setSeed(50);
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
index bdd8c4a..93934ed 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
@@ -31,35 +31,33 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
@@ -67,6 +65,7 @@
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+import edu.uci.ics.hyracks.storage.am.invertedindex.util.InvertedIndexUtils;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
@@ -104,53 +103,42 @@
int invListsFileId = fmp.lookupFileId(invListsFile);
bufferCache.openFile(invListsFileId);
- // declare btree fields
- int fieldCount = 5;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- // token (key)
- typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- // startPageId
- typeTraits[1] = new TypeTrait(4);
- // endPageId
- typeTraits[2] = new TypeTrait(4);
- // startOff
- typeTraits[3] = new TypeTrait(4);
- // numElements
- typeTraits[4] = new TypeTrait(4);
+ // Declare token type traits, and compute BTree type traits.
+ ITypeTraits[] tokenTypeTraits = new ITypeTraits[] { UTF8StringPointable.TYPE_TRAITS };
+ ITypeTraits[] btreeTypeTraits = InvertedIndexUtils.getBTreeTypeTraits(tokenTypeTraits);
// declare btree keys
int keyFieldCount = 1;
IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+ cmps[0] = PointableBinaryComparatorFactory.of(UTF8StringPointable.FACTORY).createBinaryComparator();
- MultiComparator btreeCmp = new MultiComparator(typeTraits, cmps);
+ MultiComparator btreeCmp = new MultiComparator(cmps);
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(btreeTypeTraits);
ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
- ITreeIndexFrame interiorFrame = interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, btreeCmp);
- btree.create(btreeFileId, leafFrame, metaFrame);
+ BTree btree = new BTree(bufferCache, btreeTypeTraits.length, btreeCmp, freePageManager, interiorFrameFactory,
+ leafFrameFactory);
+ btree.create(btreeFileId);
btree.open(btreeFileId);
int invListFields = 1;
- ITypeTrait[] invListTypeTraits = new ITypeTrait[invListFields];
- invListTypeTraits[0] = new TypeTrait(4);
+ ITypeTraits[] invListTypeTraits = new ITypeTraits[invListFields];
+ invListTypeTraits[0] = IntegerPointable.TYPE_TRAITS;
int invListKeys = 1;
IBinaryComparator[] invListBinCmps = new IBinaryComparator[invListKeys];
- invListBinCmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+ invListBinCmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
- MultiComparator invListCmp = new MultiComparator(invListTypeTraits, invListBinCmps);
+ MultiComparator invListCmp = new MultiComparator(invListBinCmps);
- InvertedIndex invIndex = new InvertedIndex(bufferCache, btree, invListCmp);
+ InvertedIndex invIndex = new InvertedIndex(bufferCache, btree, invListTypeTraits, invListCmp);
invIndex.open(invListsFileId);
Random rnd = new Random();
@@ -226,7 +214,7 @@
// ------- START VERIFICATION -----------
- ITreeIndexCursor btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame);
+ ITreeIndexCursor btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame, false);
FrameTupleReference searchKey = new FrameTupleReference();
RangePredicate btreePred = new RangePredicate(true, searchKey, searchKey, true, true, btreeCmp, btreeCmp);
@@ -241,7 +229,7 @@
IFrameTupleAccessor tokenAccessor = new FrameTupleAccessor(stageletCtx.getFrameSize(), tokenRecDesc);
tokenAccessor.reset(frame);
- BTreeOpContext btreeOpCtx = invIndex.getBTree().createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
+ ITreeIndexAccessor btreeAccessor = invIndex.getBTree().createAccessor();
// verify created inverted lists one-by-one
for (int i = 0; i < tokens.size(); i++) {
@@ -256,7 +244,7 @@
searchKey.reset(tokenAccessor, 0);
- invIndex.openCursor(btreeCursor, btreePred, btreeOpCtx, invListCursor);
+ invIndex.openCursor(btreeCursor, btreePred, btreeAccessor, invListCursor);
invListCursor.pinPagesSync();
int checkIndex = 0;
@@ -291,7 +279,7 @@
searchKey.reset(tokenAccessor, 0);
- invIndex.openCursor(btreeCursor, btreePred, btreeOpCtx, invListCursor);
+ invIndex.openCursor(btreeCursor, btreePred, btreeAccessor, invListCursor);
invListCursor.pinPagesSync();
Assert.assertEquals(invListCursor.hasNext(), false);
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/FixedSizeFrameTupleTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/FixedSizeFrameTupleTest.java
index 0b49ec1..9c7ec09 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/FixedSizeFrameTupleTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/FixedSizeFrameTupleTest.java
@@ -23,56 +23,53 @@
import org.junit.Test;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeFrameTupleAccessor;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeFrameTupleAppender;
public class FixedSizeFrameTupleTest {
- private static int FRAME_SIZE = 4096;
+ private static int FRAME_SIZE = 4096;
- private Random rnd = new Random(50);
+ private Random rnd = new Random(50);
- /**
- * This test verifies the correct behavior of the FixedSizeFrameTuple class.
- * Frames containing FixedSizeFrameTuple's require neither tuple slots nor
- * field slots. The tests inserts generated data into a frame until the
- * frame is full, and then verifies the frame's contents.
- *
- */
- @Test
- public void singleFieldTest() throws Exception {
- ByteBuffer buffer = ByteBuffer.allocate(FRAME_SIZE);
+ /**
+ * This test verifies the correct behavior of the FixedSizeFrameTuple class.
+ * Frames containing FixedSizeFrameTuple's require neither tuple slots nor
+ * field slots. The tests inserts generated data into a frame until the
+ * frame is full, and then verifies the frame's contents.
+ *
+ */
+ @Test
+ public void singleFieldTest() throws Exception {
+ ByteBuffer buffer = ByteBuffer.allocate(FRAME_SIZE);
- ITypeTrait[] fields = new TypeTrait[1];
- fields[0] = new TypeTrait(4);
+ ITypeTraits[] fields = new ITypeTraits[1];
+ fields[0] = IntegerPointable.TYPE_TRAITS;
- FixedSizeFrameTupleAppender ftapp = new FixedSizeFrameTupleAppender(
- FRAME_SIZE, fields);
- FixedSizeFrameTupleAccessor ftacc = new FixedSizeFrameTupleAccessor(
- FRAME_SIZE, fields);
+ FixedSizeFrameTupleAppender ftapp = new FixedSizeFrameTupleAppender(FRAME_SIZE, fields);
+ FixedSizeFrameTupleAccessor ftacc = new FixedSizeFrameTupleAccessor(FRAME_SIZE, fields);
- boolean frameHasSpace = true;
+ boolean frameHasSpace = true;
- ArrayList<Integer> check = new ArrayList<Integer>();
+ ArrayList<Integer> check = new ArrayList<Integer>();
- ftapp.reset(buffer, true);
- while (frameHasSpace) {
- int val = rnd.nextInt();
- frameHasSpace = ftapp.append(val);
- if (frameHasSpace) {
- check.add(val);
- ftapp.incrementTupleCount(1);
- }
- }
+ ftapp.reset(buffer, true);
+ while (frameHasSpace) {
+ int val = rnd.nextInt();
+ frameHasSpace = ftapp.append(val);
+ if (frameHasSpace) {
+ check.add(val);
+ ftapp.incrementTupleCount(1);
+ }
+ }
- ftacc.reset(buffer);
- for (int i = 0; i < ftacc.getTupleCount(); i++) {
- int val = IntegerSerializerDeserializer.getInt(ftacc.getBuffer()
- .array(), ftacc.getTupleStartOffset(i));
- Assert.assertEquals(check.get(i).intValue(), val);
- }
- }
+ ftacc.reset(buffer);
+ for (int i = 0; i < ftacc.getTupleCount(); i++) {
+ int val = IntegerSerializerDeserializer.getInt(ftacc.getBuffer().array(), ftacc.getTupleStartOffset(i));
+ Assert.assertEquals(check.get(i).intValue(), val);
+ }
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
index 4ef8855..1299fcb 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
@@ -17,6 +17,7 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.logging.Level;
import org.junit.Assert;
import org.junit.Before;
@@ -27,6 +28,8 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
@@ -65,7 +68,7 @@
loadData();
}
- public void loadData() throws HyracksDataException {
+ public void loadData() throws HyracksDataException, TreeIndexException, PageAllocationException {
tokens.add("compilers");
tokens.add("computer");
tokens.add("databases");
@@ -211,7 +214,10 @@
totalTime += timeEnd - timeStart;
}
double avgTime = totalTime / (double) repeats;
- LOGGER.info(i + ": " + "\"" + queryString + "\": " + avgTime + "ms");
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(i + ": " + "\"" + queryString + "\": " + avgTime
+ + "ms");
+ }
if (!panic) {
@@ -232,12 +238,16 @@
}
if (expectedResults.size() != checkIndex) {
- LOGGER.info("CHECKING");
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("CHECKING");
+ }
StringBuilder expectedStrBuilder = new StringBuilder();
for (Integer x : expectedResults) {
expectedStrBuilder.append(x + " ");
}
- LOGGER.info(expectedStrBuilder.toString());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(expectedStrBuilder.toString());
+ }
}
Assert.assertEquals(expectedResults.size(), checkIndex);
@@ -265,27 +275,39 @@
public void jaccardKeywordQueryTest() throws Exception {
JaccardSearchModifier searchModifier = new JaccardSearchModifier(1.0f);
- LOGGER.info("JACCARD: " + 1.0f);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("JACCARD: " + 1.0f);
+ }
searchModifier.setJaccThresh(1.0f);
runQueries(searchModifier, 50);
- LOGGER.info("JACCARD: " + 0.9f);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("JACCARD: " + 0.9f);
+ }
searchModifier.setJaccThresh(0.9f);
runQueries(searchModifier, 50);
- LOGGER.info("JACCARD: " + 0.8f);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("JACCARD: " + 0.8f);
+ }
searchModifier.setJaccThresh(0.8f);
runQueries(searchModifier, 50);
- LOGGER.info("JACCARD: " + 0.7f);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("JACCARD: " + 0.7f);
+ }
searchModifier.setJaccThresh(0.7f);
runQueries(searchModifier, 50);
- LOGGER.info("JACCARD: " + 0.6f);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("JACCARD: " + 0.6f);
+ }
searchModifier.setJaccThresh(0.6f);
runQueries(searchModifier, 50);
- LOGGER.info("JACCARD: " + 0.5f);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("JACCARD: " + 0.5f);
+ }
searchModifier.setJaccThresh(0.5f);
runQueries(searchModifier, 50);
}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
index c87ce9b..eb2b39c 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex;
import java.io.DataOutputStream;
@@ -5,6 +20,7 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.logging.Level;
import org.junit.Before;
import org.junit.Test;
@@ -14,6 +30,8 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.PageAllocationException;
+import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
@@ -112,7 +130,7 @@
}
}
- public void loadData() throws IOException {
+ public void loadData() throws IOException, TreeIndexException, PageAllocationException {
List<TokenIdPair> pairs = new ArrayList<TokenIdPair>();
// generate pairs for subsequent sorting and bulk-loading
int id = 0;
@@ -214,7 +232,9 @@
}
// remove trailing newline
strBuilder.deleteCharAt(strBuilder.length() - 1);
- LOGGER.info(strBuilder.toString());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(strBuilder.toString());
+ }
}
}
@@ -238,15 +258,21 @@
public void jaccardQueryTest() throws Exception {
JaccardSearchModifier searchModifier = new JaccardSearchModifier(1.0f);
- LOGGER.info("JACCARD: " + 0.9f);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("JACCARD: " + 0.9f);
+ }
searchModifier.setJaccThresh(0.9f);
runQueries(searchModifier, 5);
- LOGGER.info("JACCARD: " + 0.8f);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("JACCARD: " + 0.8f);
+ }
searchModifier.setJaccThresh(0.8f);
runQueries(searchModifier, 5);
- LOGGER.info("JACCARD: " + 0.7f);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("JACCARD: " + 0.7f);
+ }
searchModifier.setJaccThresh(0.7f);
runQueries(searchModifier, 5);
}
@@ -261,15 +287,21 @@
EditDistanceSearchModifier searchModifier = new EditDistanceSearchModifier(
3, 0);
- LOGGER.info("EDIT DISTANCE: " + 1);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("EDIT DISTANCE: " + 1);
+ }
searchModifier.setEdThresh(1);
runQueries(searchModifier, 5);
- LOGGER.info("EDIT DISTANCE: " + 2);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("EDIT DISTANCE: " + 2);
+ }
searchModifier.setEdThresh(2);
runQueries(searchModifier, 5);
- LOGGER.info("EDIT DISTANCE: " + 3);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("EDIT DISTANCE: " + 3);
+ }
searchModifier.setEdThresh(3);
runQueries(searchModifier, 5);
}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/WordTokenizerTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/WordTokenizerTest.java
index 57fe306..53fb96d 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/WordTokenizerTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/WordTokenizerTest.java
@@ -42,181 +42,180 @@
public class WordTokenizerTest {
- private String text = "Hello World, I would like to inform you of the importance of Foo Bar. Yes, Foo Bar. Jürgen.";
- private byte[] inputBuffer;
+ private String text = "Hello World, I would like to inform you of the importance of Foo Bar. Yes, Foo Bar. Jürgen.";
+ private byte[] inputBuffer;
- private ArrayList<String> expectedUTF8Tokens = new ArrayList<String>();
- private ArrayList<Integer> expectedHashedUTF8Tokens = new ArrayList<Integer>();
- private ArrayList<Integer> expectedCountedHashedUTF8Tokens = new ArrayList<Integer>();
+ private ArrayList<String> expectedUTF8Tokens = new ArrayList<String>();
+ private ArrayList<Integer> expectedHashedUTF8Tokens = new ArrayList<Integer>();
+ private ArrayList<Integer> expectedCountedHashedUTF8Tokens = new ArrayList<Integer>();
- @Before
- public void init() throws IOException {
- // serialize text into bytes
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- DataOutput dos = new DataOutputStream(baos);
- dos.writeUTF(text);
- inputBuffer = baos.toByteArray();
+ private boolean isSeparator(char c) {
+ return !(Character.isLetterOrDigit(c) || Character.getType(c) == Character.OTHER_LETTER || Character.getType(c) == Character.OTHER_NUMBER);
+ }
+
+ private void tokenize(String text, ArrayList<String> tokens) {
+ String lowerCaseText = text.toLowerCase();
+ int startIx = 0;
+
+ // Skip separators at beginning of string.
+ while(isSeparator(lowerCaseText.charAt(startIx))) {
+ startIx++;
+ }
+ while(startIx < lowerCaseText.length()) {
+ while(startIx < lowerCaseText.length() && isSeparator(lowerCaseText.charAt(startIx))) {
+ startIx++;
+ }
+ int tokenStart = startIx;
+
+ while(startIx < lowerCaseText.length() && !isSeparator(lowerCaseText.charAt(startIx))) {
+ startIx++;
+ }
+ int tokenEnd = startIx;
+
+ // Emit token.
+ String token = lowerCaseText.substring(tokenStart, tokenEnd);
+
+ tokens.add(token);
+ }
+ }
+
+ @Before
+ public void init() throws IOException {
+ // serialize text into bytes
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutput dos = new DataOutputStream(baos);
+ dos.writeUTF(text);
+ inputBuffer = baos.toByteArray();
+
+ // init expected string tokens
+ tokenize(text, expectedUTF8Tokens);
+
+ // hashed tokens ignoring token count
+ for (int i = 0; i < expectedUTF8Tokens.size(); i++) {
+ int hash = tokenHash(expectedUTF8Tokens.get(i), 1);
+ expectedHashedUTF8Tokens.add(hash);
+ }
- // init expected string tokens
- expectedUTF8Tokens.add("hello");
- expectedUTF8Tokens.add("world");
- expectedUTF8Tokens.add("i");
- expectedUTF8Tokens.add("would");
- expectedUTF8Tokens.add("like");
- expectedUTF8Tokens.add("to");
- expectedUTF8Tokens.add("inform");
- expectedUTF8Tokens.add("you");
- expectedUTF8Tokens.add("of");
- expectedUTF8Tokens.add("the");
- expectedUTF8Tokens.add("importance");
- expectedUTF8Tokens.add("of");
- expectedUTF8Tokens.add("foo");
- expectedUTF8Tokens.add("bar");
- expectedUTF8Tokens.add("yes");
- expectedUTF8Tokens.add("foo");
- expectedUTF8Tokens.add("bar");
- expectedUTF8Tokens.add("jürgen");
+ // hashed tokens using token count
+ HashMap<String, Integer> tokenCounts = new HashMap<String, Integer>();
+ for (int i = 0; i < expectedUTF8Tokens.size(); i++) {
+ Integer count = tokenCounts.get(expectedUTF8Tokens.get(i));
+ if (count == null) {
+ count = 1;
+ tokenCounts.put(expectedUTF8Tokens.get(i), count);
+ } else {
+ count++;
+ }
- // hashed tokens ignoring token count
- for (int i = 0; i < expectedUTF8Tokens.size(); i++) {
- int hash = tokenHash(expectedUTF8Tokens.get(i), 1);
- expectedHashedUTF8Tokens.add(hash);
- }
+ int hash = tokenHash(expectedUTF8Tokens.get(i), count);
+ expectedCountedHashedUTF8Tokens.add(hash);
+ }
+ }
- // hashed tokens using token count
- HashMap<String, Integer> tokenCounts = new HashMap<String, Integer>();
- for (int i = 0; i < expectedUTF8Tokens.size(); i++) {
- Integer count = tokenCounts.get(expectedUTF8Tokens.get(i));
- if (count == null) {
- count = 1;
- tokenCounts.put(expectedUTF8Tokens.get(i), count);
- } else {
- count++;
- }
+ @Test
+ public void testWordTokenizerWithCountedHashedUTF8Tokens() throws IOException {
- int hash = tokenHash(expectedUTF8Tokens.get(i), count);
- expectedCountedHashedUTF8Tokens.add(hash);
- }
- }
+ HashedUTF8WordTokenFactory tokenFactory = new HashedUTF8WordTokenFactory();
+ DelimitedUTF8StringBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(false, false,
+ tokenFactory);
- @Test
- public void testWordTokenizerWithCountedHashedUTF8Tokens()
- throws IOException {
+ tokenizer.reset(inputBuffer, 0, inputBuffer.length);
- HashedUTF8WordTokenFactory tokenFactory = new HashedUTF8WordTokenFactory();
- DelimitedUTF8StringBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(
- false, false, tokenFactory);
+ int tokenCount = 0;
- tokenizer.reset(inputBuffer, 0, inputBuffer.length);
+ while (tokenizer.hasNext()) {
+ tokenizer.next();
- int tokenCount = 0;
+ // serialize token
+ ByteArrayOutputStream tokenBaos = new ByteArrayOutputStream();
+ DataOutput tokenDos = new DataOutputStream(tokenBaos);
- while (tokenizer.hasNext()) {
- tokenizer.next();
+ IToken token = tokenizer.getToken();
+ token.serializeToken(tokenDos);
- // serialize token
- ByteArrayOutputStream tokenBaos = new ByteArrayOutputStream();
- DataOutput tokenDos = new DataOutputStream(tokenBaos);
+ // deserialize token
+ ByteArrayInputStream bais = new ByteArrayInputStream(tokenBaos.toByteArray());
+ DataInput in = new DataInputStream(bais);
- IToken token = tokenizer.getToken();
- token.serializeToken(tokenDos);
+ Integer hashedToken = in.readInt();
- // deserialize token
- ByteArrayInputStream bais = new ByteArrayInputStream(
- tokenBaos.toByteArray());
- DataInput in = new DataInputStream(bais);
+ Assert.assertEquals(hashedToken, expectedCountedHashedUTF8Tokens.get(tokenCount));
- Integer hashedToken = in.readInt();
+ tokenCount++;
+ }
+ }
- // System.out.println(hashedToken);
+ @Test
+ public void testWordTokenizerWithHashedUTF8Tokens() throws IOException {
- Assert.assertEquals(hashedToken,
- expectedCountedHashedUTF8Tokens.get(tokenCount));
+ HashedUTF8WordTokenFactory tokenFactory = new HashedUTF8WordTokenFactory();
+ DelimitedUTF8StringBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, false, tokenFactory);
- tokenCount++;
- }
- }
+ tokenizer.reset(inputBuffer, 0, inputBuffer.length);
- @Test
- public void testWordTokenizerWithHashedUTF8Tokens() throws IOException {
+ int tokenCount = 0;
- HashedUTF8WordTokenFactory tokenFactory = new HashedUTF8WordTokenFactory();
- DelimitedUTF8StringBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(
- true, false, tokenFactory);
+ while (tokenizer.hasNext()) {
+ tokenizer.next();
- tokenizer.reset(inputBuffer, 0, inputBuffer.length);
+ // serialize token
+ ByteArrayOutputStream tokenBaos = new ByteArrayOutputStream();
+ DataOutput tokenDos = new DataOutputStream(tokenBaos);
- int tokenCount = 0;
+ IToken token = tokenizer.getToken();
+ token.serializeToken(tokenDos);
- while (tokenizer.hasNext()) {
- tokenizer.next();
+ // deserialize token
+ ByteArrayInputStream bais = new ByteArrayInputStream(tokenBaos.toByteArray());
+ DataInput in = new DataInputStream(bais);
- // serialize token
- ByteArrayOutputStream tokenBaos = new ByteArrayOutputStream();
- DataOutput tokenDos = new DataOutputStream(tokenBaos);
+ Integer hashedToken = in.readInt();
- IToken token = tokenizer.getToken();
- token.serializeToken(tokenDos);
+ Assert.assertEquals(expectedHashedUTF8Tokens.get(tokenCount), hashedToken);
- // deserialize token
- ByteArrayInputStream bais = new ByteArrayInputStream(
- tokenBaos.toByteArray());
- DataInput in = new DataInputStream(bais);
+ tokenCount++;
+ }
+ }
- Integer hashedToken = in.readInt();
+ @Test
+ public void testWordTokenizerWithUTF8Tokens() throws IOException {
- // System.out.println(hashedToken);
+ UTF8WordTokenFactory tokenFactory = new UTF8WordTokenFactory();
+ DelimitedUTF8StringBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, false, tokenFactory);
- Assert.assertEquals(expectedHashedUTF8Tokens.get(tokenCount),
- hashedToken);
+ tokenizer.reset(inputBuffer, 0, inputBuffer.length);
- tokenCount++;
- }
- }
+ int tokenCount = 0;
- @Test
- public void testWordTokenizerWithUTF8Tokens() throws IOException {
+ while (tokenizer.hasNext()) {
+ tokenizer.next();
- UTF8WordTokenFactory tokenFactory = new UTF8WordTokenFactory();
- DelimitedUTF8StringBinaryTokenizer tokenizer = new DelimitedUTF8StringBinaryTokenizer(
- true, false, tokenFactory);
+ // serialize hashed token
+ ByteArrayOutputStream tokenBaos = new ByteArrayOutputStream();
+ DataOutput tokenDos = new DataOutputStream(tokenBaos);
- tokenizer.reset(inputBuffer, 0, inputBuffer.length);
+ IToken token = tokenizer.getToken();
+ token.serializeToken(tokenDos);
- int tokenCount = 0;
+ // deserialize token
+ ByteArrayInputStream bais = new ByteArrayInputStream(tokenBaos.toByteArray());
+ DataInput in = new DataInputStream(bais);
- while (tokenizer.hasNext()) {
- tokenizer.next();
+ String strToken = in.readUTF();
- // serialize hashed token
- ByteArrayOutputStream tokenBaos = new ByteArrayOutputStream();
- DataOutput tokenDos = new DataOutputStream(tokenBaos);
+ Assert.assertEquals(expectedUTF8Tokens.get(tokenCount), strToken);
- IToken token = tokenizer.getToken();
- token.serializeToken(tokenDos);
+ tokenCount++;
+ }
+ }
- // deserialize token
- ByteArrayInputStream bais = new ByteArrayInputStream(
- tokenBaos.toByteArray());
- DataInput in = new DataInputStream(bais);
-
- String strToken = in.readUTF();
-
- // System.out.println(strToken);
-
- Assert.assertEquals(expectedUTF8Tokens.get(tokenCount), strToken);
-
- tokenCount++;
- }
- }
-
- // JAQL
- public int tokenHash(String token, int tokenCount) {
- int h = AbstractUTF8Token.GOLDEN_RATIO_32;
- for (int i = 0; i < token.length(); i++) {
- h ^= token.charAt(i);
- h *= AbstractUTF8Token.GOLDEN_RATIO_32;
- }
- return h + tokenCount;
- }
+ // JAQL Hash
+ public int tokenHash(String token, int tokenCount) {
+ int h = AbstractUTF8Token.GOLDEN_RATIO_32;
+ for (int i = 0; i < token.length(); i++) {
+ h ^= token.charAt(i);
+ h *= AbstractUTF8Token.GOLDEN_RATIO_32;
+ }
+ return h + tokenCount;
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/.classpath b/hyracks-tests/hyracks-storage-am-rtree-test/.classpath
deleted file mode 100644
index f2cc5f7..0000000
--- a/hyracks-tests/hyracks-storage-am-rtree-test/.classpath
+++ /dev/null
@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" output="target/test-classes" path="src/test/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
- <classpathentry kind="con" path="org.maven.ide.eclipse.MAVEN2_CLASSPATH_CONTAINER"/>
- <classpathentry kind="output" path="target/classes"/>
-</classpath>
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/.project b/hyracks-tests/hyracks-storage-am-rtree-test/.project
deleted file mode 100644
index ea7e36b..0000000
--- a/hyracks-tests/hyracks-storage-am-rtree-test/.project
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
- <name>hyracks-storage-am-rtree-test</name>
- <comment></comment>
- <projects>
- </projects>
- <buildSpec>
- <buildCommand>
- <name>org.eclipse.jdt.core.javabuilder</name>
- <arguments>
- </arguments>
- </buildCommand>
- <buildCommand>
- <name>org.maven.ide.eclipse.maven2Builder</name>
- <arguments>
- </arguments>
- </buildCommand>
- </buildSpec>
- <natures>
- <nature>org.eclipse.jdt.core.javanature</nature>
- <nature>org.maven.ide.eclipse.maven2Nature</nature>
- </natures>
-</projectDescription>
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/.settings/org.eclipse.jdt.core.prefs b/hyracks-tests/hyracks-storage-am-rtree-test/.settings/org.eclipse.jdt.core.prefs
deleted file mode 100644
index 375e12e..0000000
--- a/hyracks-tests/hyracks-storage-am-rtree-test/.settings/org.eclipse.jdt.core.prefs
+++ /dev/null
@@ -1,6 +0,0 @@
-#Fri May 20 19:34:07 PDT 2011
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/.settings/org.maven.ide.eclipse.prefs b/hyracks-tests/hyracks-storage-am-rtree-test/.settings/org.maven.ide.eclipse.prefs
deleted file mode 100644
index 99b89a6..0000000
--- a/hyracks-tests/hyracks-storage-am-rtree-test/.settings/org.maven.ide.eclipse.prefs
+++ /dev/null
@@ -1,9 +0,0 @@
-#Thu Jan 06 11:27:16 PST 2011
-activeProfiles=
-eclipse.preferences.version=1
-fullBuildGoals=process-test-resources
-includeModules=false
-resolveWorkspaceProjects=true
-resourceFilterGoals=process-resources resources\:testResources
-skipCompilerPlugin=true
-version=1
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml b/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml
index cc02dca..bd1b82e 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-rtree-test</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -34,20 +34,20 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-am-rtree</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-test-support</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>test</scope>
</dependency>
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTest.java
index 25c5370..8f619a4 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/AbstractRTreeTest.java
@@ -18,11 +18,13 @@
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.logging.Logger;
import org.junit.AfterClass;
public abstract class AbstractRTreeTest {
+ protected static final Logger LOGGER = Logger.getLogger(AbstractRTreeTest.class.getName());
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
"ddMMyy-hhmmssSS");
protected final static String tmpDir = System.getProperty("java.io.tmpdir");
@@ -30,10 +32,6 @@
protected final static String fileName = tmpDir + sep
+ simpleDateFormat.format(new Date());
- protected void print(String str) {
- System.err.print(str);
- }
-
@AfterClass
public static void cleanup() throws Exception {
File f = new File(fileName);
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
index 907d5dc..9fe2c94 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/RTreeTest.java
@@ -19,6 +19,7 @@
import java.io.File;
import java.nio.ByteBuffer;
import java.util.Random;
+import java.util.logging.Level;
import org.junit.Test;
@@ -26,21 +27,23 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.DoubleBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.util.TupleUtils;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
@@ -48,827 +51,719 @@
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.impls.TreeDiskOrderScanCursor;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.utility.TreeIndexStats;
-import edu.uci.ics.hyracks.storage.am.common.utility.TreeIndexStatsGatherer;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStats;
+import edu.uci.ics.hyracks.storage.am.common.util.TreeIndexStatsGatherer;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeFrame;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.DoublePrimitiveValueProviderFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.IntegerPrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeOpContext;
import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestUtils;
public class RTreeTest extends AbstractRTreeTest {
+ private static final int PAGE_SIZE = 256;
+ private static final int NUM_PAGES = 10;
+ private static final int MAX_OPEN_FILES = 10;
+ private static final int HYRACKS_FRAME_SIZE = 128;
+ private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+ // create an R-tree of two dimensions
+ // fill the R-tree with random values using insertions
+ // perform ordered scan
+ @Test
+ public void test01() throws Exception {
- // create an R-tree of two dimensions
- // fill the R-tree with random values using insertions
- // perform ordered scan
- @Test
- public void test01() throws Exception {
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
+ bufferCache.openFile(fileId);
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES,
- MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
+ // declare keys
+ int keyFieldCount = 4;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ cmps[1] = cmps[0];
+ cmps[2] = cmps[0];
+ cmps[3] = cmps[0];
- // declare keys
- int keyFieldCount = 4;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = DoubleBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
+ // declare tuple fields
+ int fieldCount = 7;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = DoublePointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
- // declare tuple fields
- int fieldCount = 7;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(8);
- typeTraits[1] = new TypeTrait(8);
- typeTraits[2] = new TypeTrait(8);
- typeTraits[3] = new TypeTrait(8);
- typeTraits[4] = new TypeTrait(8);
- typeTraits[5] = new TypeTrait(4);
- typeTraits[6] = new TypeTrait(8);
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ cmps.length, DoublePointable.FACTORY);
- // declare value providers
- IPrimitiveValueProvider[] valueProviders = new IPrimitiveValueProvider[keyFieldCount];
- valueProviders[0] = DoublePrimitiveValueProviderFactory.INSTANCE
- .createPrimitiveValueProvider();
- valueProviders[1] = valueProviders[0];
- valueProviders[2] = valueProviders[0];
- valueProviders[3] = valueProviders[0];
+ MultiComparator cmp = new MultiComparator(cmps);
- MultiComparator cmp = new MultiComparator(typeTraits, cmps,
- valueProviders);
+ RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- typeTraits);
+ ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
+ IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
+ IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory
- .createFrame();
- IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(
- bufferCache, fileId, 0, metaFrameFactory);
+ RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ rtree.create(fileId);
+ rtree.open(fileId);
- RTree rtree = new RTree(bufferCache, freePageManager,
- interiorFrameFactory, leafFrameFactory, cmp);
- rtree.create(fileId, leafFrame, metaFrame);
- rtree.open(fileId);
+ ByteBuffer hyracksFrame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(hyracksFrame);
+ FrameTupleReference tuple = new FrameTupleReference();
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(
- ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
+ ITreeIndexAccessor indexAccessor = rtree.createAccessor();
- RTreeOpContext insertOpCtx = rtree.createOpContext(IndexOp.INSERT,
- leafFrame, interiorFrame, metaFrame);
+ Random rnd = new Random();
+ rnd.setSeed(50);
- Random rnd = new Random();
- rnd.setSeed(50);
+ Random rnd2 = new Random();
+ rnd2.setSeed(50);
+ for (int i = 0; i < 5000; i++) {
- Random rnd2 = new Random();
- rnd2.setSeed(50);
- for (int i = 0; i < 5000; i++) {
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
+ double pk1 = rnd2.nextDouble();
+ int pk2 = rnd2.nextInt();
+ double pk3 = rnd2.nextDouble();
- double pk1 = rnd2.nextDouble();
- int pk2 = rnd2.nextInt();
- double pk3 = rnd2.nextDouble();
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
+ appender.reset(hyracksFrame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- tb.getSize());
+ tuple.reset(accessor, 0);
- tuple.reset(accessor, 0);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
- if (i % 1000 == 0) {
- print("INSERTING " + i + " " + Math.min(p1x, p2x) + " "
- + Math.min(p1y, p2y) + " " + Math.max(p1x, p2x) + " "
- + Math.max(p1y, p2y) + "\n");
- }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
- try {
- rtree.insert(tuple, insertOpCtx);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+ String rtreeStats = rtree.printStats();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rtreeStats);
+ }
- // rtree.printTree(leafFrame, interiorFrame, recDescSers);
- // System.err.println();
+ // disk-order scan
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("DISK-ORDER SCAN:");
+ }
+ TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
+ indexAccessor.diskOrderScan(diskOrderCursor);
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference frameTuple = diskOrderCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, recDescSers);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ diskOrderCursor.close();
+ }
- String rtreeStats = rtree.printStats();
- print(rtreeStats);
+ TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
+ rtree.getRootPageId());
+ TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
+ String string = stats.toString();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(string);
+ }
- // disk-order scan
- print("DISK-ORDER SCAN:\n");
- TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(
- leafFrame);
- RTreeOpContext diskOrderScanOpCtx = rtree.createOpContext(
- IndexOp.DISKORDERSCAN, leafFrame, null, null);
- rtree.diskOrderScan(diskOrderCursor, leafFrame, metaFrame,
- diskOrderScanOpCtx);
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference frameTuple = diskOrderCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- diskOrderCursor.close();
- }
+ rtree.close();
+ bufferCache.closeFile(fileId);
+ bufferCache.close();
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(
- bufferCache, freePageManager, fileId, rtree.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(leafFrame,
- interiorFrame, metaFrame);
- String string = stats.toString();
- System.err.println(string);
+ }
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
+ // create an R-tree of two dimensions
+ // fill the R-tree with random values using insertions
+ // and then delete all the tuples which result of an empty R-tree
+ @Test
+ public void test02() throws Exception {
- }
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
+ bufferCache.openFile(fileId);
- // create an R-tree of two dimensions
- // fill the R-tree with random values using insertions
- // and then delete all the tuples which result of an empty R-tree
- @Test
- public void test02() throws Exception {
+ // declare keys
+ int keyFieldCount = 4;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ cmps[1] = cmps[0];
+ cmps[2] = cmps[0];
+ cmps[3] = cmps[0];
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES,
- MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
+ // declare tuple fields
+ int fieldCount = 7;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = DoublePointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
- // declare keys
- int keyFieldCount = 4;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = DoubleBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ cmps.length, DoublePointable.FACTORY);
- // declare tuple fields
- int fieldCount = 7;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(8);
- typeTraits[1] = new TypeTrait(8);
- typeTraits[2] = new TypeTrait(8);
- typeTraits[3] = new TypeTrait(8);
- typeTraits[4] = new TypeTrait(8);
- typeTraits[5] = new TypeTrait(4);
- typeTraits[6] = new TypeTrait(8);
+ MultiComparator cmp = new MultiComparator(cmps);
- // declare value providers
- IPrimitiveValueProvider[] valueProviders = new IPrimitiveValueProvider[keyFieldCount];
- valueProviders[0] = DoublePrimitiveValueProviderFactory.INSTANCE
- .createPrimitiveValueProvider();
- valueProviders[1] = valueProviders[0];
- valueProviders[2] = valueProviders[0];
- valueProviders[3] = valueProviders[0];
+ RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
- MultiComparator cmp = new MultiComparator(typeTraits, cmps,
- valueProviders);
+ ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- typeTraits);
+ IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
+ IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
+ RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ rtree.create(fileId);
+ rtree.open(fileId);
- IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory
- .createFrame();
- IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(
- bufferCache, fileId, 0, metaFrameFactory);
+ ByteBuffer hyracksFrame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
- RTree rtree = new RTree(bufferCache, freePageManager,
- interiorFrameFactory, leafFrameFactory, cmp);
- rtree.create(fileId, leafFrame, metaFrame);
- rtree.open(fileId);
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(hyracksFrame);
+ FrameTupleReference tuple = new FrameTupleReference();
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
+ ITreeIndexAccessor indexAccessor = rtree.createAccessor();
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(
- ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
+ Random rnd = new Random();
+ rnd.setSeed(50);
- RTreeOpContext insertOpCtx = rtree.createOpContext(IndexOp.INSERT,
- leafFrame, interiorFrame, metaFrame);
+ for (int i = 0; i < 5000; i++) {
- Random rnd = new Random();
- rnd.setSeed(50);
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
- for (int i = 0; i < 5000; i++) {
+ double pk1 = rnd.nextDouble();
+ int pk2 = rnd.nextInt();
+ double pk3 = rnd.nextDouble();
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
- double pk1 = rnd.nextDouble();
- int pk2 = rnd.nextInt();
- double pk3 = rnd.nextDouble();
+ appender.reset(hyracksFrame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
+ tuple.reset(accessor, 0);
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- tb.getSize());
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
- tuple.reset(accessor, 0);
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
- if (i % 1000 == 0) {
- print("INSERTING " + i + " " + Math.min(p1x, p2x) + " "
- + Math.min(p1y, p2y) + " " + Math.max(p1x, p2x) + " "
- + Math.max(p1y, p2y) + "\n");
- }
+ String rtreeStats = rtree.printStats();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rtreeStats);
+ }
- try {
- rtree.insert(tuple, insertOpCtx);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+ rnd.setSeed(50);
+ for (int i = 0; i < 5000; i++) {
- // rtree.printTree(leafFrame, interiorFrame, recDescSers);
- // System.err.println();
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
- String rtreeStats = rtree.printStats();
- print(rtreeStats);
+ double pk1 = rnd.nextDouble();
+ int pk2 = rnd.nextInt();
+ double pk3 = rnd.nextDouble();
- RTreeOpContext deleteOpCtx = rtree.createOpContext(IndexOp.DELETE,
- leafFrame, interiorFrame, metaFrame);
- rnd.setSeed(50);
- for (int i = 0; i < 5000; i++) {
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
+ appender.reset(hyracksFrame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
- double pk1 = rnd.nextDouble();
- int pk2 = rnd.nextInt();
- double pk3 = rnd.nextDouble();
+ tuple.reset(accessor, 0);
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("DELETING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- tb.getSize());
+ try {
+ indexAccessor.delete(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
- tuple.reset(accessor, 0);
+ TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
+ rtree.getRootPageId());
+ TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
+ String string = stats.toString();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(string);
+ }
- if (i % 1000 == 0) {
- print("DELETING " + i + " " + Math.min(p1x, p2x) + " "
- + Math.min(p1y, p2y) + " " + Math.max(p1x, p2x) + " "
- + Math.max(p1y, p2y) + "\n");
- }
+ rtree.close();
+ bufferCache.closeFile(fileId);
+ bufferCache.close();
- try {
- rtree.delete(tuple, deleteOpCtx);
+ }
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(
- bufferCache, freePageManager, fileId, rtree.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(leafFrame,
- interiorFrame, metaFrame);
- String string = stats.toString();
- System.err.println(string);
-
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
-
- }
-
- // create an R-tree of three dimensions
- // fill the R-tree with random values using insertions
- // perform ordered scan
- @Test
- public void test03() throws Exception {
-
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES,
- MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
-
- // declare keys
- int keyFieldCount = 6;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = DoubleBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
- cmps[4] = cmps[0];
- cmps[5] = cmps[0];
+ // create an R-tree of three dimensions
+ // fill the R-tree with random values using insertions
+ // perform ordered scan
+ @Test
+ public void test03() throws Exception {
- // declare tuple fields
- int fieldCount = 9;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(8);
- typeTraits[1] = new TypeTrait(8);
- typeTraits[2] = new TypeTrait(8);
- typeTraits[3] = new TypeTrait(8);
- typeTraits[4] = new TypeTrait(8);
- typeTraits[5] = new TypeTrait(8);
- typeTraits[6] = new TypeTrait(8);
- typeTraits[7] = new TypeTrait(4);
- typeTraits[8] = new TypeTrait(8);
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
+ bufferCache.openFile(fileId);
- // declare value providers
- IPrimitiveValueProvider[] valueProviders = new IPrimitiveValueProvider[keyFieldCount];
- valueProviders[0] = DoublePrimitiveValueProviderFactory.INSTANCE
- .createPrimitiveValueProvider();
- valueProviders[1] = valueProviders[0];
- valueProviders[2] = valueProviders[0];
- valueProviders[3] = valueProviders[0];
- valueProviders[4] = valueProviders[0];
- valueProviders[5] = valueProviders[0];
+ // declare keys
+ int keyFieldCount = 6;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ cmps[1] = cmps[0];
+ cmps[2] = cmps[0];
+ cmps[3] = cmps[0];
+ cmps[4] = cmps[0];
+ cmps[5] = cmps[0];
- MultiComparator cmp = new MultiComparator(typeTraits, cmps,
- valueProviders);
+ // declare tuple fields
+ int fieldCount = 9;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = DoublePointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
+ typeTraits[7] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[8] = DoublePointable.TYPE_TRAITS;
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- typeTraits);
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ cmps.length, DoublePointable.FACTORY);
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
+ MultiComparator cmp = new MultiComparator(cmps);
- IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory
- .createFrame();
- IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(
- bufferCache, fileId, 0, metaFrameFactory);
+ RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
- RTree rtree = new RTree(bufferCache, freePageManager,
- interiorFrameFactory, leafFrameFactory, cmp);
- rtree.create(fileId, leafFrame, metaFrame);
- rtree.open(fileId);
+ ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
+ IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
+ IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(
- ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
+ RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ rtree.create(fileId);
+ rtree.open(fileId);
- RTreeOpContext insertOpCtx = rtree.createOpContext(IndexOp.INSERT,
- leafFrame, interiorFrame, metaFrame);
+ ByteBuffer hyracksFrame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
- Random rnd = new Random();
- rnd.setSeed(50);
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(hyracksFrame);
+ FrameTupleReference tuple = new FrameTupleReference();
- for (int i = 0; i < 5000; i++) {
+ ITreeIndexAccessor indexAccessor = rtree.createAccessor();
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p1z = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
- double p2z = rnd.nextDouble();
+ Random rnd = new Random();
+ rnd.setSeed(50);
- double pk1 = rnd.nextDouble();
- int pk2 = rnd.nextInt();
- double pk3 = rnd.nextDouble();
+ for (int i = 0; i < 5000; i++) {
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1z, p2z),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1z, p2z),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p1z = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
+ double p2z = rnd.nextDouble();
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- tb.getSize());
+ double pk1 = rnd.nextDouble();
+ int pk2 = rnd.nextInt();
+ double pk3 = rnd.nextDouble();
- tuple.reset(accessor, 0);
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1z, p2z), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1z, p2z), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
- if (i % 1000 == 0) {
- print("INSERTING " + i + " " + Math.min(p1x, p2x) + " "
- + Math.min(p1y, p2y) + " " + Math.min(p1z, p2z) + " "
- + " " + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y)
- + " " + Math.max(p1z, p2z) + "\n");
- }
+ appender.reset(hyracksFrame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
- try {
- rtree.insert(tuple, insertOpCtx);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+ tuple.reset(accessor, 0);
- // rtree.printTree(leafFrame, interiorFrame, recDescSers);
- // System.err.println();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.min(p1z, p2z) + " " + " " + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y) + " "
+ + Math.max(p1z, p2z));
+ }
+ }
- String rtreeStats = rtree.printStats();
- print(rtreeStats);
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
- // disk-order scan
- print("DISK-ORDER SCAN:\n");
- TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(
- leafFrame);
- RTreeOpContext diskOrderScanOpCtx = rtree.createOpContext(
- IndexOp.DISKORDERSCAN, leafFrame, null, null);
- rtree.diskOrderScan(diskOrderCursor, leafFrame, metaFrame,
- diskOrderScanOpCtx);
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference frameTuple = diskOrderCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- diskOrderCursor.close();
- }
+ String rtreeStats = rtree.printStats();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rtreeStats);
+ }
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(
- bufferCache, freePageManager, fileId, rtree.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(leafFrame,
- interiorFrame, metaFrame);
- String string = stats.toString();
- System.err.println(string);
+ // disk-order scan
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("DISK-ORDER SCAN:");
+ }
+ TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
+ indexAccessor.diskOrderScan(diskOrderCursor);
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference frameTuple = diskOrderCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, recDescSers);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ diskOrderCursor.close();
+ }
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
+ TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
+ rtree.getRootPageId());
+ TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
+ String string = stats.toString();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(string);
+ }
- }
+ rtree.close();
+ bufferCache.closeFile(fileId);
+ bufferCache.close();
- // create an R-tree of two dimensions
- // fill the R-tree with random integer key values using insertions
- // perform ordered scan
- @Test
- public void test04() throws Exception {
+ }
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES,
- MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
+ // create an R-tree of two dimensions
+ // fill the R-tree with random integer key values using insertions
+ // perform ordered scan
+ @Test
+ public void test04() throws Exception {
- // declare keys
- int keyFieldCount = 4;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = IntegerBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
+ bufferCache.openFile(fileId);
- // declare tuple fields
- int fieldCount = 7;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(4);
- typeTraits[1] = new TypeTrait(4);
- typeTraits[2] = new TypeTrait(4);
- typeTraits[3] = new TypeTrait(4);
- typeTraits[4] = new TypeTrait(8);
- typeTraits[5] = new TypeTrait(4);
- typeTraits[6] = new TypeTrait(8);
+ // declare keys
+ int keyFieldCount = 4;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(IntegerPointable.FACTORY).createBinaryComparator();
+ cmps[1] = cmps[0];
+ cmps[2] = cmps[0];
+ cmps[3] = cmps[0];
- // declare value providers
- IPrimitiveValueProvider[] valueProviders = new IPrimitiveValueProvider[keyFieldCount];
- valueProviders[0] = IntegerPrimitiveValueProviderFactory.INSTANCE
- .createPrimitiveValueProvider();
- valueProviders[1] = valueProviders[0];
- valueProviders[2] = valueProviders[0];
- valueProviders[3] = valueProviders[0];
+ // declare tuple fields
+ int fieldCount = 7;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[1] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[2] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[3] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[4] = DoublePointable.TYPE_TRAITS;
+ typeTraits[5] = IntegerPointable.TYPE_TRAITS;
+ typeTraits[6] = DoublePointable.TYPE_TRAITS;
- MultiComparator cmp = new MultiComparator(typeTraits, cmps,
- valueProviders);
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ cmps.length, IntegerPointable.FACTORY);
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- typeTraits);
+ MultiComparator cmp = new MultiComparator(cmps);
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
+ RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
- IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory
- .createFrame();
- IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(
- bufferCache, fileId, 0, metaFrameFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
- RTree rtree = new RTree(bufferCache, freePageManager,
- interiorFrameFactory, leafFrameFactory, cmp);
- rtree.create(fileId, leafFrame, metaFrame);
- rtree.open(fileId);
+ IRTreeFrame interiorFrame = (IRTreeFrame) interiorFrameFactory.createFrame();
+ IRTreeFrame leafFrame = (IRTreeFrame) leafFrameFactory.createFrame();
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
+ RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ rtree.create(fileId);
+ rtree.open(fileId);
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = {
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(
- ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
+ ByteBuffer hyracksFrame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
- RTreeOpContext insertOpCtx = rtree.createOpContext(IndexOp.INSERT,
- leafFrame, interiorFrame, metaFrame);
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(hyracksFrame);
+ FrameTupleReference tuple = new FrameTupleReference();
- Random rnd = new Random();
- rnd.setSeed(50);
+ ITreeIndexAccessor indexAccessor = rtree.createAccessor();
- Random rnd2 = new Random();
- rnd2.setSeed(50);
- for (int i = 0; i < 5000; i++) {
+ Random rnd = new Random();
+ rnd.setSeed(50);
- int p1x = rnd.nextInt();
- int p1y = rnd.nextInt();
- int p2x = rnd.nextInt();
- int p2y = rnd.nextInt();
+ Random rnd2 = new Random();
+ rnd2.setSeed(50);
+ for (int i = 0; i < 5000; i++) {
- double pk1 = rnd2.nextDouble();
- int pk2 = rnd2.nextInt();
- double pk3 = rnd2.nextDouble();
+ int p1x = rnd.nextInt();
+ int p1y = rnd.nextInt();
+ int p2x = rnd.nextInt();
+ int p2y = rnd.nextInt();
- tb.reset();
- IntegerSerializerDeserializer.INSTANCE.serialize(
- Math.min(p1x, p2x), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(
- Math.min(p1y, p2y), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(
- Math.max(p1x, p2x), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(
- Math.max(p1y, p2y), dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
- tb.addFieldEndOffset();
+ double pk1 = rnd2.nextDouble();
+ int pk2 = rnd2.nextInt();
+ double pk3 = rnd2.nextDouble();
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- tb.getSize());
+ tb.reset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk1, dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk2, dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(pk3, dos);
+ tb.addFieldEndOffset();
- tuple.reset(accessor, 0);
+ appender.reset(hyracksFrame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
- if (i % 1000 == 0) {
- print("INSERTING " + i + " " + Math.min(p1x, p2x) + " "
- + Math.min(p1y, p2y) + " " + Math.max(p1x, p2x) + " "
- + Math.max(p1y, p2y) + "\n");
- }
+ tuple.reset(accessor, 0);
- try {
- rtree.insert(tuple, insertOpCtx);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
- // rtree.printTree(leafFrame, interiorFrame, recDescSers);
- // System.err.println();
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
- String rtreeStats = rtree.printStats();
- print(rtreeStats);
+ String rtreeStats = rtree.printStats();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rtreeStats);
+ }
- // disk-order scan
- print("DISK-ORDER SCAN:\n");
- TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(
- leafFrame);
- RTreeOpContext diskOrderScanOpCtx = rtree.createOpContext(
- IndexOp.DISKORDERSCAN, leafFrame, null, null);
- rtree.diskOrderScan(diskOrderCursor, leafFrame, metaFrame,
- diskOrderScanOpCtx);
- try {
- while (diskOrderCursor.hasNext()) {
- diskOrderCursor.next();
- ITupleReference frameTuple = diskOrderCursor.getTuple();
- String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- diskOrderCursor.close();
- }
+ // disk-order scan
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("DISK-ORDER SCAN:");
+ }
+ TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
+ indexAccessor.diskOrderScan(diskOrderCursor);
+ try {
+ while (diskOrderCursor.hasNext()) {
+ diskOrderCursor.next();
+ ITupleReference frameTuple = diskOrderCursor.getTuple();
+ String rec = TupleUtils.printTuple(frameTuple, recDescSers);
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(rec);
+ }
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ diskOrderCursor.close();
+ }
- TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(
- bufferCache, freePageManager, fileId, rtree.getRootPageId());
- TreeIndexStats stats = statsGatherer.gatherStats(leafFrame,
- interiorFrame, metaFrame);
- String string = stats.toString();
- System.err.println(string);
+ TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId,
+ rtree.getRootPageId());
+ TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
+ String string = stats.toString();
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(string);
+ }
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
+ rtree.close();
+ bufferCache.closeFile(fileId);
+ bufferCache.close();
- }
+ }
}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java
index b04afe5..a232e37 100644
--- a/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java
+++ b/hyracks-tests/hyracks-storage-am-rtree-test/src/test/java/edu/uci/ics/hyracks/storage/am/rtree/SearchCursorTest.java
@@ -23,6 +23,7 @@
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Random;
+import java.util.logging.Level;
import org.junit.Test;
@@ -30,20 +31,22 @@
import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTraits;
import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.DoublePointable;
+import edu.uci.ics.hyracks.data.std.primitive.IntegerPointable;
import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.DoubleBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.DoubleSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProvider;
+import edu.uci.ics.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexAccessor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexCursor;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
@@ -51,234 +54,199 @@
import edu.uci.ics.hyracks.storage.am.common.api.TreeIndexException;
import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.IndexOp;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.rtree.api.IRTreeLeafFrame;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMInteriorFrameFactory;
import edu.uci.ics.hyracks.storage.am.rtree.frames.RTreeNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.DoublePrimitiveValueProviderFactory;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTree;
-import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeOpContext;
import edu.uci.ics.hyracks.storage.am.rtree.impls.RTreeSearchCursor;
import edu.uci.ics.hyracks.storage.am.rtree.impls.SearchPredicate;
import edu.uci.ics.hyracks.storage.am.rtree.tuples.RTreeTypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.rtree.util.RTreeUtils;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
import edu.uci.ics.hyracks.test.support.TestUtils;
public class SearchCursorTest extends AbstractRTreeTest {
- private static final int PAGE_SIZE = 256;
- private static final int NUM_PAGES = 10;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 128;
- private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+ private static final int PAGE_SIZE = 256;
+ private static final int NUM_PAGES = 10;
+ private static final int MAX_OPEN_FILES = 10;
+ private static final int HYRACKS_FRAME_SIZE = 128;
+ private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
- // create an R-tree of two dimensions
- // fill the R-tree with random values using insertions
- // and then perform range search
- @Test
- public void searchCursorTest() throws Exception {
+ // create an R-tree of two dimensions
+ // fill the R-tree with random values using insertions
+ // and then perform range search
+ @Test
+ public void searchCursorTest() throws Exception {
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES,
- MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder
- .getBufferCache(ctx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder
- .getFileMapProvider(ctx);
- FileReference file = new FileReference(new File(fileName));
- bufferCache.createFile(file);
- int fileId = fmp.lookupFileId(file);
- bufferCache.openFile(fileId);
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(ctx);
+ FileReference file = new FileReference(new File(fileName));
+ bufferCache.createFile(file);
+ int fileId = fmp.lookupFileId(file);
+ bufferCache.openFile(fileId);
- // declare keys
- int keyFieldCount = 4;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = DoubleBinaryComparatorFactory.INSTANCE
- .createBinaryComparator();
- cmps[1] = cmps[0];
- cmps[2] = cmps[0];
- cmps[3] = cmps[0];
+ // declare keys
+ int keyFieldCount = 4;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = PointableBinaryComparatorFactory.of(DoublePointable.FACTORY).createBinaryComparator();
+ cmps[1] = cmps[0];
+ cmps[2] = cmps[0];
+ cmps[3] = cmps[0];
- // declare tuple fields
- int fieldCount = 5;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- typeTraits[0] = new TypeTrait(8);
- typeTraits[1] = new TypeTrait(8);
- typeTraits[2] = new TypeTrait(8);
- typeTraits[3] = new TypeTrait(8);
- typeTraits[4] = new TypeTrait(4);
+ // declare tuple fields
+ int fieldCount = 5;
+ ITypeTraits[] typeTraits = new ITypeTraits[fieldCount];
+ typeTraits[0] = DoublePointable.TYPE_TRAITS;
+ typeTraits[1] = DoublePointable.TYPE_TRAITS;
+ typeTraits[2] = DoublePointable.TYPE_TRAITS;
+ typeTraits[3] = DoublePointable.TYPE_TRAITS;
+ typeTraits[4] = IntegerPointable.TYPE_TRAITS;
- // declare value providers
- IPrimitiveValueProvider[] valueProviders = new IPrimitiveValueProvider[keyFieldCount];
- valueProviders[0] = DoublePrimitiveValueProviderFactory.INSTANCE
- .createPrimitiveValueProvider();
- valueProviders[1] = valueProviders[0];
- valueProviders[2] = valueProviders[0];
- valueProviders[3] = valueProviders[0];
+ // create value providers
+ IPrimitiveValueProviderFactory[] valueProviderFactories = RTreeUtils.createPrimitiveValueProviderFactories(
+ cmps.length, DoublePointable.FACTORY);
- MultiComparator cmp = new MultiComparator(typeTraits, cmps,
- valueProviders);
+ MultiComparator cmp = new MultiComparator(cmps);
- RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(
- typeTraits);
+ RTreeTypeAwareTupleWriterFactory tupleWriterFactory = new RTreeTypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(
- tupleWriterFactory, keyFieldCount);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
+ ITreeIndexFrameFactory interiorFrameFactory = new RTreeNSMInteriorFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexFrameFactory leafFrameFactory = new RTreeNSMLeafFrameFactory(tupleWriterFactory,
+ valueProviderFactories);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
- IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory
- .createFrame();
- IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory
- .createFrame();
- IFreePageManager freePageManager = new LinkedListFreePageManager(
- bufferCache, fileId, 0, metaFrameFactory);
+ IRTreeInteriorFrame interiorFrame = (IRTreeInteriorFrame) interiorFrameFactory.createFrame();
+ IRTreeLeafFrame leafFrame = (IRTreeLeafFrame) leafFrameFactory.createFrame();
+ IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, fileId, 0, metaFrameFactory);
- RTree rtree = new RTree(bufferCache, freePageManager,
- interiorFrameFactory, leafFrameFactory, cmp);
- rtree.create(fileId, leafFrame, metaFrame);
- rtree.open(fileId);
+ RTree rtree = new RTree(bufferCache, fieldCount, cmp, freePageManager, interiorFrameFactory, leafFrameFactory);
+ rtree.create(fileId);
+ rtree.open(fileId);
- ByteBuffer hyracksFrame = ctx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(cmp.getFieldCount());
- DataOutput dos = tb.getDataOutput();
+ ByteBuffer hyracksFrame = ctx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(fieldCount);
+ DataOutput dos = tb.getDataOutput();
- @SuppressWarnings("rawtypes")
- ISerializerDeserializer[] recDescSers = {
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- DoubleSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(
- ctx.getFrameSize(), recDesc);
- accessor.reset(hyracksFrame);
- FrameTupleReference tuple = new FrameTupleReference();
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer[] recDescSers = { DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, DoubleSerializerDeserializer.INSTANCE,
+ DoubleSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE };
+ RecordDescriptor recDesc = new RecordDescriptor(recDescSers);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(ctx.getFrameSize(), recDesc);
+ accessor.reset(hyracksFrame);
+ FrameTupleReference tuple = new FrameTupleReference();
- RTreeOpContext insertOpCtx = rtree.createOpContext(IndexOp.INSERT,
- leafFrame, interiorFrame, metaFrame);
+ ITreeIndexAccessor indexAccessor = rtree.createAccessor();
- Random rnd = new Random();
- rnd.setSeed(50);
- for (int i = 0; i < 5000; i++) {
+ Random rnd = new Random();
+ rnd.setSeed(50);
+ for (int i = 0; i < 5000; i++) {
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
- int pk = rnd.nextInt();
+ int pk = rnd.nextInt();
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk, dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk, dos);
+ tb.addFieldEndOffset();
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- tb.getSize());
+ appender.reset(hyracksFrame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
- tuple.reset(accessor, 0);
+ tuple.reset(accessor, 0);
- if (i % 1000 == 0) {
- print("INSERTING " + i + " " + Math.min(p1x, p2x) + " "
- + Math.min(p1y, p2y) + " " + Math.max(p1x, p2x) + " "
- + Math.max(p1y, p2y) + "\n");
- }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ if (i % 1000 == 0) {
+ LOGGER.info("INSERTING " + i + " " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
+ }
- try {
- rtree.insert(tuple, insertOpCtx);
- } catch (TreeIndexException e) {
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
+ try {
+ indexAccessor.insert(tuple);
+ } catch (TreeIndexException e) {
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
- for (int i = 0; i < 50; i++) {
- double p1x = rnd.nextDouble();
- double p1y = rnd.nextDouble();
- double p2x = rnd.nextDouble();
- double p2y = rnd.nextDouble();
+ for (int i = 0; i < 50; i++) {
+ double p1x = rnd.nextDouble();
+ double p1y = rnd.nextDouble();
+ double p2x = rnd.nextDouble();
+ double p2y = rnd.nextDouble();
- int pk = rnd.nextInt();
+ int pk = rnd.nextInt();
- tb.reset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x),
- dos);
- tb.addFieldEndOffset();
- DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y),
- dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(pk, dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.min(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1x, p2x), dos);
+ tb.addFieldEndOffset();
+ DoubleSerializerDeserializer.INSTANCE.serialize(Math.max(p1y, p2y), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(pk, dos);
+ tb.addFieldEndOffset();
- appender.reset(hyracksFrame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
- tb.getSize());
+ appender.reset(hyracksFrame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
- tuple.reset(accessor, 0);
+ tuple.reset(accessor, 0);
- print(i + " Searching for: " + Math.min(p1x, p2x) + " "
- + Math.min(p1y, p2y) + " " + Math.max(p1x, p2x) + " "
- + Math.max(p1y, p2y) + "\n");
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info(i + " Searching for: " + Math.min(p1x, p2x) + " " + Math.min(p1y, p2y) + " "
+ + Math.max(p1x, p2x) + " " + Math.max(p1y, p2y));
+ }
- ITreeIndexCursor searchCursor = new RTreeSearchCursor(
- interiorFrame, leafFrame);
- SearchPredicate searchPredicate = new SearchPredicate(tuple, cmp);
+ ITreeIndexCursor searchCursor = new RTreeSearchCursor(interiorFrame, leafFrame);
+ SearchPredicate searchPredicate = new SearchPredicate(tuple, cmp);
- RTreeOpContext searchOpCtx = rtree.createOpContext(IndexOp.SEARCH,
- leafFrame, interiorFrame, metaFrame);
- rtree.search(searchCursor, searchPredicate, searchOpCtx);
+ indexAccessor.search(searchCursor, searchPredicate);
- ArrayList<Integer> results = new ArrayList<Integer>();
- try {
- while (searchCursor.hasNext()) {
- searchCursor.next();
- ITupleReference frameTuple = searchCursor.getTuple();
- ByteArrayInputStream inStream = new ByteArrayInputStream(
- frameTuple.getFieldData(4),
- frameTuple.getFieldStart(4),
- frameTuple.getFieldLength(4));
- DataInput dataIn = new DataInputStream(inStream);
- Integer res = IntegerSerializerDeserializer.INSTANCE
- .deserialize(dataIn);
- results.add(res);
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- searchCursor.close();
- }
+ ArrayList<Integer> results = new ArrayList<Integer>();
+ try {
+ while (searchCursor.hasNext()) {
+ searchCursor.next();
+ ITupleReference frameTuple = searchCursor.getTuple();
+ ByteArrayInputStream inStream = new ByteArrayInputStream(frameTuple.getFieldData(4),
+ frameTuple.getFieldStart(4), frameTuple.getFieldLength(4));
+ DataInput dataIn = new DataInputStream(inStream);
+ Integer res = IntegerSerializerDeserializer.INSTANCE.deserialize(dataIn);
+ results.add(res);
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ searchCursor.close();
+ }
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("There are " + results.size() + " objects that satisfy the query");
+ }
+ }
- System.err.println("There are " + results.size()
- + " objects that satisfy the query");
- }
-
- rtree.close();
- bufferCache.closeFile(fileId);
- bufferCache.close();
- }
+ rtree.close();
+ bufferCache.closeFile(fileId);
+ bufferCache.close();
+ }
}
\ No newline at end of file
diff --git a/hyracks-tests/hyracks-storage-common-test/pom.xml b/hyracks-tests/hyracks-storage-common-test/pom.xml
index ca68f85..cbe34ef 100644
--- a/hyracks-tests/hyracks-storage-common-test/pom.xml
+++ b/hyracks-tests/hyracks-storage-common-test/pom.xml
@@ -2,12 +2,12 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-common-test</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<build>
@@ -34,19 +34,19 @@
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-control-nc</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-storage-common</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-test-support</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<type>jar</type>
<scope>test</scope>
</dependency>
diff --git a/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java b/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java
new file mode 100644
index 0000000..6c91cdd
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-common-test/src/test/java/edu/uci/ics/hyracks/storage/common/BufferCacheRegressionTests.java
@@ -0,0 +1,180 @@
+package edu.uci.ics.hyracks.storage.common;
+
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+
+import edu.uci.ics.hyracks.api.context.IHyracksTaskContext;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileHandle;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.api.io.IIOManager;
+import edu.uci.ics.hyracks.api.io.IIOManager.FileReadWriteMode;
+import edu.uci.ics.hyracks.api.io.IIOManager.FileSyncMode;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
+import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public class BufferCacheRegressionTests {
+ protected static final String tmpDir = System.getProperty("java.io.tmpdir");
+ protected static final String sep = System.getProperty("file.separator");
+
+ protected String fileName = tmpDir + sep + "flushTestFile";
+
+ private static final int PAGE_SIZE = 256;
+ private static final int HYRACKS_FRAME_SIZE = PAGE_SIZE;
+ private IHyracksTaskContext ctx = TestUtils.create(HYRACKS_FRAME_SIZE);
+
+ // We want to test the following behavior when reclaiming a file slot in the
+ // buffer cache:
+ // 1. If the file being evicted was deleted, then its dirty pages should be
+ // invalidated, but most not be flushed.
+ // 2. If the file was not deleted, then we must flush its dirty pages.
+ @Test
+ public void testFlushBehaviorOnFileEviction() throws IOException {
+ File f = new File(fileName);
+ if (f.exists()) {
+ f.delete();
+ }
+ flushBehaviorTest(true);
+ flushBehaviorTest(false);
+ }
+
+ private void flushBehaviorTest(boolean deleteFile) throws IOException {
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, 10, 1);
+
+ IBufferCache bufferCache = TestStorageManagerComponentHolder
+ .getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder
+ .getFileMapProvider(ctx);
+
+ FileReference firstFileRef = new FileReference(new File(fileName));
+ bufferCache.createFile(firstFileRef);
+ int firstFileId = fmp.lookupFileId(firstFileRef);
+ bufferCache.openFile(firstFileId);
+
+ // Fill the first page with known data and make it dirty by write
+ // latching it.
+ ICachedPage writePage = bufferCache.pin(
+ BufferedFileHandle.getDiskPageId(firstFileId, 0), true);
+ writePage.acquireWriteLatch();
+ try {
+ ByteBuffer buf = writePage.getBuffer();
+ for (int i = 0; i < buf.capacity(); i++) {
+ buf.put(Byte.MAX_VALUE);
+ }
+ } finally {
+ writePage.releaseWriteLatch();
+ bufferCache.unpin(writePage);
+ }
+ bufferCache.closeFile(firstFileId);
+ if (deleteFile) {
+ bufferCache.deleteFile(firstFileId);
+ }
+
+ // Create a file with the same name.
+ FileReference secondFileRef = new FileReference(new File(fileName));
+ bufferCache.createFile(secondFileRef);
+ int secondFileId = fmp.lookupFileId(secondFileRef);
+
+ // This open will replace the firstFileRef's slot in the BufferCache,
+ // causing it's pages to be cleaned up. We want to make sure that those
+ // dirty pages are not flushed to the disk, because the file was
+ // declared as deleted, and
+ // somebody might be already using the same filename again (having been
+ // assigned a different fileId).
+ bufferCache.openFile(secondFileId);
+
+ // Manually open the file and inspect it's contents. We cannot simply
+ // ask the BufferCache to pin the page, because it would return the same
+ // physical memory again, and for performance reasons pages are never
+ // reset with 0's.
+ IIOManager ioManager = ctx.getIOManager();
+ FileReference testFileRef = new FileReference(new File(fileName));
+ FileHandle testFileHandle = new FileHandle(testFileRef);
+ testFileHandle.open(FileReadWriteMode.READ_ONLY,
+ FileSyncMode.METADATA_SYNC_DATA_SYNC);
+ ByteBuffer testBuffer = ByteBuffer.allocate(PAGE_SIZE);
+ ioManager.syncRead(testFileHandle, 0, testBuffer);
+ for (int i = 0; i < testBuffer.capacity(); i++) {
+ if (deleteFile) {
+ // We deleted the file. We expect to see a clean buffer.
+ if (testBuffer.get(i) == Byte.MAX_VALUE) {
+ fail("Page 0 of deleted file was fazily flushed in openFile(), "
+ + "corrupting the data of a newly created file with the same name.");
+ }
+ } else {
+ // We didn't delete the file. We expect to see a buffer full of
+ // Byte.MAX_VALUE.
+ if (testBuffer.get(i) != Byte.MAX_VALUE) {
+ fail("Page 0 of closed file was not flushed when properly, when reclaiming the file slot of fileId 0 in the BufferCache.");
+ }
+ }
+ }
+ testFileHandle.close();
+ bufferCache.closeFile(secondFileId);
+ if (deleteFile) {
+ bufferCache.deleteFile(secondFileId);
+ }
+ bufferCache.close();
+ }
+
+ // Tests the behavior of the BufferCache when more than all pages are
+ // pinned. We expect an exception.
+ @Test
+ public void testPinningAllPages() throws HyracksDataException {
+ int numPages = 10;
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, numPages, 1);
+
+ IBufferCache bufferCache = TestStorageManagerComponentHolder
+ .getBufferCache(ctx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder
+ .getFileMapProvider(ctx);
+
+ FileReference firstFileRef = new FileReference(new File(fileName));
+ bufferCache.createFile(firstFileRef);
+ int fileId = fmp.lookupFileId(firstFileRef);
+ bufferCache.openFile(fileId);
+
+ // Pin all pages.
+ ICachedPage[] pages = new ICachedPage[numPages];
+ for (int i = 0; i < numPages; ++i) {
+ pages[i] = bufferCache.pin(
+ BufferedFileHandle.getDiskPageId(fileId, i), true);
+ }
+
+ // Try to pin another page. We expect a HyracksDataException.
+ ICachedPage errorPage = null;
+ try {
+ errorPage = bufferCache.pin(
+ BufferedFileHandle.getDiskPageId(fileId, numPages), true);
+ } catch (HyracksDataException e) {
+ // This is the expected outcome.
+ // The BufferCache should still be able to function properly.
+ // Try unpinning all pages.
+ for (int i = 0; i < numPages; ++i) {
+ bufferCache.unpin(pages[i]);
+ }
+ // Now try pinning the page that failed above again.
+ errorPage = bufferCache.pin(
+ BufferedFileHandle.getDiskPageId(fileId, numPages), true);
+ // Unpin it.
+ bufferCache.unpin(errorPage);
+ // Cleanup.
+ bufferCache.closeFile(fileId);
+ bufferCache.close();
+ return;
+ } catch (Exception e) {
+ fail("Expected a HyracksDataException when pinning more pages than available but got another exception: "
+ + e.getMessage());
+ }
+ fail("Expected a HyracksDataException when pinning more pages than available.");
+ }
+}
diff --git a/hyracks-tests/pom.xml b/hyracks-tests/pom.xml
index 13abbd7..b120ba82 100644
--- a/hyracks-tests/pom.xml
+++ b/hyracks-tests/pom.xml
@@ -2,13 +2,13 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks-tests</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<packaging>pom</packaging>
<parent>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
</parent>
<modules>
diff --git a/pom.xml b/pom.xml
index 6771a7a..0ab0646 100644
--- a/pom.xml
+++ b/pom.xml
@@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.uci.ics.hyracks</groupId>
<artifactId>hyracks</artifactId>
- <version>0.1.8-SNAPSHOT</version>
+ <version>0.2.0-SNAPSHOT</version>
<packaging>pom</packaging>
<build>
@@ -15,6 +15,19 @@
<goals>package source:jar javadoc:jar deploy:deploy</goals>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>versions-maven-plugin</artifactId>
+ <version>1.2</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkMode>pertest</forkMode>
+ <argLine>-enableassertions -Djava.util.logging.config.file=${user.home}/logging.properties -Xdebug -Xrunjdwp:transport=dt_socket,server=y,address=8000,suspend=n</argLine>
+ </configuration>
+ </plugin>
</plugins>
</build>
@@ -66,14 +79,16 @@
</pluginRepositories>
<modules>
+ <module>hyracks-ipc</module>
<module>hyracks-api</module>
<module>hyracks-dataflow-common</module>
<module>hyracks-dataflow-std</module>
<module>hyracks-dataflow-hadoop</module>
<module>hyracks-control-common</module>
- <module>hyracks-admin-console</module>
<module>hyracks-control-cc</module>
+ <module>hyracks-net</module>
<module>hyracks-control-nc</module>
+ <module>hyracks-data</module>
<module>hyracks-cli</module>
<module>hyracks-storage-common</module>
<module>hyracks-storage-am-common</module>
@@ -86,5 +101,6 @@
<module>hyracks-examples</module>
<module>hyracks-documentation</module>
<module>hyracks-hadoop-compat</module>
+ <module>hyracks-algebricks</module>
</modules>
</project>
