apache-jena/bin/tdbloader2index - jena - Git at Google

 #!/usr/bin/env bash

 ## Licensed to the Apache Software Foundation (ASF) under one
 ## or more contributor license agreements.  See the NOTICE file
 ## distributed with this work for additional information
 ## regarding copyright ownership.  The ASF licenses this file
 ## to you under the Apache License, Version 2.0 (the
 ## "License"); you may not use this file except in compliance
 ## with the License.  You may obtain a copy of the License at
 ##
 ##     http://www.apache.org/licenses/LICENSE-2.0
 ##
 ## Unless required by applicable law or agreed to in writing, software
 ## distributed under the License is distributed on an "AS IS" BASIS,
 ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 ## See the License for the specific language governing permissions and
 ## limitations under the License.

 # The environment for this sub-script is setup by "tdbloader2"

 function resolveLink() {
   local NAME=$1

   if [ -L "$NAME" ]; then
     case "$OSTYPE" in
       darwin*|bsd*)
         # BSD style readlink behaves differently to GNU readlink
         # Have to manually follow links
         while [ -L "$NAME" ]; do
           NAME=$(readlink -- "$NAME")
         done
         ;;
       *)
         # Assuming standard GNU readlink with -f for
         # canonicalize
         NAME=$(readlink -f -- "$NAME")
         ;;
     esac
   fi

   echo "$NAME"
 }

 # Pull in common functions
 if [ -z "$JENA_HOME" ]; then
   echo "JENA_HOME is not set"
   exit 1
 fi
 # If JENA_HOME is a symbolic link need to resolve
 if [ -L "${JENA_HOME}" ]; then
   JENA_HOME=$(resolveLink "$JENA_HOME")
   # If link is relative
   case "$JENA_HOME" in
     /*)
       # Already absolute
       ;;
     *)
       # Relative, make absolute
       JENA_HOME=$(dirname "$JENA_HOME")
       ;;
   esac
   export JENA_HOME
   echo "Resolved symbolic links for JENA_HOME to $JENA_HOME"
 fi

 if [ -e "${JENA_HOME}/bin/tdbloader2common" ]; then
   # Can source common functions
   source "${JENA_HOME}/bin/tdbloader2common"
 else
   echo "Unable to locate common functions script tdbloader2common"
   exit 1
 fi

 function printUsage() {
   cat << EOF
 tdbloader2index - TDB Bulk Loader - Index Phase

 Usage: tdbloader2index --loc <Directory> [Options]

 Bulk Loader for TDB which generates the Index files based upon the
 temporary data files generated by tdbloader2data.  This command relies
 on POSIX utilities so will only work on POSIX operating systems.

 This command can only be used to create new database. If you wish to
 bulk load to an existing database please use tdbloader instead.

 Required options are as follows:

   -l <DatabaseDirectory>
   --loc <DatabaseDirectory>
     Sets the location in which the database should be created.

     This location must be a directory and must be empty, if a
     non-existent path is specified it will be created as a new
     directory.

 Common additional options are as follows:

   -h
   --help
     Prints this help summary and exits

 Advanced additional options are as follows:

   -d
   --debug
     Enable debug mode, adds extra debug output

   -j <JvmArgs>
   --jvm-args <JvmArgs>
     Sets the arguments that should be passed to the JVM for the
     JVM based portions of the build.

     Generally it is best to not change these unless you have been
     specifically advised to.  The scripts will use appropriate
     defaults if this is not specified.

     In particular be careful increasing the heap size since many
     parts of TDB actually use memory mapped files that live
     outside the heap so if the heap is too large the heap may
     conflict with the memory mapped files for memory space.

   -k
   --keep-work
     Keeps the temporary work files around after they are no longer
     needed.  May be useful for debugging.

   -s <SortArgs>
   --sort-args <SortArgs>
     Sets the arguments that should be passed to sort for the sort
     based portions of the build.

     Generally it is best not to change these as the scripts will
     use appropriate defaults for your system.

   -t
   --trace
     Enable trace mode, essentially sets -x within the scripts
 EOF
 }

 # Exit on error.
 set -e

 # Sort order is ASCII
 export LC_ALL="C"

 # Process Arguments
 LOC=
 KEEP_WORK=0
 DEBUG=0
 JVM_ARGS=
 SORT_ARGS="${SORT_ARGS:-}"
 if [ -n "$SORT_ARGS" ]; then
   echo "Using SORT_ARGS: $SORT_ARGS"
 fi

 while [ $# -gt 0 ]
 do
   ARG=$1
   case "$ARG" in
     -d|--debug)
       # Debug Mode
       shift
       DEBUG=1
       ;;
     -h|--help)
       printUsage
       exit 0
       ;;
     -j|--jvm-args)
       # JVM Arguments
       shift
       JVM_ARGS="$1"
       shift
       ;;
     -k|--keep-work)
       # Keep work files
       shift
       KEEP_WORK=1
       ;;
     -l|--loc|-loc)
       # Location space separated
       shift
       LOC="$1"
       shift
       ;;
     -*loc=*)
       # Location = separated
       LOC=${ARG/-*loc=/}
       shift
       ;;
     -s|--sort-args)
       # Sort arguments
       shift
       SORT_ARGS=$1
       shift
       ;;
     -t|--trace)
       # Trace mode
       shift
       set -x
       ;;
     *)
       # Additional options are not supported
       abort 1 "Unrecognized option $ARG"
       ;;
   esac
 done

 # Verify arguments
 if [ -z "$LOC" ]; then
   abort 1 "Required database location not specified"
 fi

 # Make LOC absolute
 ABS_LOC=$(makeAbsolute "$LOC")
 if [ "$ABS_LOC" != "$LOC" ]; then
   LOC="$ABS_LOC"
   debug "Absolute database location is $LOC"
 fi

 # Check location
 if [ ! -e "$LOC" ]; then
   abort 1 "Database location specified does not exist: $LOC"
 fi
 if [ ! -d "$LOC" ]; then
   abort 1 "Database location is not a directory: $LOC"
 fi

 # Locate and check data text files
 DATA_TRIPLES="$LOC/data-triples.tmp"
 DATA_QUADS="$LOC/data-quads.tmp"

 if [ ! -e "$DATA_TRIPLES" ]; then
   abort 1 "No triples text file found in database location, please run the tdbloader2data script first"
 fi
 if [ ! -e "$DATA_QUADS" ]; then
   abort 1 "No quads text file found in database location, please run the tdbloader2data script first"
 fi

 debug "Data text files are $DATA_TRIPLES and $DATA_QUADS"

 # Prepare sort arguments
 if [ -z "$SORT_ARGS" ]; then
     SORT_ARGS="--buffer-size=50%"
     if [[ "$SORT_ARGS" != *"--parallel="* ]]; then
         # --parallel is not always available.
         # Temporarily disable exit on error while we check for --parallel support
         set +e
         sort --parallel=3 < /dev/null 2>/dev/null
         if [ $? = 0 ]; then
           SORT_ARGS="$SORT_ARGS --parallel=3"
         fi
         set -e
     fi
 fi

 # Prepare JVM arguments
 JVM_ARGS=${JVM_ARGS:--Xmx1200M}
 debug "JVM Arguments are $JVM_ARGS"

 # Classpath set in "tdbloader2"
 if [ -z "$JENA_CP" ]; then
   abort 1 "Classpath not provided : set JENA_CP"
 fi
 debug "Jena Classpath is $JENA_CP"

 # ---- Index intermediates
 # All files are written S P O / G S P O columns per row but in different sort orders.
 info "Index Building Phase"

 # Check where we are storing temporary sort files
 debug "Sort Arguments: $SORT_ARGS"
 SORT_TEMP_DIR=
 if [[ "$SORT_ARGS" == *"-T "* ]]; then
   # Specified via -T argument
   SORT_TEMP_DIR=(${SORT_ARGS/-T /})
   SORT_TEMP_DIR=${SORT_TEMP_DIR[0]}
 elif [[ "$SORT_ARGS" == *"--temporary-directory="* ]]; then
   # Specified via --temporary-directory argument
   SORT_TEMP_DIR=(${SORT_ARGS/--temporary-directory=/})
   SORT_TEMP_DIR=${SORT_TEMP_DIR[0]}
 else
   # Using the system temp directory
   SORT_TEMP_DIR="$TMPDIR"
 fi
 if [ -n "$SORT_TEMP_DIR" ]; then
   # If we've figured out the sort temporary directory then check it
   SORT_TEMP_DIR=$(makeAbsolute "$SORT_TEMP_DIR")
   debug "Sort Temp Directory: $SORT_TEMP_DIR"
   SORT_DRIVE_INFO=($(getDriveInfo "${SORT_TEMP_DIR}"))
   if [ "${#SORT_DRIVE_INFO[@]}" -gt 0 ]; then
     debug "Sort Temp Directory is on disk ${SORT_DRIVE_INFO[0]} which has ${SORT_DRIVE_INFO[2]}% free space (${SORT_DRIVE_INFO[3]} bytes)"

     if [ "${SORT_DRIVE_INFO[2]}" -le 10 ]; then
       warn "-----"
       warn "Sort Temp Directory ${SORT_TEMP_DIR} is on disk ${SORT_DRIVE_INFO[0]} which only has ${SORT_DRIVE_INFO[2]}% free space (${SORT_DRIVE_INFO[3]} bytes) available"
       warn "This may result in sort failures if the data to be indexed is large"
       warn "-----"
     fi
   fi
 fi

 generate_index()
 {
     local KEYS="$1"
     local DATA="$2"
     local IDX=$3
     local WORK="$LOC/$IDX-txt"

     if [ ! -s "$DATA" ]; then
       debug "Skipping Index $IDX as no relevant data to index"
       return
     fi

     info "Creating Index $IDX"

     # For various purposes we need to know the size of the input data
     local SIZE=$(getSize "$DATA")
     debug "Size of data to be sorted is $SIZE bytes"

     # Verify that we have enough space to sort the data

     # Firstly check that the output disk has sufficient space
     local WORK_DRIVE_INFO=($(getDriveInfo "$LOC"))
     if [ "${#WORK_DRIVE_INFO[@]}" -gt 0 ]; then
       if [ "${SIZE}" -ge "${WORK_DRIVE_INFO[3]}" ]; then
         # If there is insufficient disk space then we can abort now
         abort 1 "Insufficient free space on database drive ${WORK_DRIVE_INFO[0]}, there are ${WORK_DRIVE_INFO[3]} bytes free but ${SIZE} bytes are required"
       else
         debug "Sufficient free space on database drive ${WORK_DRIVE_INFO[0]} to attempt sorting data file ${DATA} (${SIZE} bytes required from ${WORK_DRIVE_INFO[3]} bytes free)"
       fi
     fi

     # Secondly check if there is enough space to sort in-memory or if sort may need to do an external sort
     # We only issue warnings when the sort is likely to be external because there are various factors
     # such as virtual memory and OS file caching that may complicate this
     FREE_MEM=$(getFreeMem)
     if [ "$FREE_MEM" -ge 0 ]; then
       if [ "$SIZE" -ge "$FREE_MEM" ]; then
         debug "Insufficient free memory to sort data in-memory, sort will need to perform an external sort using Temp Directory ${SORT_TEMP_DIR}"

         # Check for disk space on temporary disk
         if [ -n "${SORT_TEMP_DIR}" ]; then
           SORT_DRIVE_INFO=($(getDriveInfo "${SORT_TEMP_DIR}"))
           if [ "${#SORT_DRIVE_INFO[@]}" -gt 0 ]; then
             if [ "$SIZE" -ge "${SORT_DRIVE_INFO[3]}" ]; then
               warn "There may be insufficient for sort to perform an external sort using Temp Directory ${SORT_TEMP_DIR} (${SIZE} bytes required but only ${SORT_DRIVE_INFO[3]} bytes free)"
             fi
           fi
         fi
       else
         debug "Should be sufficient free memory ($FREE_MEM bytes) for sort to be fully in-memory"
       fi
     else
       debug "Unable to determine free memory on your OS, can't check whether sort will be in-memory or external sort using Temp Directory ${SORT_TEMP_DIR}"
     fi

     # Sort the input data
     info "Sort $IDX"
     debug "Sorting $DATA into work file $WORK"
     sort $SORT_ARGS -u $KEYS < "$DATA" > $WORK
     info "Sort $IDX Completed"

     # Build into an index
     info "Build $IDX"
     rm -f "$LOC/$IDX.dat"
     rm -f "$LOC/$IDX.idn"
     java $JVM_ARGS -cp "$JENA_CP" "$PKG".CmdIndexBuild "$LOC" "$IDX" "$WORK"
     info "Build $IDX Completed"

     # Remove work file unless keeping
     if [ $KEEP_WORK = 0 ]; then
       debug "Cleaning up work file $WORK"
       rm "$WORK"
     fi
 }

 K1="-k 1,1"
 K2="-k 2,2"
 K3="-k 3,3"
 K4="-k 4,4"

 generate_index "$K1 $K2 $K3" "$DATA_TRIPLES" SPO

 generate_index "$K2 $K3 $K1" "$DATA_TRIPLES" POS

 generate_index "$K3 $K1 $K2" "$DATA_TRIPLES" OSP

 generate_index "$K1 $K2 $K3 $K4" "$DATA_QUADS" GSPO

 generate_index "$K1 $K3 $K4 $K2" "$DATA_QUADS" GPOS

 generate_index "$K1 $K4 $K2 $K3" "$DATA_QUADS" GOSP

 generate_index "$K2 $K3 $K4 $K1" "$DATA_QUADS" SPOG

 generate_index "$K3 $K4 $K2 $K1" "$DATA_QUADS" POSG

 generate_index "$K4 $K2 $K3 $K1" "$DATA_QUADS" OSPG

 info "Index Building Phase Completed"

 # ---- Clean up.
 if [ $KEEP_WORK = 0 ]; then
   debug "Cleaning up data files $DATA_TRIPLES and $DATA_QUADS"
   rm -f "$DATA_TRIPLES" "$DATA_QUADS"
 fi
	#!/usr/bin/env bash

	## Licensed to the Apache Software Foundation (ASF) under one
	## or more contributor license agreements. See the NOTICE file
	## distributed with this work for additional information
	## regarding copyright ownership. The ASF licenses this file
	## to you under the Apache License, Version 2.0 (the
	## "License"); you may not use this file except in compliance
	## with the License. You may obtain a copy of the License at
	##
	## http://www.apache.org/licenses/LICENSE-2.0
	##
	## Unless required by applicable law or agreed to in writing, software
	## distributed under the License is distributed on an "AS IS" BASIS,
	## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	## See the License for the specific language governing permissions and
	## limitations under the License.

	# The environment for this sub-script is setup by "tdbloader2"

	function resolveLink() {
	local NAME=$1

	if [ -L "$NAME" ]; then
	case "$OSTYPE" in
	darwin\|bsd)
	# BSD style readlink behaves differently to GNU readlink
	# Have to manually follow links
	while [ -L "$NAME" ]; do
	NAME=$(readlink -- "$NAME")
	done
	;;
	*)
	# Assuming standard GNU readlink with -f for
	# canonicalize
	NAME=$(readlink -f -- "$NAME")
	;;
	esac
	fi

	echo "$NAME"
	}

	# Pull in common functions
	if [ -z "$JENA_HOME" ]; then
	echo "JENA_HOME is not set"
	exit 1
	fi
	# If JENA_HOME is a symbolic link need to resolve
	if [ -L "${JENA_HOME}" ]; then
	JENA_HOME=$(resolveLink "$JENA_HOME")
	# If link is relative
	case "$JENA_HOME" in
	/*)
	# Already absolute
	;;
	*)
	# Relative, make absolute
	JENA_HOME=$(dirname "$JENA_HOME")
	;;
	esac
	export JENA_HOME
	echo "Resolved symbolic links for JENA_HOME to $JENA_HOME"
	fi

	if [ -e "${JENA_HOME}/bin/tdbloader2common" ]; then
	# Can source common functions
	source "${JENA_HOME}/bin/tdbloader2common"
	else
	echo "Unable to locate common functions script tdbloader2common"
	exit 1
	fi

	function printUsage() {
	cat << EOF
	tdbloader2index - TDB Bulk Loader - Index Phase

	Usage: tdbloader2index --loc <Directory> [Options]

	Bulk Loader for TDB which generates the Index files based upon the
	temporary data files generated by tdbloader2data. This command relies
	on POSIX utilities so will only work on POSIX operating systems.

	This command can only be used to create new database. If you wish to
	bulk load to an existing database please use tdbloader instead.

	Required options are as follows:

	-l <DatabaseDirectory>
	--loc <DatabaseDirectory>
	Sets the location in which the database should be created.

	This location must be a directory and must be empty, if a
	non-existent path is specified it will be created as a new
	directory.

	Common additional options are as follows:

	-h
	--help
	Prints this help summary and exits

	Advanced additional options are as follows:

	-d
	--debug
	Enable debug mode, adds extra debug output

	-j <JvmArgs>
	--jvm-args <JvmArgs>
	Sets the arguments that should be passed to the JVM for the
	JVM based portions of the build.

	Generally it is best to not change these unless you have been
	specifically advised to. The scripts will use appropriate
	defaults if this is not specified.

	In particular be careful increasing the heap size since many
	parts of TDB actually use memory mapped files that live
	outside the heap so if the heap is too large the heap may
	conflict with the memory mapped files for memory space.

	-k
	--keep-work
	Keeps the temporary work files around after they are no longer
	needed. May be useful for debugging.

	-s <SortArgs>
	--sort-args <SortArgs>
	Sets the arguments that should be passed to sort for the sort
	based portions of the build.

	Generally it is best not to change these as the scripts will
	use appropriate defaults for your system.

	-t
	--trace
	Enable trace mode, essentially sets -x within the scripts
	EOF
	}

	# Exit on error.
	set -e

	# Sort order is ASCII
	export LC_ALL="C"

	# Process Arguments
	LOC=
	KEEP_WORK=0
	DEBUG=0
	JVM_ARGS=
	SORT_ARGS="${SORT_ARGS:-}"
	if [ -n "$SORT_ARGS" ]; then
	echo "Using SORT_ARGS: $SORT_ARGS"
	fi

	while [ $# -gt 0 ]
	do
	ARG=$1
	case "$ARG" in
	-d\|--debug)
	# Debug Mode
	shift
	DEBUG=1
	;;
	-h\|--help)
	printUsage
	exit 0
	;;
	-j\|--jvm-args)
	# JVM Arguments
	shift
	JVM_ARGS="$1"
	shift
	;;
	-k\|--keep-work)
	# Keep work files
	shift
	KEEP_WORK=1
	;;
	-l\|--loc\|-loc)
	# Location space separated
	shift
	LOC="$1"
	shift
	;;
	-loc=)
	# Location = separated
	LOC=${ARG/-*loc=/}
	shift
	;;
	-s\|--sort-args)
	# Sort arguments
	shift
	SORT_ARGS=$1
	shift
	;;
	-t\|--trace)
	# Trace mode
	shift
	set -x
	;;
	*)
	# Additional options are not supported
	abort 1 "Unrecognized option $ARG"
	;;
	esac
	done

	# Verify arguments
	if [ -z "$LOC" ]; then
	abort 1 "Required database location not specified"
	fi

	# Make LOC absolute
	ABS_LOC=$(makeAbsolute "$LOC")
	if [ "$ABS_LOC" != "$LOC" ]; then
	LOC="$ABS_LOC"
	debug "Absolute database location is $LOC"
	fi

	# Check location
	if [ ! -e "$LOC" ]; then
	abort 1 "Database location specified does not exist: $LOC"
	fi
	if [ ! -d "$LOC" ]; then
	abort 1 "Database location is not a directory: $LOC"
	fi

	# Locate and check data text files
	DATA_TRIPLES="$LOC/data-triples.tmp"
	DATA_QUADS="$LOC/data-quads.tmp"

	if [ ! -e "$DATA_TRIPLES" ]; then
	abort 1 "No triples text file found in database location, please run the tdbloader2data script first"
	fi
	if [ ! -e "$DATA_QUADS" ]; then
	abort 1 "No quads text file found in database location, please run the tdbloader2data script first"
	fi

	debug "Data text files are $DATA_TRIPLES and $DATA_QUADS"

	# Prepare sort arguments
	if [ -z "$SORT_ARGS" ]; then
	SORT_ARGS="--buffer-size=50%"
	if [[ "$SORT_ARGS" != "--parallel=" ]]; then
	# --parallel is not always available.
	# Temporarily disable exit on error while we check for --parallel support
	set +e
	sort --parallel=3 < /dev/null 2>/dev/null
	if [ $? = 0 ]; then
	SORT_ARGS="$SORT_ARGS --parallel=3"
	fi
	set -e
	fi
	fi

	# Prepare JVM arguments
	JVM_ARGS=${JVM_ARGS:--Xmx1200M}
	debug "JVM Arguments are $JVM_ARGS"

	# Classpath set in "tdbloader2"
	if [ -z "$JENA_CP" ]; then
	abort 1 "Classpath not provided : set JENA_CP"
	fi
	debug "Jena Classpath is $JENA_CP"

	# ---- Index intermediates
	# All files are written S P O / G S P O columns per row but in different sort orders.
	info "Index Building Phase"

	# Check where we are storing temporary sort files
	debug "Sort Arguments: $SORT_ARGS"
	SORT_TEMP_DIR=
	if [[ "$SORT_ARGS" == "-T " ]]; then
	# Specified via -T argument
	SORT_TEMP_DIR=(${SORT_ARGS/-T /})
	SORT_TEMP_DIR=${SORT_TEMP_DIR[0]}
	elif [[ "$SORT_ARGS" == "--temporary-directory=" ]]; then
	# Specified via --temporary-directory argument
	SORT_TEMP_DIR=(${SORT_ARGS/--temporary-directory=/})
	SORT_TEMP_DIR=${SORT_TEMP_DIR[0]}
	else
	# Using the system temp directory
	SORT_TEMP_DIR="$TMPDIR"
	fi
	if [ -n "$SORT_TEMP_DIR" ]; then
	# If we've figured out the sort temporary directory then check it
	SORT_TEMP_DIR=$(makeAbsolute "$SORT_TEMP_DIR")
	debug "Sort Temp Directory: $SORT_TEMP_DIR"
	SORT_DRIVE_INFO=($(getDriveInfo "${SORT_TEMP_DIR}"))
	if [ "${#SORT_DRIVE_INFO[@]}" -gt 0 ]; then
	debug "Sort Temp Directory is on disk ${SORT_DRIVE_INFO[0]} which has ${SORT_DRIVE_INFO[2]}% free space (${SORT_DRIVE_INFO[3]} bytes)"

	if [ "${SORT_DRIVE_INFO[2]}" -le 10 ]; then
	warn "-----"
	warn "Sort Temp Directory ${SORT_TEMP_DIR} is on disk ${SORT_DRIVE_INFO[0]} which only has ${SORT_DRIVE_INFO[2]}% free space (${SORT_DRIVE_INFO[3]} bytes) available"
	warn "This may result in sort failures if the data to be indexed is large"
	warn "-----"
	fi
	fi
	fi

	generate_index()
	{
	local KEYS="$1"
	local DATA="$2"
	local IDX=$3
	local WORK="$LOC/$IDX-txt"

	if [ ! -s "$DATA" ]; then
	debug "Skipping Index $IDX as no relevant data to index"
	return
	fi

	info "Creating Index $IDX"

	# For various purposes we need to know the size of the input data
	local SIZE=$(getSize "$DATA")
	debug "Size of data to be sorted is $SIZE bytes"

	# Verify that we have enough space to sort the data

	# Firstly check that the output disk has sufficient space
	local WORK_DRIVE_INFO=($(getDriveInfo "$LOC"))
	if [ "${#WORK_DRIVE_INFO[@]}" -gt 0 ]; then
	if [ "${SIZE}" -ge "${WORK_DRIVE_INFO[3]}" ]; then
	# If there is insufficient disk space then we can abort now
	abort 1 "Insufficient free space on database drive ${WORK_DRIVE_INFO[0]}, there are ${WORK_DRIVE_INFO[3]} bytes free but ${SIZE} bytes are required"
	else
	debug "Sufficient free space on database drive ${WORK_DRIVE_INFO[0]} to attempt sorting data file ${DATA} (${SIZE} bytes required from ${WORK_DRIVE_INFO[3]} bytes free)"
	fi
	fi

	# Secondly check if there is enough space to sort in-memory or if sort may need to do an external sort
	# We only issue warnings when the sort is likely to be external because there are various factors
	# such as virtual memory and OS file caching that may complicate this
	FREE_MEM=$(getFreeMem)
	if [ "$FREE_MEM" -ge 0 ]; then
	if [ "$SIZE" -ge "$FREE_MEM" ]; then
	debug "Insufficient free memory to sort data in-memory, sort will need to perform an external sort using Temp Directory ${SORT_TEMP_DIR}"

	# Check for disk space on temporary disk
	if [ -n "${SORT_TEMP_DIR}" ]; then
	SORT_DRIVE_INFO=($(getDriveInfo "${SORT_TEMP_DIR}"))
	if [ "${#SORT_DRIVE_INFO[@]}" -gt 0 ]; then
	if [ "$SIZE" -ge "${SORT_DRIVE_INFO[3]}" ]; then
	warn "There may be insufficient for sort to perform an external sort using Temp Directory ${SORT_TEMP_DIR} (${SIZE} bytes required but only ${SORT_DRIVE_INFO[3]} bytes free)"
	fi
	fi
	fi
	else
	debug "Should be sufficient free memory ($FREE_MEM bytes) for sort to be fully in-memory"
	fi
	else
	debug "Unable to determine free memory on your OS, can't check whether sort will be in-memory or external sort using Temp Directory ${SORT_TEMP_DIR}"
	fi

	# Sort the input data
	info "Sort $IDX"
	debug "Sorting $DATA into work file $WORK"
	sort $SORT_ARGS -u $KEYS < "$DATA" > $WORK
	info "Sort $IDX Completed"

	# Build into an index
	info "Build $IDX"
	rm -f "$LOC/$IDX.dat"
	rm -f "$LOC/$IDX.idn"
	java $JVM_ARGS -cp "$JENA_CP" "$PKG".CmdIndexBuild "$LOC" "$IDX" "$WORK"
	info "Build $IDX Completed"

	# Remove work file unless keeping
	if [ $KEEP_WORK = 0 ]; then
	debug "Cleaning up work file $WORK"
	rm "$WORK"
	fi
	}

	K1="-k 1,1"
	K2="-k 2,2"
	K3="-k 3,3"
	K4="-k 4,4"

	generate_index "$K1 $K2 $K3" "$DATA_TRIPLES" SPO

	generate_index "$K2 $K3 $K1" "$DATA_TRIPLES" POS

	generate_index "$K3 $K1 $K2" "$DATA_TRIPLES" OSP

	generate_index "$K1 $K2 $K3 $K4" "$DATA_QUADS" GSPO

	generate_index "$K1 $K3 $K4 $K2" "$DATA_QUADS" GPOS

	generate_index "$K1 $K4 $K2 $K3" "$DATA_QUADS" GOSP

	generate_index "$K2 $K3 $K4 $K1" "$DATA_QUADS" SPOG

	generate_index "$K3 $K4 $K2 $K1" "$DATA_QUADS" POSG

	generate_index "$K4 $K2 $K3 $K1" "$DATA_QUADS" OSPG

	info "Index Building Phase Completed"

	# ---- Clean up.
	if [ $KEEP_WORK = 0 ]; then
	debug "Cleaning up data files $DATA_TRIPLES and $DATA_QUADS"
	rm -f "$DATA_TRIPLES" "$DATA_QUADS"
	fi