DATAFU-153 datafu-spark support for Python 3
Signed-off-by: Matthew Hayes <mhayes@apache.org>
diff --git a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala
index 5ca9f0d..36e8615 100644
--- a/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala
+++ b/datafu-spark/src/main/scala/spark/utils/overwrites/SparkPythonRunner.scala
@@ -110,13 +110,16 @@
writer: BufferedWriter,
reader: BufferedReader): String = {
writer.write("import traceback\n")
+ writer.write("import sys\n")
writer.write("try:\n")
- writer.write(" execfile('" + filename + "')\n")
+ writer.write(" if sys.version_info < (3, 0):\n")
+ writer.write(s" execfile('$filename')\n")
+ writer.write(" else:\n")
+ writer.write(s" exec(open('$filename').read())\n")
writer.write(" print (\"*!?flush reader!?*\")\n")
writer.write("except Exception as e:\n")
writer.write(" traceback.print_exc()\n")
writer.write(" print (\"*!?flush error reader!?*\")\n\n")
-// writer.write(" exit(1)\n\n")
writer.flush()
var output = ""
var line: String = reader.readLine
diff --git a/datafu-spark/src/test/resources/python_tests/pyfromscala.py b/datafu-spark/src/test/resources/python_tests/pyfromscala.py
index 3162ff4..03fcf92 100644
--- a/datafu-spark/src/test/resources/python_tests/pyfromscala.py
+++ b/datafu-spark/src/test/resources/python_tests/pyfromscala.py
@@ -28,7 +28,7 @@
import os
-print os.getcwd()
+print(os.getcwd())
###############################################################
@@ -70,9 +70,9 @@
txt_df = sqlContext.read.csv('src/test/resources/text.csv', sep=DEL, schema=schema)
-print type(txt_df)
-print dir(txt_df)
-print txt_df.count()
+print(type(txt_df))
+print(dir(txt_df))
+print(txt_df.count())
txt_df.show()
