make python code compatible with both python2 and python3 (#3412)
* make code compatible with both python2 and 3
diff --git a/examples/src/python/bolt/stateful_count_bolt.py b/examples/src/python/bolt/stateful_count_bolt.py
index 2f0eb1f..a3ea6b6 100644
--- a/examples/src/python/bolt/stateful_count_bolt.py
+++ b/examples/src/python/bolt/stateful_count_bolt.py
@@ -36,7 +36,7 @@
self.logger.info("Checkpoint Snapshot recovered : %s" % str(self.recovered_state))
def pre_save(self, checkpoint_id):
- for (k, v) in self.counter.items():
+ for (k, v) in list(self.counter.items()):
self.recovered_state.put(k, v)
self.logger.info("Checkpoint Snapshot %s : %s" % (checkpoint_id, str(self.recovered_state)))
diff --git a/examples/src/python/spout/stateful_word_spout.py b/examples/src/python/spout/stateful_word_spout.py
index b127167..9f368df 100644
--- a/examples/src/python/spout/stateful_word_spout.py
+++ b/examples/src/python/spout/stateful_word_spout.py
@@ -37,7 +37,7 @@
def pre_save(self, checkpoint_id):
# Purely for debugging purposes
- for (k, v) in self.counter.items():
+ for (k, v) in list(self.counter.items()):
self.recovered_state.put(k, v)
self.logger.info("Checkpoint Snapshot %s : %s" % (checkpoint_id, str(self.recovered_state)))
diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py
index 01f59a9..7fe871c 100755
--- a/heron/executor/src/python/heron_executor.py
+++ b/heron/executor/src/python/heron_executor.py
@@ -236,11 +236,10 @@
self.tmaster_stats_port = parsed_args.tmaster_stats_port
self.heron_internals_config_file = parsed_args.heron_internals_config_file
self.override_config_file = parsed_args.override_config_file
- self.component_ram_map =\
- map(lambda x: {x.split(':')[0]:
- int(x.split(':')[1])}, parsed_args.component_ram_map.split(','))
- self.component_ram_map =\
- functools.reduce(lambda x, y: dict(x.items() + y.items()), self.component_ram_map)
+ self.component_ram_map = [{x.split(':')[0]:int(x.split(':')[1])}
+ for x in parsed_args.component_ram_map.split(',')]
+ self.component_ram_map = functools.reduce(lambda x, y: dict(list(x.items()) + list(y.items())),
+ self.component_ram_map)
# component_jvm_opts_in_base64 itself is a base64-encoding-json-map, which is appended with
# " at the start and end. It also escapes "=" to "&equals" due to aurora limitation
@@ -256,7 +255,7 @@
base64.b64decode(parsed_args.component_jvm_opts.
lstrip('"').rstrip('"').replace('(61)', '=').replace('=', '='))
if component_jvm_opts_in_json != "":
- for (k, v) in json.loads(component_jvm_opts_in_json).items():
+ for (k, v) in list(json.loads(component_jvm_opts_in_json).items()):
# In json, the component name and JVM options are still in base64 encoding
self.component_jvm_opts[base64.b64decode(k)] = base64.b64decode(v)
@@ -366,7 +365,7 @@
parser.add_argument("--is-stateful", required=True)
parser.add_argument("--checkpoint-manager-classpath", required=True)
parser.add_argument("--checkpoint-manager-port", required=True)
- parser.add_argument("--checkpoint-manager-ram", type=long, required=True)
+ parser.add_argument("--checkpoint-manager-ram", type=int, required=True)
parser.add_argument("--stateful-config-file", required=True)
parser.add_argument("--health-manager-mode", required=True)
parser.add_argument("--health-manager-classpath", required=True)
@@ -793,7 +792,7 @@
'--zkhostportlist=%s' % self.state_manager_connection,
'--zkroot=%s' % self.state_manager_root,
'--stmgr_id=%s' % self.stmgr_ids[self.shard],
- '--instance_ids=%s' % ','.join(map(lambda x: x[0], instance_info)),
+ '--instance_ids=%s' % ','.join([x[0] for x in instance_info]),
'--myhost=%s' % self.master_host,
'--data_port=%s' % str(self.master_port),
'--local_data_port=%s' % str(self.tmaster_controller_port),
@@ -958,8 +957,8 @@
def _kill_processes(self, commands):
# remove the command from processes_to_monitor and kill the process
with self.process_lock:
- for command_name, command in commands.items():
- for process_info in self.processes_to_monitor.values():
+ for command_name, command in list(commands.items()):
+ for process_info in list(self.processes_to_monitor.values()):
if process_info.name == command_name:
del self.processes_to_monitor[process_info.pid]
Log.info("Killing %s process with pid %d: %s" %
@@ -978,7 +977,7 @@
Log.info("Start processes")
processes_to_monitor = {}
# First start all the processes
- for (name, command) in commands.items():
+ for (name, command) in list(commands.items()):
p = self._run_process(name, command)
processes_to_monitor[p.pid] = ProcessInfo(p, name, command)
@@ -999,7 +998,7 @@
(pid, status) = os.wait()
with self.process_lock:
- if pid in self.processes_to_monitor.keys():
+ if pid in list(self.processes_to_monitor.keys()):
old_process_info = self.processes_to_monitor[pid]
name = old_process_info.name
command = old_process_info.command
@@ -1061,10 +1060,10 @@
# if the current command has a matching command in the updated commands we keep it
# otherwise we kill it
- for current_name, current_command in current_commands.items():
+ for current_name, current_command in list(current_commands.items()):
# We don't restart tmaster since it watches the packing plan and updates itself. The stream
# manager is restarted just to reset state, but we could update it to do so without a restart
- if current_name in updated_commands.keys() and \
+ if current_name in list(updated_commands.keys()) and \
current_command == updated_commands[current_name] and \
not current_name.startswith('stmgr-'):
commands_to_keep[current_name] = current_command
@@ -1072,8 +1071,8 @@
commands_to_kill[current_name] = current_command
# updated commands not in the keep list need to be started
- for updated_name, updated_command in updated_commands.items():
- if updated_name not in commands_to_keep.keys():
+ for updated_name, updated_command in list(updated_commands.items()):
+ if updated_name not in list(commands_to_keep.keys()):
commands_to_start[updated_name] = updated_command
return commands_to_kill, commands_to_keep, commands_to_start
@@ -1083,8 +1082,8 @@
Then starts new ones required and kills old ones no longer required.
'''
with self.process_lock:
- current_commands = dict(map((lambda process: (process.name, process.command)),
- self.processes_to_monitor.values()))
+ current_commands = dict(list(map((lambda process: (process.name, process.command)),
+ list(self.processes_to_monitor.values()))))
updated_commands = self.get_commands_to_run()
# get the commands to kill, keep and start
@@ -1176,7 +1175,7 @@
Log.info('Executor terminated; exiting all process in executor.')
# Kill child processes first and wait for log collection to finish
- for pid in executor.processes_to_monitor.keys():
+ for pid in list(executor.processes_to_monitor.keys()):
os.kill(pid, signal.SIGTERM)
time.sleep(5)
@@ -1192,7 +1191,7 @@
sid = os.getsid(pid)
# POSIX prohibits the change of the process group ID of a session leader
- if pid <> sid:
+ if pid != sid:
Log.info('Set up process group; executor becomes leader')
os.setpgrp() # create new process group, become its leader
diff --git a/heron/executor/tests/python/heron_executor_unittest.py b/heron/executor/tests/python/heron_executor_unittest.py
index 7fbd800..a69ee72 100644
--- a/heron/executor/tests/python/heron_executor_unittest.py
+++ b/heron/executor/tests/python/heron_executor_unittest.py
@@ -92,7 +92,7 @@
def build_packing_plan(self, instance_distribution):
packing_plan = PackingPlan()
- for container_id in instance_distribution.keys():
+ for container_id in list(instance_distribution.keys()):
container_plan = packing_plan.container_plans.add()
container_plan.id = int(container_id)
for (component_name, global_task_id, component_index) in instance_distribution[container_id]:
@@ -293,11 +293,11 @@
def test_update_packing_plan(self):
self.executor_0.update_packing_plan(self.packing_plan_expected)
- self.assertEquals(self.packing_plan_expected, self.executor_0.packing_plan)
- self.assertEquals({1: "stmgr-1", 7: "stmgr-7"}, self.executor_0.stmgr_ids)
- self.assertEquals(
+ self.assertEqual(self.packing_plan_expected, self.executor_0.packing_plan)
+ self.assertEqual({1: "stmgr-1", 7: "stmgr-7"}, self.executor_0.stmgr_ids)
+ self.assertEqual(
{0: "metricsmgr-0", 1: "metricsmgr-1", 7: "metricsmgr-7"}, self.executor_0.metricsmgr_ids)
- self.assertEquals(
+ self.assertEqual(
{0: "heron-shell-0", 1: "heron-shell-1", 7: "heron-shell-7"}, self.executor_0.heron_shell_ids)
def test_launch_container_0(self):
@@ -315,17 +315,13 @@
monitored_processes = executor.processes_to_monitor
# convert to (pid, name, command)
- found_processes = list(map(lambda process_info:
- (process_info.pid, process_info.name, process_info.command_str),
- executor.processes))
- found_monitored = list(map(lambda pinfo:
- (pinfo[0], pinfo[1].name, pinfo[1].command_str),
- monitored_processes.items()))
+ found_processes = list([(process_info.pid, process_info.name, process_info.command_str) for process_info in executor.processes])
+ found_monitored = list([(pinfo[0], pinfo[1].name, pinfo[1].command_str) for pinfo in list(monitored_processes.items())])
found_processes.sort(key=lambda tuple: tuple[0])
found_monitored.sort(key=lambda tuple: tuple[0])
print("do_test_commands - found_processes: %s found_monitored: %s" \
% (found_processes, found_monitored))
- self.assertEquals(found_processes, found_monitored)
+ self.assertEqual(found_processes, found_monitored)
print("do_test_commands - expected_processes: %s monitored_processes: %s" \
% (expected_processes, monitored_processes))
@@ -337,18 +333,18 @@
current_commands = self.executor_1.get_commands_to_run()
temp_dict = dict(
- map((lambda process_info: (process_info.name, process_info.command.split(' '))),
- self.expected_processes_container_1))
+ list(map((lambda process_info: (process_info.name, process_info.command.split(' '))),
+ self.expected_processes_container_1)))
current_json = json.dumps(current_commands, sort_keys=True, cls=CommandEncoder).split(' ')
temp_json = json.dumps(temp_dict, sort_keys=True).split(' ')
- print ("current_json: %s" % current_json)
- print ("temp_json: %s" % temp_json)
+ print("current_json: %s" % current_json)
+ print("temp_json: %s" % temp_json)
# better test error report
for (s1, s2) in zip(current_json, temp_json):
- self.assertEquals(s1, s2)
+ self.assertEqual(s1, s2)
# update instance distribution
new_packing_plan = self.build_packing_plan(
@@ -360,20 +356,20 @@
commands_to_kill, commands_to_keep, commands_to_start = \
self.executor_1.get_command_changes(current_commands, updated_commands)
- self.assertEquals(['container_1_exclaim1_2', 'stmgr-1'], sorted(commands_to_kill.keys()))
- self.assertEquals(
+ self.assertEqual(['container_1_exclaim1_2', 'stmgr-1'], sorted(commands_to_kill.keys()))
+ self.assertEqual(
['container_1_exclaim1_1', 'container_1_word_3', 'heron-shell-1', 'metricsmgr-1'],
sorted(commands_to_keep.keys()))
- self.assertEquals(['container_1_word_2', 'stmgr-1'], sorted(commands_to_start.keys()))
+ self.assertEqual(['container_1_word_2', 'stmgr-1'], sorted(commands_to_start.keys()))
def assert_processes(self, expected_processes, found_processes):
- self.assertEquals(len(expected_processes), len(found_processes))
+ self.assertEqual(len(expected_processes), len(found_processes))
for expected_process in expected_processes:
self.assert_process(expected_process, found_processes)
def assert_process(self, expected_process, found_processes):
pid = expected_process.pid
self.assertTrue(found_processes[pid])
- self.assertEquals(expected_process.name, found_processes[pid].name)
- self.assertEquals(expected_process.command, found_processes[pid].command_str)
- self.assertEquals(1, found_processes[pid].attempts)
+ self.assertEqual(expected_process.name, found_processes[pid].name)
+ self.assertEqual(expected_process.command, found_processes[pid].command_str)
+ self.assertEqual(1, found_processes[pid].attempts)
diff --git a/heron/instance/src/python/BUILD b/heron/instance/src/python/BUILD
index 706cfbd..0e0fe9e 100644
--- a/heron/instance/src/python/BUILD
+++ b/heron/instance/src/python/BUILD
@@ -19,6 +19,7 @@
deps = [":instance-py"],
reqs = [
'colorlog==2.6.1',
+ 'future==0.18.2',
'PyYAML==3.13'
]
)
diff --git a/heron/instance/src/python/basics/__init__.py b/heron/instance/src/python/basics/__init__.py
index 58be1fb..f8e1609 100644
--- a/heron/instance/src/python/basics/__init__.py
+++ b/heron/instance/src/python/basics/__init__.py
@@ -17,5 +17,5 @@
'''module for basic python heron component'''
__all__ = ['bolt_instance.py', 'spout_instance.py', 'base_instance']
-from bolt_instance import BoltInstance
-from spout_instance import SpoutInstance
+from .bolt_instance import BoltInstance
+from .spout_instance import SpoutInstance
diff --git a/heron/instance/src/python/basics/bolt_instance.py b/heron/instance/src/python/basics/bolt_instance.py
index b158945..972baf0 100644
--- a/heron/instance/src/python/basics/bolt_instance.py
+++ b/heron/instance/src/python/basics/bolt_instance.py
@@ -21,7 +21,7 @@
'''bolt_instance.py: module for base bolt for python topology'''
import time
-import Queue
+import queue
import heronpy.api.api_constants as api_constants
from heronpy.api.state.stateful_component import StatefulComponent
@@ -181,7 +181,7 @@
while not self.in_stream.is_empty():
try:
tuples = self.in_stream.poll()
- except Queue.Empty:
+ except queue.Empty:
break
if isinstance(tuples, tuple_pb2.HeronTupleSet):
diff --git a/heron/instance/src/python/basics/spout_instance.py b/heron/instance/src/python/basics/spout_instance.py
index 0d1bf92..3b4dccf 100644
--- a/heron/instance/src/python/basics/spout_instance.py
+++ b/heron/instance/src/python/basics/spout_instance.py
@@ -20,7 +20,7 @@
'''spout_instance.py: module for base spout for python topology'''
-import Queue
+import queue
import time
import collections
@@ -185,7 +185,7 @@
while not self.in_stream.is_empty():
try:
tuples = self.in_stream.poll()
- except Queue.Empty:
+ except queue.Empty:
break
if isinstance(tuples, tuple_pb2.HeronTupleSet):
diff --git a/heron/instance/src/python/network/gateway_looper.py b/heron/instance/src/python/network/gateway_looper.py
index ca4f3b5..4b13605 100644
--- a/heron/instance/src/python/network/gateway_looper.py
+++ b/heron/instance/src/python/network/gateway_looper.py
@@ -26,7 +26,7 @@
import time
import select
-from event_looper import EventLooper
+from .event_looper import EventLooper
from heron.common.src.python.utils.log import Log
class GatewayLooper(EventLooper):
@@ -83,7 +83,7 @@
error_lst = []
if self.sock_map is not None:
- for fd, obj in self.sock_map.items():
+ for fd, obj in list(self.sock_map.items()):
is_r = obj.readable()
is_w = obj.writable()
if is_r:
diff --git a/heron/instance/src/python/utils/metrics/metrics_helper.py b/heron/instance/src/python/utils/metrics/metrics_helper.py
index 6762ebb..39555ba 100644
--- a/heron/instance/src/python/utils/metrics/metrics_helper.py
+++ b/heron/instance/src/python/utils/metrics/metrics_helper.py
@@ -41,7 +41,7 @@
def register_metrics(self, metrics_collector, interval):
"""Registers its metrics to a given metrics collector with a given interval"""
- for field, metrics in self.metrics.items():
+ for field, metrics in list(self.metrics.items()):
metrics_collector.register_metric(field, metrics, interval)
def update_count(self, name, incr_by=1, key=None):
@@ -379,7 +379,7 @@
if metric_value is None:
return
elif isinstance(metric_value, dict):
- for key, value in metric_value.items():
+ for key, value in list(metric_value.items()):
if key is not None and value is not None:
self._add_data_to_message(message, name + "/" + str(key), value)
self._add_data_to_message(message, "%s/%s" % (name, str(key)), value)
diff --git a/heron/instance/src/python/utils/misc/communicator.py b/heron/instance/src/python/utils/misc/communicator.py
index f8eb5ad..f43d4d8 100644
--- a/heron/instance/src/python/utils/misc/communicator.py
+++ b/heron/instance/src/python/utils/misc/communicator.py
@@ -20,7 +20,7 @@
'''communicator.py: module responsible for communication between Python heron modules'''
import sys
-import Queue
+from queue import Queue, Full, Empty
from heron.common.src.python.utils.log import Log
@@ -40,7 +40,7 @@
"""
self._producer_callback = producer_cb
self._consumer_callback = consumer_cb
- self._buffer = Queue.Queue()
+ self._buffer = Queue()
self.capacity = sys.maxsize
def register_capacity(self, capacity):
@@ -72,9 +72,9 @@
if self._producer_callback is not None:
self._producer_callback()
return ret
- except Queue.Empty:
+ except Empty:
Log.debug("%s: Empty in poll()" % str(self))
- raise Queue.Empty
+ raise Empty
def offer(self, item):
"""Offer to the buffer
@@ -87,9 +87,9 @@
if self._consumer_callback is not None:
self._consumer_callback()
return True
- except Queue.Full:
+ except Full:
Log.debug("%s: Full in offer()" % str(self))
- raise Queue.Full
+ raise Full
def clear(self):
"""Clear the buffer"""
diff --git a/heron/instance/src/python/utils/misc/custom_grouping_helper.py b/heron/instance/src/python/utils/misc/custom_grouping_helper.py
index 64b5a1a..18819d4 100644
--- a/heron/instance/src/python/utils/misc/custom_grouping_helper.py
+++ b/heron/instance/src/python/utils/misc/custom_grouping_helper.py
@@ -45,7 +45,7 @@
def prepare(self, context):
"""Prepares the custom grouping for this component"""
- for stream_id, targets in self.targets.items():
+ for stream_id, targets in list(self.targets.items()):
for target in targets:
target.prepare(context, stream_id)
diff --git a/heron/instance/src/python/utils/system_config.py b/heron/instance/src/python/utils/system_config.py
index 7cd251a..6327578 100644
--- a/heron/instance/src/python/utils/system_config.py
+++ b/heron/instance/src/python/utils/system_config.py
@@ -28,7 +28,7 @@
def merge(default, override):
if isinstance(default, dict) and isinstance(override, dict):
- for k, v in override.items():
+ for k, v in list(override.items()):
Log.info("Add overriding configuration '%s'", k)
if k not in default:
default[k] = v
diff --git a/heron/instance/src/python/utils/topology/topology_context_impl.py b/heron/instance/src/python/utils/topology/topology_context_impl.py
index 9824069..95687a0 100644
--- a/heron/instance/src/python/utils/topology/topology_context_impl.py
+++ b/heron/instance/src/python/utils/topology/topology_context_impl.py
@@ -110,7 +110,7 @@
def get_component_tasks(self, component_id):
"""Returns the task ids allocated for the given component id"""
ret = []
- for task_id, comp_id in self.task_to_component_map.items():
+ for task_id, comp_id in list(self.task_to_component_map.items()):
if comp_id == component_id:
ret.append(task_id)
return ret
diff --git a/heron/instance/tests/python/BUILD b/heron/instance/tests/python/BUILD
index 4776db2..c7aea1e 100644
--- a/heron/instance/tests/python/BUILD
+++ b/heron/instance/tests/python/BUILD
@@ -12,5 +12,6 @@
"pytest==3.2.2",
"unittest2==1.1.0",
"mock==1.0.1",
+ 'future==0.18.2',
],
)
diff --git a/heron/instance/tests/python/mock_protobuf.py b/heron/instance/tests/python/mock_protobuf.py
index 13e6d4d..13ed8e3 100644
--- a/heron/instance/tests/python/mock_protobuf.py
+++ b/heron/instance/tests/python/mock_protobuf.py
@@ -38,7 +38,7 @@
proto_config = topology_pb2.Config()
config_serializer = PythonSerializer()
assert isinstance(config_dict, dict)
- for key, value in config_dict.items():
+ for key, value in list(config_dict.items()):
if isinstance(value, bool):
kvs = proto_config.kvs.add()
kvs.key = key
diff --git a/heron/instance/tests/python/utils/communicator_unittest.py b/heron/instance/tests/python/utils/communicator_unittest.py
index 2865f2f..31f3a48 100644
--- a/heron/instance/tests/python/utils/communicator_unittest.py
+++ b/heron/instance/tests/python/utils/communicator_unittest.py
@@ -20,7 +20,7 @@
# pylint: disable=missing-docstring
-import Queue
+from queue import Empty
import unittest
from heron.instance.src.python.utils.misc import HeronCommunicator
@@ -41,7 +41,7 @@
def test_empty(self):
communicator = HeronCommunicator(producer_cb=None, consumer_cb=None)
- with self.assertRaises(Queue.Empty):
+ with self.assertRaises(Empty):
communicator.poll()
def test_producer_callback(self):
diff --git a/heron/instance/tests/python/utils/metrics_helper_unittest.py b/heron/instance/tests/python/utils/metrics_helper_unittest.py
index 7e81c8d..ad22024 100644
--- a/heron/instance/tests/python/utils/metrics_helper_unittest.py
+++ b/heron/instance/tests/python/utils/metrics_helper_unittest.py
@@ -45,7 +45,7 @@
def test_register_metrics(self):
self.metrics_helper.register_metrics(self.metrics_collector, 60)
- for name, metric in self.metrics.items():
+ for name, metric in list(self.metrics.items()):
self.assertEqual(self.metrics_collector.metrics_map[name], metric)
self.assertEqual(len(self.metrics_collector.time_bucket_in_sec_to_metrics_name[60]), 4)
self.assertIn(60, self.metrics_collector.registered_timers)
diff --git a/heron/instance/tests/python/utils/mock_generator.py b/heron/instance/tests/python/utils/mock_generator.py
index 5ece51d..8809096 100644
--- a/heron/instance/tests/python/utils/mock_generator.py
+++ b/heron/instance/tests/python/utils/mock_generator.py
@@ -93,8 +93,8 @@
pplan = mock_protobuf.get_mock_pplan(topology=topology, instances=instances)
keys = ["instance_id", "task_id", "comp_index", "comp_name"]
- zipped = zip(instance_ids, task_ids, comp_indexes, comp_names)
- return pplan, [dict(zip(keys, z)) for z in zipped]
+ zipped = list(zip(instance_ids, task_ids, comp_indexes, comp_names))
+ return pplan, [dict(list(zip(keys, z))) for z in zipped]
def make_data_tuple_from_list(lst, serializer=PythonSerializer()):
"""Make HeronDataTuple from a list of objects"""
diff --git a/heron/shell/src/python/BUILD b/heron/shell/src/python/BUILD
index 505e0f7..f263395 100644
--- a/heron/shell/src/python/BUILD
+++ b/heron/shell/src/python/BUILD
@@ -11,6 +11,7 @@
reqs = [
"requests==2.12.3",
"tornado==4.0.2",
+ "future==0.18.2",
],
)
diff --git a/heron/shell/src/python/handlers/__init__.py b/heron/shell/src/python/handlers/__init__.py
index 264022b..ce39dcc 100644
--- a/heron/shell/src/python/handlers/__init__.py
+++ b/heron/shell/src/python/handlers/__init__.py
@@ -15,15 +15,15 @@
# specific language governing permissions and limitations
# under the License.
''' __init__ '''
-from browsehandler import BrowseHandler
-from downloadhandler import DownloadHandler
-from filedatahandler import FileDataHandler
-from filehandler import FileHandler
-from filestatshandler import FileStatsHandler
-from jmaphandler import JmapHandler
-from jstackhandler import JstackHandler
-from memoryhistogramhandler import MemoryHistogramHandler
-from pmaphandler import PmapHandler
-from pidhandler import PidHandler
-from killexecutorhandler import KillExecutorHandler
-from healthhandler import HealthHandler
+from .browsehandler import BrowseHandler
+from .downloadhandler import DownloadHandler
+from .filedatahandler import FileDataHandler
+from .filehandler import FileHandler
+from .filestatshandler import FileStatsHandler
+from .jmaphandler import JmapHandler
+from .jstackhandler import JstackHandler
+from .memoryhistogramhandler import MemoryHistogramHandler
+from .pmaphandler import PmapHandler
+from .pidhandler import PidHandler
+from .killexecutorhandler import KillExecutorHandler
+from .healthhandler import HealthHandler
diff --git a/heron/shell/src/python/handlers/killexecutorhandler.py b/heron/shell/src/python/handlers/killexecutorhandler.py
index 4f899a0..fb2e56d 100644
--- a/heron/shell/src/python/handlers/killexecutorhandler.py
+++ b/heron/shell/src/python/handlers/killexecutorhandler.py
@@ -20,10 +20,13 @@
''' killexecutorhandler.py '''
+from future.standard_library import install_aliases
+install_aliases()
+
import logging
import os
import signal
-import urlparse
+from urllib.parse import parse_qsl
import tornado.web
from tornado.options import options
@@ -46,7 +49,7 @@
logger = logging.getLogger(__file__)
logger.info("Received 'Killing process' request")
- data = dict(urlparse.parse_qsl(self.request.body))
+ data = dict(parse_qsl(self.request.body))
# check shared secret
sharedSecret = data.get('secret')
diff --git a/heron/shell/src/python/utils.py b/heron/shell/src/python/utils.py
index 5df4cca..ffbe2ce 100644
--- a/heron/shell/src/python/utils.py
+++ b/heron/shell/src/python/utils.py
@@ -182,7 +182,7 @@
Feed output of one command to the next and return final output
Returns string output of chained application of commands.
"""
- command = ' | '.join(map(lambda x: ' '.join(x), cmd_list))
+ command = ' | '.join([' '.join(x) for x in cmd_list])
chained_proc = functools.reduce(pipe, [None] + cmd_list)
stdout_builder = proc.async_stdout_builder(chained_proc)
chained_proc.wait()
diff --git a/heron/statemgrs/src/python/config.py b/heron/statemgrs/src/python/config.py
index c06be6c..14975b3 100644
--- a/heron/statemgrs/src/python/config.py
+++ b/heron/statemgrs/src/python/config.py
@@ -46,7 +46,7 @@
"""
Names of all state locations must be unique.
"""
- names = map(lambda loc: loc["name"], self.locations)
+ names = [loc["name"] for loc in self.locations]
assert len(names) == len(set(names)), "Names of state locations must be unique"
def get_all_state_locations(self):
@@ -59,4 +59,4 @@
"""
Return a list of state locations of a particular type.
"""
- return filter(lambda location: location['type'] == location_type, self.locations)
+ return [location for location in self.locations if location['type'] == location_type]
diff --git a/heron/statemgrs/src/python/filestatemanager.py b/heron/statemgrs/src/python/filestatemanager.py
index fa620bb..ca56e56 100644
--- a/heron/statemgrs/src/python/filestatemanager.py
+++ b/heron/statemgrs/src/python/filestatemanager.py
@@ -99,7 +99,7 @@
For all the topologies in the watchers, check if the data
in directory has changed. Trigger the callback if it has.
"""
- for topology, callbacks in watchers.items():
+ for topology, callbacks in list(watchers.items()):
file_path = os.path.join(path, topology)
data = ""
if os.path.exists(file_path):
@@ -117,9 +117,8 @@
topologies = []
if os.path.isdir(topologies_path):
- topologies = list(filter(
- lambda f: os.path.isfile(os.path.join(topologies_path, f)),
- os.listdir(topologies_path)))
+ topologies = list([f for f in os.listdir(topologies_path)
+ if os.path.isfile(os.path.join(topologies_path, f))])
if set(topologies) != set(self.topologies_directory):
for callback in self.topologies_watchers:
callback(topologies)
@@ -166,8 +165,8 @@
self.topologies_watchers.append(callback)
else:
topologies_path = self.get_topologies_path()
- return filter(lambda f: os.path.isfile(os.path.join(topologies_path, f)),
- os.listdir(topologies_path))
+ return [f for f in os.listdir(topologies_path)
+ if os.path.isfile(os.path.join(topologies_path, f))]
def get_topology(self, topologyName, callback=None):
"""get topology"""
diff --git a/heron/statemgrs/src/python/statemanager.py b/heron/statemgrs/src/python/statemanager.py
index 94066f1..a4affff 100644
--- a/heron/statemgrs/src/python/statemanager.py
+++ b/heron/statemgrs/src/python/statemanager.py
@@ -20,8 +20,10 @@
''' statemanager.py '''
import abc
+
import socket
import subprocess
+import six
from heron.statemgrs.src.python.log import Log as LOG
@@ -33,15 +35,13 @@
HERON_TOPOLOGIES_KEY = "{0}/topologies"
# pylint: disable=too-many-public-methods, attribute-defined-outside-init
-class StateManager:
+class StateManager(six.with_metaclass(abc.ABCMeta)):
"""
This is the abstract base class for state manager. It provides methods to get/set/delete various
state from the state store. The getters accept an optional callback, which will watch for state
changes of the object and invoke the callback when one occurs.
"""
- __metaclass__ = abc.ABCMeta
-
TIMEOUT_SECONDS = 5
@property
diff --git a/heron/statemgrs/src/python/zkstatemanager.py b/heron/statemgrs/src/python/zkstatemanager.py
index f130ead..438a019 100644
--- a/heron/statemgrs/src/python/zkstatemanager.py
+++ b/heron/statemgrs/src/python/zkstatemanager.py
@@ -40,7 +40,7 @@
from kazoo.exceptions import ZookeeperError
def _makehostportlist(hostportlist):
- return ','.join(map(lambda hp: "%s:%i" % hp, hostportlist))
+ return ','.join(["%s:%i" % hp for hp in hostportlist])
# pylint: disable=attribute-defined-outside-init
class ZkStateManager(StateManager):
diff --git a/heron/statemgrs/tests/python/configloader_unittest.py b/heron/statemgrs/tests/python/configloader_unittest.py
index bdc381b..3549e66 100644
--- a/heron/statemgrs/tests/python/configloader_unittest.py
+++ b/heron/statemgrs/tests/python/configloader_unittest.py
@@ -40,7 +40,7 @@
return configloader.load_state_manager_locations(cluster, yaml_path)
def test_load_state_manager_locations_aurora(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/vagrant/.herondata/repository/state/aurora',
@@ -49,7 +49,7 @@
}], self.load_locations('aurora'))
def test_load_state_manager_locations_local(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/local',
@@ -58,7 +58,7 @@
}], self.load_locations('local'))
def test_load_state_manager_locations_localzk(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': '127.0.0.1:2181',
'name': 'zk',
'rootpath': '/heron',
@@ -67,7 +67,7 @@
}], self.load_locations('localzk'))
def test_load_state_manager_locations_marathon(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/marathon',
@@ -76,7 +76,7 @@
}], self.load_locations('marathon'))
def test_load_state_manager_locations_mesos(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/mesos',
@@ -85,7 +85,7 @@
}], self.load_locations('mesos'))
def test_load_state_manager_locations_slurm(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/slurm',
@@ -94,7 +94,7 @@
}], self.load_locations('slurm'))
def test_load_state_manager_locations_yarn(self):
- self.assertEquals([{
+ self.assertEqual([{
'hostport': 'LOCALMODE',
'name': 'local',
'rootpath': '/user/fake/.herondata/repository/state/yarn',
diff --git a/heron/statemgrs/tests/python/statemanagerfactory_unittest.py b/heron/statemgrs/tests/python/statemanagerfactory_unittest.py
index e56c90c..dea8cfe 100644
--- a/heron/statemgrs/tests/python/statemanagerfactory_unittest.py
+++ b/heron/statemgrs/tests/python/statemanagerfactory_unittest.py
@@ -34,7 +34,7 @@
'rootpath':'/heron', 'tunnelhost':'127.0.0.1'}])
statemanagers = statemanagerfactory.get_all_zk_state_managers(conf)
# 1 state_location should result in 1 state manager
- self.assertEquals(1, len(statemanagers))
+ self.assertEqual(1, len(statemanagers))
statemanager = statemanagers[0]
# statemanager.hostportlist should contain both host port pairs
diff --git a/heron/tools/admin/src/python/main.py b/heron/tools/admin/src/python/main.py
index 57348b6..4b27601 100644
--- a/heron/tools/admin/src/python/main.py
+++ b/heron/tools/admin/src/python/main.py
@@ -51,7 +51,7 @@
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers and print help
- for choice, subparser in subparsers_action.choices.items():
+ for choice, subparser in list(subparsers_action.choices.items()):
print("Subparser '{}'".format(choice))
print(subparser.format_help())
return
diff --git a/heron/tools/admin/src/python/standalone.py b/heron/tools/admin/src/python/standalone.py
index 85ee341..6caf196 100644
--- a/heron/tools/admin/src/python/standalone.py
+++ b/heron/tools/admin/src/python/standalone.py
@@ -16,6 +16,7 @@
# under the License.
''' standalone.py '''
+from __future__ import print_function
from collections import OrderedDict
from subprocess import call
import subprocess
@@ -185,11 +186,11 @@
elif action == Action.GET:
action_type = cl_args["type"]
if action_type == Get.SERVICE_URL:
- print get_service_url(cl_args)
+ print(get_service_url(cl_args))
elif action_type == Get.HERON_UI_URL:
- print get_heron_ui_url(cl_args)
+ print(get_heron_ui_url(cl_args))
elif action_type == Get.HERON_TRACKER_URL:
- print get_heron_tracker_url(cl_args)
+ print(get_heron_tracker_url(cl_args))
else:
raise ValueError("Invalid get action %s" % action_type)
elif action == Action.INFO:
@@ -312,7 +313,7 @@
file_contents = ""
with open(src, 'r') as tf:
file_contents = tf.read()
- for key, value in replacements_dict.iteritems():
+ for key, value in replacements_dict.items():
file_contents = file_contents.replace(key, value)
if not file_contents:
@@ -372,7 +373,7 @@
info['roles'] = roles
info['urls'] = urls
- print json.dumps(info, indent=2)
+ print(json.dumps(info, indent=2))
def add_additional_args(parsers):
'''
@@ -858,13 +859,13 @@
return output[0].strip("\n")
def check_sure(cl_args, prompt):
- yes = raw_input("%s" % prompt + ' (yes/no): ')
+ yes = input("%s" % prompt + ' (yes/no): ')
if yes == "y" or yes == "yes":
return True
elif yes == "n" or yes == "no":
return False
else:
- print 'Invalid input. Please input "yes" or "no"'
+ print('Invalid input. Please input "yes" or "no"')
def get_jobs(cl_args, nomad_addr):
r = requests.get("http://%s:4646/v1/jobs" % nomad_addr)
diff --git a/heron/tools/cli/src/python/BUILD b/heron/tools/cli/src/python/BUILD
index 1f33ab3..a59ec07 100644
--- a/heron/tools/cli/src/python/BUILD
+++ b/heron/tools/cli/src/python/BUILD
@@ -14,7 +14,8 @@
"PyYAML==3.13",
"enum34==1.1.6",
"requests==2.12.3",
- "netifaces==0.10.6"
+ "netifaces==0.10.6",
+ "future==0.18.2"
],
)
diff --git a/heron/tools/cli/src/python/cli_helper.py b/heron/tools/cli/src/python/cli_helper.py
index 439621d..4b73661 100644
--- a/heron/tools/cli/src/python/cli_helper.py
+++ b/heron/tools/cli/src/python/cli_helper.py
@@ -58,7 +58,7 @@
################################################################################
def flatten_args(fargs):
temp_args = []
- for k, v in fargs.items():
+ for k, v in list(fargs.items()):
if isinstance(v, list):
temp_args.extend([(k, value) for value in v])
else:
diff --git a/heron/tools/cli/src/python/config.py b/heron/tools/cli/src/python/config.py
index d926ebd..a48a4f0 100644
--- a/heron/tools/cli/src/python/config.py
+++ b/heron/tools/cli/src/python/config.py
@@ -91,7 +91,7 @@
cluster = cl_args['cluster']
config = cliconfig.cluster_config(cluster)
if config:
- for k, v in config.items():
+ for k, v in list(config.items()):
print("%s = %s" % (str(k), str(v)))
else:
print("No config for cluster %s" % cluster)
diff --git a/heron/tools/cli/src/python/main.py b/heron/tools/cli/src/python/main.py
index 11b4695..a8257c0 100644
--- a/heron/tools/cli/src/python/main.py
+++ b/heron/tools/cli/src/python/main.py
@@ -69,7 +69,7 @@
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers and print help
- for choice, subparser in subparsers_action.choices.items():
+ for choice, subparser in list(subparsers_action.choices.items()):
print("Subparser '{}'".format(choice))
print(subparser.format_help())
return
diff --git a/heron/tools/cli/src/python/opts.py b/heron/tools/cli/src/python/opts.py
index a15cbc6..be29bf2 100644
--- a/heron/tools/cli/src/python/opts.py
+++ b/heron/tools/cli/src/python/opts.py
@@ -37,7 +37,7 @@
:return:
'''
opt_list = []
- for (key, value) in config_opts.items():
+ for (key, value) in list(config_opts.items()):
opt_list.append('%s=%s' % (key, value))
all_opts = (','.join(opt_list)).replace(' ', '%%%%')
diff --git a/heron/tools/cli/src/python/submit.py b/heron/tools/cli/src/python/submit.py
index f147619..57df7fb 100644
--- a/heron/tools/cli/src/python/submit.py
+++ b/heron/tools/cli/src/python/submit.py
@@ -19,13 +19,16 @@
# under the License.
''' submit.py '''
+from future.standard_library import install_aliases
+install_aliases()
+
import glob
import logging
import os
import tempfile
import requests
import subprocess
-import urlparse
+from urllib.parse import urlparse
from heron.common.src.python.utils.log import Log
from heron.proto import topology_pb2
@@ -418,7 +421,7 @@
# get the topology file name
topology_file = cl_args['topology-file-name']
- if urlparse.urlparse(topology_file).scheme:
+ if urlparse(topology_file).scheme:
cl_args['topology-file-name'] = download(topology_file, cl_args['cluster'])
topology_file = cl_args['topology-file-name']
Log.debug("download uri to local file: %s", topology_file)
diff --git a/heron/tools/cli/src/python/version.py b/heron/tools/cli/src/python/version.py
index 1371e48..260f2ae 100644
--- a/heron/tools/cli/src/python/version.py
+++ b/heron/tools/cli/src/python/version.py
@@ -100,7 +100,7 @@
r = service_method(service_apiurl)
if r.status_code != requests.codes.ok:
Log.error(r.json().get('message', "Unknown error from API server %d" % r.status_code))
- sorted_items = sorted(r.json().items(), key=lambda tup: tup[0])
+ sorted_items = sorted(list(r.json().items()), key=lambda tup: tup[0])
for key, value in sorted_items:
print("%s : %s" % (key, value))
except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as err:
diff --git a/heron/tools/common/src/python/access/__init__.py b/heron/tools/common/src/python/access/__init__.py
index 8e6a8a6..9639b9f 100644
--- a/heron/tools/common/src/python/access/__init__.py
+++ b/heron/tools/common/src/python/access/__init__.py
@@ -17,5 +17,5 @@
''' access module '''
__all__ = ['access']
-from heron_api import *
-from query import *
+from .heron_api import *
+from .query import *
diff --git a/heron/tools/common/src/python/access/heron_api.py b/heron/tools/common/src/python/access/heron_api.py
index ebf908d..c5a5203 100644
--- a/heron/tools/common/src/python/access/heron_api.py
+++ b/heron/tools/common/src/python/access/heron_api.py
@@ -25,8 +25,8 @@
import tornado.httpclient
import tornado.gen
from tornado.options import options
-from fetch import fetch_url_as_json
-from query import QueryHandler
+from .fetch import fetch_url_as_json
+from .query import QueryHandler
# pylint: disable=bad-whitespace
CLUSTER_URL_FMT = "%s/clusters"
@@ -243,7 +243,7 @@
request_url = tornado.httputil.url_concat(
create_url(LOGICALPLAN_URL_FMT), params)
lplan = yield fetch_url_as_json(request_url)
- comps = lplan['spouts'].keys() + lplan['bolts'].keys()
+ comps = list(lplan['spouts'].keys()) + list(lplan['bolts'].keys())
raise tornado.gen.Return(comps)
################################################################################
@@ -263,7 +263,7 @@
request_url = tornado.httputil.url_concat(
create_url(PHYSICALPLAN_URL_FMT), params)
pplan = yield fetch_url_as_json(request_url)
- instances = pplan['instances'].keys()
+ instances = list(pplan['instances'].keys())
raise tornado.gen.Return(instances)
@@ -404,7 +404,7 @@
all_instances = instances if isinstance(instances, list) else [instances]
# append each metric to the url
- for _, metric_name in metrics.items():
+ for _, metric_name in list(metrics.items()):
request_url = tornado.httputil.url_concat(request_url, dict(metricname=metric_name[0]))
# append each instance to the url
@@ -856,7 +856,7 @@
timelines.extend(result["timeline"])
result = self.get_metric_response(timerange, timelines, is_max)
else:
- data = self.compute_max(res.values())
+ data = self.compute_max(list(res.values()))
result = self.get_metric_response(timerange, data, is_max)
raise tornado.gen.Return(result)
@@ -871,10 +871,10 @@
# key set is empty. These components are filtered out first.
filtered_ts = [ts for ts in multi_ts if len(ts["timeline"][0]["data"]) > 0]
if len(filtered_ts) > 0 and len(filtered_ts[0]["timeline"]) > 0:
- keys = filtered_ts[0]["timeline"][0]["data"].keys()
+ keys = list(filtered_ts[0]["timeline"][0]["data"].keys())
timelines = ([res["timeline"][0]["data"][key] for key in keys] for res in filtered_ts)
values = (max(v) for v in zip(*timelines))
- return dict(zip(keys, values))
+ return dict(list(zip(keys, values)))
return {}
# pylint: disable=no-self-use
diff --git a/heron/tools/common/src/python/access/tracker_access.py b/heron/tools/common/src/python/access/tracker_access.py
index 4e5f4b0..8b49b97 100644
--- a/heron/tools/common/src/python/access/tracker_access.py
+++ b/heron/tools/common/src/python/access/tracker_access.py
@@ -48,7 +48,7 @@
def queries_map():
"""map from query parameter to query name"""
qs = _all_metric_queries()
- return dict(zip(qs[0], qs[1]) + zip(qs[2], qs[3]))
+ return dict(list(zip(qs[0], qs[1])) + list(zip(qs[2], qs[3])))
def get_clusters():
diff --git a/heron/tools/common/src/python/utils/config.py b/heron/tools/common/src/python/utils/config.py
index e08c403..42d395f 100644
--- a/heron/tools/common/src/python/utils/config.py
+++ b/heron/tools/common/src/python/utils/config.py
@@ -101,7 +101,7 @@
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers
- for choice, subparser in subparsers_action.choices.items():
+ for choice, subparser in list(subparsers_action.choices.items()):
if choice == command:
return subparser
return None
@@ -467,7 +467,7 @@
with open(release_file) as release_info:
release_map = yaml.load(release_info)
- release_items = sorted(release_map.items(), key=lambda tup: tup[0])
+ release_items = sorted(list(release_map.items()), key=lambda tup: tup[0])
for key, value in release_items:
print("%s : %s" % (key, value))
diff --git a/heron/tools/explorer/src/python/logicalplan.py b/heron/tools/explorer/src/python/logicalplan.py
index c7e4085..f6eb245 100644
--- a/heron/tools/explorer/src/python/logicalplan.py
+++ b/heron/tools/explorer/src/python/logicalplan.py
@@ -62,9 +62,9 @@
def to_table(components, topo_info):
""" normalize raw logical plan info to table """
inputs, outputs = defaultdict(list), defaultdict(list)
- for ctype, component in components.items():
+ for ctype, component in list(components.items()):
if ctype == 'bolts':
- for component_name, component_info in component.items():
+ for component_name, component_info in list(component.items()):
for input_stream in component_info['inputs']:
input_name = input_stream['component_name']
inputs[component_name].append(input_name)
@@ -72,11 +72,11 @@
info = []
spouts_instance = topo_info['physical_plan']['spouts']
bolts_instance = topo_info['physical_plan']['bolts']
- for ctype, component in components.items():
+ for ctype, component in list(components.items()):
# stages is an int so keep going
if ctype == "stages":
continue
- for component_name, component_info in component.items():
+ for component_name, component_info in list(component.items()):
row = [ctype[:-1], component_name]
if ctype == 'spouts':
row.append(len(spouts_instance[component_name]))
diff --git a/heron/tools/explorer/src/python/physicalplan.py b/heron/tools/explorer/src/python/physicalplan.py
index 72c5dfa..50ad7c3 100644
--- a/heron/tools/explorer/src/python/physicalplan.py
+++ b/heron/tools/explorer/src/python/physicalplan.py
@@ -76,7 +76,7 @@
""" normalize raw metrics API result to table """
all_queries = tracker_access.metric_queries()
m = tracker_access.queries_map()
- names = metrics.values()[0].keys()
+ names = list(metrics.values())[0].keys()
stats = []
for n in names:
info = [n]
@@ -86,7 +86,7 @@
except KeyError:
pass
stats.append(info)
- header = ['container id'] + [m[k] for k in all_queries if k in metrics.keys()]
+ header = ['container id'] + [m[k] for k in all_queries if k in list(metrics.keys())]
return stats, header
@@ -97,8 +97,8 @@
topology = cl_args['topology-name']
try:
result = tracker_access.get_topology_info(cluster, env, topology, role)
- spouts = result['physical_plan']['spouts'].keys()
- bolts = result['physical_plan']['bolts'].keys()
+ spouts = list(result['physical_plan']['spouts'].keys())
+ bolts = list(result['physical_plan']['bolts'].keys())
components = spouts + bolts
cname = cl_args['component']
if cname:
@@ -134,7 +134,7 @@
topology = cl_args['topology-name']
try:
result = tracker_access.get_topology_info(cluster, env, topology, role)
- bolts = result['physical_plan']['bolts'].keys()
+ bolts = list(result['physical_plan']['bolts'].keys())
bolt_name = cl_args['bolt']
if bolt_name:
if bolt_name in bolts:
@@ -174,11 +174,11 @@
return False
containers = result['physical_plan']['stmgrs']
all_bolts, all_spouts = set(), set()
- for _, bolts in result['physical_plan']['bolts'].items():
+ for _, bolts in list(result['physical_plan']['bolts'].items()):
all_bolts = all_bolts | set(bolts)
- for _, spouts in result['physical_plan']['spouts'].items():
+ for _, spouts in list(result['physical_plan']['spouts'].items()):
all_spouts = all_spouts | set(spouts)
- stmgrs = containers.keys()
+ stmgrs = list(containers.keys())
stmgrs.sort()
if container_id is not None:
try:
diff --git a/heron/tools/explorer/src/python/topologies.py b/heron/tools/explorer/src/python/topologies.py
index 2328d7a..dab24a7 100644
--- a/heron/tools/explorer/src/python/topologies.py
+++ b/heron/tools/explorer/src/python/topologies.py
@@ -44,8 +44,8 @@
''' normalize raw result to table '''
max_count = 20
table, count = [], 0
- for role, envs_topos in result.items():
- for env, topos in envs_topos.items():
+ for role, envs_topos in list(result.items()):
+ for env, topos in list(envs_topos.items()):
for topo in topos:
count += 1
if count > max_count:
diff --git a/heron/tools/tracker/src/python/BUILD b/heron/tools/tracker/src/python/BUILD
index e3744ab..b2e40e8 100644
--- a/heron/tools/tracker/src/python/BUILD
+++ b/heron/tools/tracker/src/python/BUILD
@@ -16,6 +16,7 @@
reqs = [
"protobuf==3.6.1",
"tornado==4.0.2",
+ "future==0.18.2"
],
)
diff --git a/heron/tools/tracker/src/python/config.py b/heron/tools/tracker/src/python/config.py
index 08887b8..089b03e 100644
--- a/heron/tools/tracker/src/python/config.py
+++ b/heron/tools/tracker/src/python/config.py
@@ -71,7 +71,7 @@
"${USER}": "user",
}
dummy_formatted_url = url_format
- for key, value in valid_parameters.items():
+ for key, value in list(valid_parameters.items()):
dummy_formatted_url = dummy_formatted_url.replace(key, value)
# All $ signs must have been replaced
@@ -99,7 +99,7 @@
formatted_url = formatter
- for key, value in common_parameters.items():
+ for key, value in list(common_parameters.items()):
formatted_url = formatted_url.replace(key, value)
return formatted_url
diff --git a/heron/tools/tracker/src/python/handlers/__init__.py b/heron/tools/tracker/src/python/handlers/__init__.py
index 497a01d..5305246 100644
--- a/heron/tools/tracker/src/python/handlers/__init__.py
+++ b/heron/tools/tracker/src/python/handlers/__init__.py
@@ -15,31 +15,31 @@
# specific language governing permissions and limitations
# under the License.
''' __init__ '''
-from basehandler import BaseHandler
-from clustershandler import ClustersHandler
-from containerfilehandler import ContainerFileDataHandler
-from containerfilehandler import ContainerFileDownloadHandler
-from containerfilehandler import ContainerFileStatsHandler
-from defaulthandler import DefaultHandler
-from exceptionhandler import ExceptionHandler
-from exceptionsummaryhandler import ExceptionSummaryHandler
-from executionstatehandler import ExecutionStateHandler
-from jmaphandler import JmapHandler
-from jstackhandler import JstackHandler
-from logicalplanhandler import LogicalPlanHandler
-from machineshandler import MachinesHandler
-from mainhandler import MainHandler
-from memoryhistogramhandler import MemoryHistogramHandler
-from metadatahandler import MetaDataHandler
-from metricshandler import MetricsHandler
-from metricsqueryhandler import MetricsQueryHandler
-from metricstimelinehandler import MetricsTimelineHandler
-from physicalplanhandler import PhysicalPlanHandler
-from packingplanhandler import PackingPlanHandler
-from pidhandler import PidHandler
-from runtimestatehandler import RuntimeStateHandler
-from schedulerlocationhandler import SchedulerLocationHandler
-from stateshandler import StatesHandler
-from topologieshandler import TopologiesHandler
-from topologyconfighandler import TopologyConfigHandler
-from topologyhandler import TopologyHandler
+from .basehandler import BaseHandler
+from .clustershandler import ClustersHandler
+from .containerfilehandler import ContainerFileDataHandler
+from .containerfilehandler import ContainerFileDownloadHandler
+from .containerfilehandler import ContainerFileStatsHandler
+from .defaulthandler import DefaultHandler
+from .exceptionhandler import ExceptionHandler
+from .exceptionsummaryhandler import ExceptionSummaryHandler
+from .executionstatehandler import ExecutionStateHandler
+from .jmaphandler import JmapHandler
+from .jstackhandler import JstackHandler
+from .logicalplanhandler import LogicalPlanHandler
+from .machineshandler import MachinesHandler
+from .mainhandler import MainHandler
+from .memoryhistogramhandler import MemoryHistogramHandler
+from .metadatahandler import MetaDataHandler
+from .metricshandler import MetricsHandler
+from .metricsqueryhandler import MetricsQueryHandler
+from .metricstimelinehandler import MetricsTimelineHandler
+from .physicalplanhandler import PhysicalPlanHandler
+from .packingplanhandler import PackingPlanHandler
+from .pidhandler import PidHandler
+from .runtimestatehandler import RuntimeStateHandler
+from .schedulerlocationhandler import SchedulerLocationHandler
+from .stateshandler import StatesHandler
+from .topologieshandler import TopologiesHandler
+from .topologyconfighandler import TopologyConfigHandler
+from .topologyhandler import TopologyHandler
diff --git a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py b/heron/tools/tracker/src/python/handlers/logicalplanhandler.py
index 186cf70..2da1cd2 100644
--- a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py
+++ b/heron/tools/tracker/src/python/handlers/logicalplanhandler.py
@@ -57,7 +57,7 @@
# format the logical plan as required by the web (because of Ambrose)
# first, spouts followed by bolts
spouts_map = dict()
- for name, value in lplan['spouts'].items():
+ for name, value in list(lplan['spouts'].items()):
spouts_map[name] = dict(
config=value.get("config", dict()),
outputs=value["outputs"],
@@ -67,7 +67,7 @@
)
bolts_map = dict()
- for name, value in lplan['bolts'].items():
+ for name, value in list(lplan['bolts'].items()):
bolts_map[name] = dict(
config=value.get("config", dict()),
inputComponents=[i['component_name'] for i in value['inputs']],
diff --git a/heron/tools/tracker/src/python/handlers/runtimestatehandler.py b/heron/tools/tracker/src/python/handlers/runtimestatehandler.py
index 6148605..b776517 100644
--- a/heron/tools/tracker/src/python/handlers/runtimestatehandler.py
+++ b/heron/tools/tracker/src/python/handlers/runtimestatehandler.py
@@ -116,7 +116,7 @@
topology = self.tracker.getTopologyByClusterRoleEnvironAndName(
cluster, role, environ, topology_name)
reg_summary = yield tornado.gen.Task(self.getStmgrsRegSummary, topology.tmaster)
- for stmgr, reg in reg_summary.items():
+ for stmgr, reg in list(reg_summary.items()):
runtime_state["stmgrs"].setdefault(stmgr, {})["is_registered"] = reg
self.write_success_response(runtime_state)
except Exception as e:
diff --git a/heron/tools/tracker/src/python/javaobj.py b/heron/tools/tracker/src/python/javaobj.py
index 65e9c9a..d5978ba 100644
--- a/heron/tools/tracker/src/python/javaobj.py
+++ b/heron/tools/tracker/src/python/javaobj.py
@@ -28,8 +28,9 @@
See: http://download.oracle.com/javase/6/docs/platform/serialization/spec/protocol.html
"""
-import StringIO
import struct
+import six
+
from heron.common.src.python.utils.log import Log
def log_debug(message, ident=0):
@@ -53,12 +54,12 @@
# pylint: disable=undefined-variable
-def loads(string):
+def loads(value):
"""
Deserializes Java objects and primitive data serialized by ObjectOutputStream
from a string.
"""
- f = StringIO.StringIO(string)
+ f = six.StringIO(value)
marshaller = JavaObjectUnmarshaller(f)
marshaller.add_transformer(DefaultObjectTransformer())
return marshaller.readObject()
@@ -492,7 +493,7 @@
def _create_hexdump(self, src, length=16):
FILTER = ''.join([(len(repr(chr(x))) == 3) and chr(x) or '.' for x in range(256)])
result = []
- for i in xrange(0, len(src), length):
+ for i in range(0, len(src), length):
s = src[i:i+length]
hexa = ' '.join(["%02X"%ord(x) for x in s])
printable = s.translate(FILTER)
@@ -558,7 +559,7 @@
# pylint: disable=attribute-defined-outside-init
def dump(self, obj):
self.object_obj = obj
- self.object_stream = StringIO.StringIO()
+ self.object_stream = six.StringIO()
self._writeStreamHeader()
self.writeObject(obj)
return self.object_stream.getvalue()
diff --git a/heron/tools/tracker/src/python/main.py b/heron/tools/tracker/src/python/main.py
index d9f60e9..581894d 100644
--- a/heron/tools/tracker/src/python/main.py
+++ b/heron/tools/tracker/src/python/main.py
@@ -20,6 +20,7 @@
''' main.py '''
from __future__ import print_function
+
import argparse
import os
import signal
@@ -109,7 +110,7 @@
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers and print help
- for choice, subparser in subparsers_action.choices.items():
+ for choice, subparser in list(subparsers_action.choices.items()):
print("Subparser '{}'".format(choice))
print(subparser.format_help())
diff --git a/heron/tools/tracker/src/python/query_operators.py b/heron/tools/tracker/src/python/query_operators.py
index d9641cb..dd348a7 100644
--- a/heron/tools/tracker/src/python/query_operators.py
+++ b/heron/tools/tracker/src/python/query_operators.py
@@ -54,7 +54,7 @@
def floorTimestamps(self, start, end, timeline):
""" floor timestamp """
ret = {}
- for timestamp, value in timeline.items():
+ for timestamp, value in list(timeline.items()):
ts = timestamp / 60 * 60
if start <= ts <= end:
ret[ts] = value
@@ -143,9 +143,9 @@
}
timelines = metrics["timeline"][self.metricName]
allMetrics = []
- for instance, timeline in timelines.items():
+ for instance, timeline in list(timelines.items()):
toBeDeletedKeys = []
- for key, value in timeline.items():
+ for key, value in list(timeline.items()):
floatValue = float(value)
# Check if the value is really float or not.
# In python, float("nan") returns "nan" which is actually a float value,
@@ -207,9 +207,9 @@
def execute(self, tracker, tmaster, start, end):
# Initialize the metric to be returned with sum of all the constants.
retMetrics = Metrics(None, None, None, start, end, {})
- constants = filter(lambda ts: isinstance(ts, float), self.timeSeriesList)
+ constants = [ts for ts in self.timeSeriesList if isinstance(ts, float)]
retMetrics.setDefault(sum(constants), start, end)
- leftOverTimeSeries = filter(lambda ts: not isinstance(ts, float), self.timeSeriesList)
+ leftOverTimeSeries = [ts for ts in self.timeSeriesList if not isinstance(ts, float)]
futureMetrics = []
for timeseries in leftOverTimeSeries:
@@ -225,7 +225,7 @@
# Aggregate all of the them
for metric in allMetrics:
- for timestamp, value in metric.timeline.items():
+ for timestamp, value in list(metric.timeline.items()):
if timestamp in retMetrics.timeline:
retMetrics.timeline[timestamp] += value
raise tornado.gen.Return([retMetrics])
@@ -249,10 +249,10 @@
def execute(self, tracker, tmaster, start, end):
# Initialize the metric to be returned with max of all the constants.
retMetrics = Metrics(None, None, None, start, end, {})
- constants = filter(lambda ts: isinstance(ts, float), self.timeSeriesList)
+ constants = [ts for ts in self.timeSeriesList if isinstance(ts, float)]
if constants:
retMetrics.setDefault(max(constants), start, end)
- leftOverTimeSeries = filter(lambda ts: not isinstance(ts, float), self.timeSeriesList)
+ leftOverTimeSeries = [ts for ts in self.timeSeriesList if not isinstance(ts, float)]
futureMetrics = []
for timeseries in leftOverTimeSeries:
@@ -269,7 +269,7 @@
# Aggregate all of the them
for metric in allMetrics:
- for timestamp, value in metric.timeline.items():
+ for timestamp, value in list(metric.timeline.items()):
if start <= timestamp <= end:
if timestamp not in retMetrics.timeline:
retMetrics.timeline[timestamp] = value
@@ -302,7 +302,7 @@
@tornado.gen.coroutine
def execute(self, tracker, tmaster, start, end):
- leftOverTimeSeries = filter(lambda ts: not isinstance(ts, float), self.timeSeriesList)
+ leftOverTimeSeries = [ts for ts in self.timeSeriesList if not isinstance(ts, float)]
futureMetrics = []
for timeseries in leftOverTimeSeries:
@@ -323,14 +323,14 @@
# Aggregate all of the them
for metric in allMetrics:
- for timestamp, value in metric.timeline.items():
+ for timestamp, value in list(metric.timeline.items()):
if start <= timestamp <= end:
if timestamp not in timeline:
timeline[timestamp] = []
timeline[timestamp].append(value)
retTimeline = {}
- for timestamp, values in timeline.items():
+ for timestamp, values in list(timeline.items()):
if not values:
continue
index = int(self.quantile * 1.0 * (len(values) - 1) / 100.0)
@@ -429,7 +429,7 @@
if key not in metrics2:
continue
met = Metrics(None, None, key, start, end, {})
- for timestamp in metrics[key].timeline.keys():
+ for timestamp in list(metrics[key].timeline.keys()):
if timestamp not in metrics2[key].timeline or metrics2[key].timeline[timestamp] == 0:
metrics[key].timeline.pop(timestamp)
else:
@@ -440,11 +440,11 @@
# If first is univariate
elif len(metrics) == 1 and "" in metrics:
allMetrics = []
- for key, metric in metrics2.items():
+ for key, metric in list(metrics2.items()):
# Initialize with first metrics timeline, but second metric's instance
# because that is multivariate
met = Metrics(None, None, metric.instance, start, end, dict(metrics[""].timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metric.timeline or metric.timeline[timestamp] == 0:
met.timeline.pop(timestamp)
else:
@@ -454,10 +454,10 @@
# If second is univariate
else:
allMetrics = []
- for key, metric in metrics.items():
+ for key, metric in list(metrics.items()):
# Initialize with first metrics timeline and its instance
met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metrics2[""].timeline or metrics2[""].timeline[timestamp] == 0:
met.timeline.pop(timestamp)
else:
@@ -554,7 +554,7 @@
if key not in metrics2:
continue
met = Metrics(None, None, key, start, end, {})
- for timestamp in metrics[key].timeline.keys():
+ for timestamp in list(metrics[key].timeline.keys()):
if timestamp not in metrics2[key].timeline:
metrics[key].timeline.pop(timestamp)
else:
@@ -565,11 +565,11 @@
# If first is univariate
elif len(metrics) == 1 and "" in metrics:
allMetrics = []
- for key, metric in metrics2.items():
+ for key, metric in list(metrics2.items()):
# Initialize with first metrics timeline, but second metric's instance
# because that is multivariate
met = Metrics(None, None, metric.instance, start, end, dict(metrics[""].timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metric.timeline:
met.timeline.pop(timestamp)
else:
@@ -579,10 +579,10 @@
# If second is univariate
else:
allMetrics = []
- for key, metric in metrics.items():
+ for key, metric in list(metrics.items()):
# Initialize with first metrics timeline and its instance
met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metrics2[""].timeline:
met.timeline.pop(timestamp)
else:
@@ -677,7 +677,7 @@
if key not in metrics2:
continue
met = Metrics(None, None, key, start, end, {})
- for timestamp in metrics[key].timeline.keys():
+ for timestamp in list(metrics[key].timeline.keys()):
if timestamp not in metrics2[key].timeline:
metrics[key].timeline.pop(timestamp)
else:
@@ -688,11 +688,11 @@
# If first is univariate
elif len(metrics) == 1 and "" in metrics:
allMetrics = []
- for key, metric in metrics2.items():
+ for key, metric in list(metrics2.items()):
# Initialize with first metrics timeline, but second metric's instance
# because that is multivariate
met = Metrics(None, None, metric.instance, start, end, dict(metrics[""].timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metric.timeline:
met.timeline.pop(timestamp)
else:
@@ -702,10 +702,10 @@
# If second is univariate
else:
allMetrics = []
- for key, metric in metrics.items():
+ for key, metric in list(metrics.items()):
# Initialize with first metrics timeline and its instance
met = Metrics(None, None, metric.instance, start, end, dict(metric.timeline))
- for timestamp in met.timeline.keys():
+ for timestamp in list(met.timeline.keys()):
if timestamp not in metrics2[""].timeline:
met.timeline.pop(timestamp)
else:
diff --git a/heron/tools/tracker/src/python/topology.py b/heron/tools/tracker/src/python/topology.py
index 5198492..2deef9e 100644
--- a/heron/tools/tracker/src/python/topology.py
+++ b/heron/tools/tracker/src/python/topology.py
@@ -108,7 +108,7 @@
unregister the corresponding watch.
"""
to_remove = []
- for uid, callback in self.watches.items():
+ for uid, callback in list(self.watches.items()):
try:
callback(self)
except Exception as e:
@@ -202,7 +202,7 @@
"""
Returns a list of names of all the spouts
"""
- return map(lambda component: component.comp.name, self.spouts())
+ return [component.comp.name for component in self.spouts()]
def bolts(self):
"""
@@ -216,7 +216,7 @@
"""
Returns a list of names of all the bolts
"""
- return map(lambda component: component.comp.name, self.bolts())
+ return [component.comp.name for component in self.bolts()]
def get_machines(self):
"""
@@ -225,7 +225,7 @@
"""
if self.physical_plan:
stmgrs = list(self.physical_plan.stmgrs)
- return map(lambda s: s.host_name, stmgrs)
+ return [s.host_name for s in stmgrs]
return []
def get_status(self):
diff --git a/heron/tools/tracker/src/python/tracker.py b/heron/tools/tracker/src/python/tracker.py
index 15df468..e2d16fc 100644
--- a/heron/tools/tracker/src/python/tracker.py
+++ b/heron/tools/tracker/src/python/tracker.py
@@ -135,7 +135,7 @@
Log.info("State watch triggered for topologies.")
Log.debug("Topologies: " + str(topologies))
existingTopologies = self.getTopologiesForStateLocation(state_manager.name)
- existingTopNames = map(lambda t: t.name, existingTopologies)
+ existingTopNames = [t.name for t in existingTopologies]
Log.debug("Existing topologies: " + str(existingTopNames))
for name in existingTopNames:
if name not in topologies:
@@ -164,10 +164,10 @@
an optional role.
Raises exception if topology is not found, or more than one are found.
"""
- topologies = list(filter(lambda t: t.name == topologyName
- and t.cluster == cluster
- and (not role or t.execution_state.role == role)
- and t.environ == environ, self.topologies))
+ topologies = list([t for t in self.topologies if t.name == topologyName
+ and t.cluster == cluster
+ and (not role or t.execution_state.role == role)
+ and t.environ == environ])
if not topologies or len(topologies) > 1:
if role is not None:
raise Exception("Topology not found for {0}, {1}, {2}, {3}".format(
@@ -183,7 +183,7 @@
"""
Returns all the topologies for a given state manager.
"""
- return filter(lambda t: t.state_manager_name == name, self.topologies)
+ return [t for t in self.topologies if t.state_manager_name == name]
def addNewTopology(self, state_manager, topologyName):
"""
@@ -672,7 +672,7 @@
Raises exception if no such topology is found.
"""
# Iterate over the values to filter the desired topology.
- for (topology_name, _), topologyInfo in self.topologyInfos.items():
+ for (topology_name, _), topologyInfo in list(self.topologyInfos.items()):
executionState = topologyInfo["execution_state"]
if (topologyName == topology_name and
cluster == executionState["cluster"] and
diff --git a/heron/tools/tracker/tests/python/tracker_unittest.py b/heron/tools/tracker/tests/python/tracker_unittest.py
index 136f5d6..fcb4810 100644
--- a/heron/tools/tracker/tests/python/tracker_unittest.py
+++ b/heron/tools/tracker/tests/python/tracker_unittest.py
@@ -206,12 +206,12 @@
self.tracker.addNewTopology(mock_state_manager_1, 'top_name1')
self.assertItemsEqual(
['top_name1'],
- map(lambda t: t.name, self.tracker.topologies))
+ [t.name for t in self.tracker.topologies])
self.tracker.addNewTopology(mock_state_manager_1, 'top_name2')
self.assertItemsEqual(
['top_name1', 'top_name2'],
- map(lambda t: t.name, self.tracker.topologies))
+ [t.name for t in self.tracker.topologies])
self.assertEqual(2, mock_state_manager_1.get_pplan.call_count)
self.assertEqual(2, mock_state_manager_1.get_execution_state.call_count)
@@ -261,11 +261,11 @@
self.assertEqual(packing_plan['id'], 'ExclamationTopology')
self.assertEqual(packing_plan['container_plans'][0]['id'], 1)
self.assertEqual(packing_plan['container_plans'][0]['required_resources'],
- {'disk': 2048L, 'ram': 1024L, 'cpu': 1.0})
+ {'disk': 2048, 'ram': 1024, 'cpu': 1.0})
self.assertEqual(packing_plan['container_plans'][0]['instances'][0],
{
'component_index': 1,
- 'component_name': u'word',
- 'instance_resources': {'cpu': 1.0, 'disk': 2048L, 'ram': 1024L},
+ 'component_name': 'word',
+ 'instance_resources': {'cpu': 1.0, 'disk': 2048, 'ram': 1024},
'task_id': 1
})
diff --git a/heron/tools/ui/src/python/args.py b/heron/tools/ui/src/python/args.py
index b4bcffb..14f996d 100644
--- a/heron/tools/ui/src/python/args.py
+++ b/heron/tools/ui/src/python/args.py
@@ -39,7 +39,7 @@
# but better save than sorry
for subparsers_action in subparsers_actions:
# get all subparsers and print help
- for choice, subparser in subparsers_action.choices.items():
+ for choice, subparser in list(subparsers_action.choices.items()):
print("Subparser '{}'".format(choice))
print(subparser.format_help())
diff --git a/heron/tools/ui/src/python/handlers/api/topology.py b/heron/tools/ui/src/python/handlers/api/topology.py
index 3df74d5..f7c9100 100644
--- a/heron/tools/ui/src/python/handlers/api/topology.py
+++ b/heron/tools/ui/src/python/handlers/api/topology.py
@@ -53,8 +53,8 @@
if not 'spouts' in lplan or not 'bolts' in lplan:
self.write(dict())
return
- comp_names = lplan['spouts'].keys()
- comp_names.extend(lplan['bolts'].keys())
+ comp_names = list(lplan['spouts'].keys())
+ comp_names.extend(list(lplan['bolts'].keys()))
else:
comp_names = [comp_name]
exception_infos = dict()
@@ -64,7 +64,7 @@
# Combine exceptions from multiple component
aggregate_exceptions = dict()
- for comp_name, exception_logs in exception_infos.items():
+ for comp_name, exception_logs in list(exception_infos.items()):
for exception_log in exception_logs:
class_name = exception_log['class_name']
if class_name != '':
@@ -96,11 +96,11 @@
result = dict()
# now convert some of the fields to be displayable
- for cluster, cluster_value in topologies.items():
+ for cluster, cluster_value in list(topologies.items()):
result[cluster] = dict()
- for environ, environ_value in cluster_value.items():
+ for environ, environ_value in list(cluster_value.items()):
result[cluster][environ] = dict()
- for topology, topology_value in environ_value.items():
+ for topology, topology_value in list(environ_value.items()):
if "jobname" not in topology_value or topology_value["jobname"] is None:
continue
diff --git a/heron/tools/ui/src/python/handlers/mainhandler.py b/heron/tools/ui/src/python/handlers/mainhandler.py
index fe5b7a3..a3d82f2 100644
--- a/heron/tools/ui/src/python/handlers/mainhandler.py
+++ b/heron/tools/ui/src/python/handlers/mainhandler.py
@@ -29,4 +29,4 @@
'''
:return:
'''
- self.redirect(u"/topologies")
+ self.redirect("/topologies")
diff --git a/heron/tools/ui/src/python/handlers/ranges.py b/heron/tools/ui/src/python/handlers/ranges.py
index 20ddc82..be1f5ec 100644
--- a/heron/tools/ui/src/python/handlers/ranges.py
+++ b/heron/tools/ui/src/python/handlers/ranges.py
@@ -49,7 +49,7 @@
# form the new
time_slots = dict()
- for key, value in ranges.items():
+ for key, value in list(ranges.items()):
time_slots[key] = (now - value[0], now - value[1], value[2])
return (now, time_slots)
diff --git a/heron/tools/ui/src/python/handlers/topology.py b/heron/tools/ui/src/python/handlers/topology.py
index b08e13d..51985fe 100644
--- a/heron/tools/ui/src/python/handlers/topology.py
+++ b/heron/tools/ui/src/python/handlers/topology.py
@@ -25,8 +25,8 @@
import tornado.web
import tornado.gen
-import base
-import common
+from . import base
+from . import common
import heron.tools.common.src.python.access as access
import heron.common.src.python.utils.log as log
diff --git a/heron/tools/ui/src/python/main.py b/heron/tools/ui/src/python/main.py
index cb8b4b7..ee96365 100644
--- a/heron/tools/ui/src/python/main.py
+++ b/heron/tools/ui/src/python/main.py
@@ -187,7 +187,7 @@
# stop Tornado IO loop
def signal_handler(signum, frame):
# start a new line after ^C character because this looks nice
- print('\n', end='')
+ print('\n')
Log.debug('SIGINT received. Stopping UI')
tornado.ioloop.IOLoop.instance().stop()
diff --git a/heronpy/api/cloudpickle.py b/heronpy/api/cloudpickle.py
index f49b7e1..eb3bfef 100644
--- a/heronpy/api/cloudpickle.py
+++ b/heronpy/api/cloudpickle.py
@@ -79,7 +79,7 @@
_BUILTIN_TYPE_NAMES = {}
-for k1, v1 in types.__dict__.items():
+for k1, v1 in list(types.__dict__.items()):
if type(v1) is type: # pylint: disable=unidiomatic-typecheck
_BUILTIN_TYPE_NAMES[v1] = k1
@@ -96,7 +96,7 @@
"""
code = getattr(code, 'co_code', b'')
if not PY3:
- code = map(ord, code)
+ code = list(map(ord, code))
n = len(code)
i = 0
@@ -169,7 +169,7 @@
dispatch[types.GeneratorType] = save_unsupported
# itertools objects do not pickle!
- for v in itertools.__dict__.values():
+ for v in list(itertools.__dict__.values()):
if type(v) is type: # pylint: disable=unidiomatic-typecheck
dispatch[v] = save_unsupported
@@ -408,7 +408,7 @@
d.pop('__doc__', None)
# handle property and staticmethod
dd = {}
- for k, v in d.items():
+ for k, v in list(d.items()):
if isinstance(v, property):
k = ('property', k)
v = (v.fget, v.fset, v.fdel, v.__doc__)
@@ -708,7 +708,7 @@
# restores function attributes
def _restore_attr(obj, attr):
- for key, val in attr.items():
+ for key, val in list(attr.items()):
setattr(obj, key, val)
return obj
@@ -789,7 +789,7 @@
"""
Loads additional properties into class `cls`.
"""
- for k, v in d.items():
+ for k, v in list(d.items()):
if isinstance(k, tuple):
typ, k = k
if typ == 'property':
diff --git a/heronpy/api/component/component_spec.py b/heronpy/api/component/component_spec.py
index 7e23772..bf134d7 100644
--- a/heronpy/api/component/component_spec.py
+++ b/heronpy/api/component/component_spec.py
@@ -115,7 +115,7 @@
# iterate through self.custom_config
if self.custom_config is not None:
sanitized = self._sanitize_config(self.custom_config)
- for key, value in sanitized.items():
+ for key, value in list(sanitized.items()):
if isinstance(value, str):
kvs = proto_config.kvs.add()
kvs.key = key
@@ -149,7 +149,7 @@
raise TypeError("Component-specific configuration must be given as a dict type, given: %s"
% str(type(custom_config)))
sanitized = {}
- for key, value in custom_config.items():
+ for key, value in list(custom_config.items()):
if not isinstance(key, str):
raise TypeError("Key for component-specific configuration must be string, given: %s:%s"
% (str(type(key)), str(key)))
@@ -170,7 +170,7 @@
# sanitize inputs and get a map <GlobalStreamId -> Grouping>
input_dict = self._sanitize_inputs()
- for global_streamid, gtype in input_dict.items():
+ for global_streamid, gtype in list(input_dict.items()):
in_stream = bolt.inputs.add()
in_stream.stream.CopyFrom(self._get_stream_id(global_streamid.component_id,
global_streamid.stream_id))
@@ -196,7 +196,7 @@
if isinstance(self.inputs, dict):
# inputs are dictionary, must be either <HeronComponentSpec -> Grouping> or
# <GlobalStreamId -> Grouping>
- for key, grouping in self.inputs.items():
+ for key, grouping in list(self.inputs.items()):
if not Grouping.is_grouping_sane(grouping):
raise ValueError('A given grouping is not supported')
if isinstance(key, HeronComponentSpec):
@@ -239,7 +239,7 @@
# sanitize outputs and get a map <stream_id -> out fields>
output_map = self._sanitize_outputs()
- for stream_id, out_fields in output_map.items():
+ for stream_id, out_fields in list(output_map.items()):
out_stream = spbl.outputs.add()
out_stream.stream.CopyFrom(self._get_stream_id(self.name, stream_id))
out_stream.schema.CopyFrom(self._get_stream_schema(out_fields))
diff --git a/heronpy/api/metrics.py b/heronpy/api/metrics.py
index 89f31fa..b944d58 100644
--- a/heronpy/api/metrics.py
+++ b/heronpy/api/metrics.py
@@ -63,7 +63,7 @@
def get_value_and_reset(self):
ret = {}
- for key, value in self.value.items():
+ for key, value in list(self.value.items()):
ret[key] = value.get_value_and_reset()
return ret
@@ -144,7 +144,7 @@
def get_value_and_reset(self):
ret = {}
- for key, value in self.value.items():
+ for key, value in list(self.value.items()):
ret[key] = value.get_value_and_reset()
self.value[key] = value
return ret
diff --git a/heronpy/api/tests/python/component_unittest.py b/heronpy/api/tests/python/component_unittest.py
index a72adb3..9da4a8d 100644
--- a/heronpy/api/tests/python/component_unittest.py
+++ b/heronpy/api/tests/python/component_unittest.py
@@ -167,7 +167,7 @@
inputs_list = [GlobalStreamId("spout1", "default"), GlobalStreamId("spout2", "some_stream")]
spec = HeronComponentSpec("bolt", "bl_cls", False, 1, inputs=inputs_list)
ret = spec._sanitize_inputs()
- self.assertEqual(ret, dict(zip(inputs_list, [Grouping.SHUFFLE] * 2)))
+ self.assertEqual(ret, dict(list(zip(inputs_list, [Grouping.SHUFFLE] * 2))))
# list of neither GlobalStreamId nor HeronComponentSpec
inputs_list = [None, 123, "string", [GlobalStreamId("sp", "default")]]
diff --git a/heronpy/api/tests/python/metrics_unittest.py b/heronpy/api/tests/python/metrics_unittest.py
index e7d7866..063651a 100644
--- a/heronpy/api/tests/python/metrics_unittest.py
+++ b/heronpy/api/tests/python/metrics_unittest.py
@@ -44,8 +44,8 @@
for _ in range(10):
for key in key_list:
metric.incr(key=key)
- self.assertEqual(metric.get_value_and_reset(), dict(zip(key_list, [10] * 3)))
- self.assertEqual(metric.get_value_and_reset(), dict(zip(key_list, [0] * 3)))
+ self.assertEqual(metric.get_value_and_reset(), dict(list(zip(key_list, [10] * 3))))
+ self.assertEqual(metric.get_value_and_reset(), dict(list(zip(key_list, [0] * 3))))
metric.add_key("key4")
ret = metric.get_value_and_reset()
@@ -71,8 +71,8 @@
metric.update(key=key_list[0], value=i)
metric.update(key=key_list[1], value=i * 2)
metric.update(key=key_list[2], value=i * 3)
- self.assertEqual(metric.get_value_and_reset(), dict(zip(key_list, [5.5, 11, 16.5])))
- self.assertEqual(metric.get_value_and_reset(), dict(zip(key_list, [None] * 3)))
+ self.assertEqual(metric.get_value_and_reset(), dict(list(zip(key_list, [5.5, 11, 16.5]))))
+ self.assertEqual(metric.get_value_and_reset(), dict(list(zip(key_list, [None] * 3))))
metric.add_key("key4")
ret = metric.get_value_and_reset()
diff --git a/heronpy/api/topology.py b/heronpy/api/topology.py
index ae7396f..c62a6bd 100644
--- a/heronpy/api/topology.py
+++ b/heronpy/api/topology.py
@@ -47,7 +47,7 @@
spout_specs = {}
# Copy HeronComponentSpec items out of class_dict
specs = TopologyType.class_dict_to_specs(class_dict)
- for spec in iter(specs.values()):
+ for spec in iter(list(specs.values())):
if spec.is_spout:
TopologyType.add_spout_specs(spec, spout_specs)
else:
@@ -73,7 +73,7 @@
"""Takes a class `__dict__` and returns `HeronComponentSpec` entries"""
specs = {}
- for name, spec in class_dict.items():
+ for name, spec in list(class_dict.items()):
if isinstance(spec, HeronComponentSpec):
# Use the variable name as the specification name.
if spec.name is None:
@@ -103,7 +103,7 @@
# add defaults
topo_config.update(mcs.DEFAULT_TOPOLOGY_CONFIG)
- for name, custom_config in class_dict.items():
+ for name, custom_config in list(class_dict.items()):
if name == 'config' and isinstance(custom_config, dict):
sanitized_dict = mcs._sanitize_config(custom_config)
topo_config.update(sanitized_dict)
@@ -131,7 +131,7 @@
config = topology_pb2.Config()
conf_dict = class_dict['_topo_config']
- for key, value in conf_dict.items():
+ for key, value in list(conf_dict.items()):
if isinstance(value, str):
kvs = config.kvs.add()
kvs.key = key
@@ -243,7 +243,7 @@
These values will need to be serialized before adding to a protobuf message.
"""
sanitized = {}
- for key, value in custom_config.items():
+ for key, value in list(custom_config.items()):
if not isinstance(key, str):
raise TypeError("Key for topology-wide configuration must be string, given: %s: %s"
% (str(type(key)), str(key)))
diff --git a/heronpy/streamlet/impl/contextimpl.py b/heronpy/streamlet/impl/contextimpl.py
index d13cce9..d944ec7 100644
--- a/heronpy/streamlet/impl/contextimpl.py
+++ b/heronpy/streamlet/impl/contextimpl.py
@@ -36,7 +36,7 @@
return self._top_context.get_cluster_config()
def get_stream_name(self):
- return self._top_context.get_this_sources().keys()[0].id
+ return list(self._top_context.get_this_sources().keys())[0].id
def get_num_partitions(self):
return len(self._top_context.get_component_tasks(self._top_context.get_component_id()))
diff --git a/heronpy/streamlet/impl/joinbolt.py b/heronpy/streamlet/impl/joinbolt.py
index efa64f7..199e33b 100644
--- a/heronpy/streamlet/impl/joinbolt.py
+++ b/heronpy/streamlet/impl/joinbolt.py
@@ -80,7 +80,7 @@
if not isinstance(userdata, collections.Iterable) or len(userdata) != 2:
raise RuntimeError("Join tuples must be iterable of length 2")
self._add(userdata[0], userdata[1], tup.component, mymap)
- for (key, values) in mymap.items():
+ for (key, values) in list(mymap.items()):
if self._join_type == JoinBolt.INNER:
if values[0] and values[1]:
self.inner_join_and_emit(key, values, window_config)
diff --git a/heronpy/streamlet/impl/reducebykeyandwindowbolt.py b/heronpy/streamlet/impl/reducebykeyandwindowbolt.py
index a8267b9..7cd4a93 100644
--- a/heronpy/streamlet/impl/reducebykeyandwindowbolt.py
+++ b/heronpy/streamlet/impl/reducebykeyandwindowbolt.py
@@ -62,7 +62,7 @@
if not isinstance(userdata, collections.Iterable) or len(userdata) != 2:
raise RuntimeError("ReduceByWindow tuples must be iterable of length 2")
self._add(userdata[0], userdata[1], mymap)
- for (key, values) in mymap.items():
+ for (key, values) in list(mymap.items()):
result = values[0]
for value in values[1:]:
result = self.reduce_function(result, value)
diff --git a/integration_test/src/python/http_server/main.py b/integration_test/src/python/http_server/main.py
index 9a7dcd1..95c396c 100644
--- a/integration_test/src/python/http_server/main.py
+++ b/integration_test/src/python/http_server/main.py
@@ -108,13 +108,13 @@
if key in self.result_map:
# fix the duplicate record issue
for comp in self.result_map[key]:
- if comp.keys()[0] == data.keys()[0]:
+ if list(comp.keys())[0] == list(data.keys())[0]:
break
else:
self.result_map[key].append(data)
else:
self.result_map[key] = [data]
- self.write("Results written successfully: topology " + key + ' instance ' + data.keys()[0])
+ self.write("Results written successfully: topology " + key + ' instance ' + list(data.keys())[0])
else:
raise tornado.web.HTTPError(status_code=404, log_message="Invalid key %s" % key)
diff --git a/integration_test/src/python/integration_test/core/BUILD b/integration_test/src/python/integration_test/core/BUILD
index 11251e6..c052ec3 100644
--- a/integration_test/src/python/integration_test/core/BUILD
+++ b/integration_test/src/python/integration_test/core/BUILD
@@ -7,4 +7,5 @@
"//heronpy/api:heron-python-py",
"//heron/common/src/python:common-py"
],
+ reqs = ["future==0.18.2"]
)
diff --git a/integration_test/src/python/integration_test/core/aggregator_bolt.py b/integration_test/src/python/integration_test/core/aggregator_bolt.py
index 2460a42..4fd6cf1 100644
--- a/integration_test/src/python/integration_test/core/aggregator_bolt.py
+++ b/integration_test/src/python/integration_test/core/aggregator_bolt.py
@@ -19,9 +19,12 @@
# under the License.
'''aggregator_bolt.py'''
-import httplib
+from future.standard_library import install_aliases
+install_aliases()
+
+import http.client
import json
-from urlparse import urlparse
+from urllib.parse import urlparse
from heron.common.src.python.utils.log import Log
from ..core import constants as integ_constants
@@ -43,7 +46,7 @@
self.result.append(tup.values[0])
def _post_result_to_server(self, json_result):
- conn = httplib.HTTPConnection(self.parsed_url.netloc)
+ conn = http.client.HTTPConnection(self.parsed_url.netloc)
conn.request("POST", self.parsed_url.path, json_result)
response = conn.getresponse()
if response.status == 200:
diff --git a/integration_test/src/python/integration_test/core/integration_test_bolt.py b/integration_test/src/python/integration_test/core/integration_test_bolt.py
index 41b0581..cfb4d7d 100644
--- a/integration_test/src/python/integration_test/core/integration_test_bolt.py
+++ b/integration_test/src/python/integration_test/core/integration_test_bolt.py
@@ -59,7 +59,7 @@
upstream_components = set()
self.terminal_to_receive = 0
- for streamId in context.get_this_sources().keys():
+ for streamId in list(context.get_this_sources().keys()):
# streamId is topology_pb2.StreamId protobuf message
upstream_components.add(streamId.component_name)
for comp_name in upstream_components:
diff --git a/integration_test/src/python/integration_test/core/test_topology_builder.py b/integration_test/src/python/integration_test/core/test_topology_builder.py
index e2a77a7..d64a874 100644
--- a/integration_test/src/python/integration_test/core/test_topology_builder.py
+++ b/integration_test/src/python/integration_test/core/test_topology_builder.py
@@ -122,7 +122,7 @@
# building a graph directed from children to parents, by looking only on bolts
# since spouts don't have parents
- for name, bolt_spec in self.bolts.iteritems():
+ for name, bolt_spec in self.bolts.items():
if name == self.TERMINAL_BOLT_NAME:
continue
@@ -144,20 +144,20 @@
non_terminals = set()
# 1. terminal bolts need upstream components, because we don't want isolated bolts
# 2. terminal bolts should not exist in the prev.values(), meaning that no downstream
- for parent_set in self.prev.values():
+ for parent_set in list(self.prev.values()):
non_terminals.update(parent_set)
- for bolt_name in self.prev.keys():
+ for bolt_name in list(self.prev.keys()):
if bolt_name not in non_terminals:
terminals.add(bolt_name)
# will also consider the cases with spouts without children
- for spout_name in self.spouts.keys():
+ for spout_name in list(self.spouts.keys()):
if spout_name not in non_terminals:
terminals.add(spout_name)
# add all grouping to components
- for child in self.prev.keys():
+ for child in list(self.prev.keys()):
for parent in self.prev[child]:
self._add_all_grouping(child, parent, integ_const.INTEGRATION_TEST_CONTROL_STREAM_ID)
diff --git a/integration_test/src/python/local_test_runner/BUILD b/integration_test/src/python/local_test_runner/BUILD
index bebc1dd..65b5658 100644
--- a/integration_test/src/python/local_test_runner/BUILD
+++ b/integration_test/src/python/local_test_runner/BUILD
@@ -7,7 +7,10 @@
resources = [
"resources/test.conf",
],
- reqs = ["argparse==1.4.0"],
+ reqs = [
+ "argparse==1.4.0",
+ "future==0.18.2",
+ ],
deps = [
"//heron/common/src/python:common-py",
"//integration_test/src/python/common",
diff --git a/integration_test/src/python/local_test_runner/test_template.py b/integration_test/src/python/local_test_runner/test_template.py
index 78fe215..3ad5fa5 100644
--- a/integration_test/src/python/local_test_runner/test_template.py
+++ b/integration_test/src/python/local_test_runner/test_template.py
@@ -19,11 +19,14 @@
# under the License.
""" test_template.py """
+from future.standard_library import install_aliases
+install_aliases()
+
import json
import logging
import os
import time
-import urllib
+from urllib.request import urlopen
import signal
import subprocess
from collections import namedtuple
@@ -252,7 +255,7 @@
url = 'http://localhost:%s/topologies/physicalplan?' % self.params['trackerPort']\
+ 'cluster=local&environ=default&topology=IntegrationTest_LocalReadWriteTopology'
logging.debug("Fetching physical plan from %s", url)
- response = urllib.urlopen(url)
+ response = urlopen(url)
physical_plan_json = json.loads(response.read())
if 'result' not in physical_plan_json:
diff --git a/integration_test/src/python/test_runner/BUILD b/integration_test/src/python/test_runner/BUILD
index f104569..569c2bf 100644
--- a/integration_test/src/python/test_runner/BUILD
+++ b/integration_test/src/python/test_runner/BUILD
@@ -9,7 +9,10 @@
resources = [
"resources/test.json",
],
- reqs = ["argparse==1.4.0"],
+ reqs = [
+ "argparse==1.4.0",
+ "future==0.18.2"
+ ],
deps = [
"//heron/common/src/python:common-py",
"//integration_test/src/python/common",
diff --git a/integration_test/src/python/test_runner/main.py b/integration_test/src/python/test_runner/main.py
index 0d4d096..a09803a 100644
--- a/integration_test/src/python/test_runner/main.py
+++ b/integration_test/src/python/test_runner/main.py
@@ -24,7 +24,7 @@
import sys
import time
import uuid
-from httplib import HTTPConnection
+from http.client import HTTPConnection
from threading import Lock, Thread
from ..common import status
@@ -82,7 +82,7 @@
# need to convert from a list of json objects to a string of a python list,
# without the unicode using double quotes, not single quotes.
- return str(map(lambda x: str(x), result)).replace("'", '"')
+ return str([str(x) for x in result]).replace("'", '"')
except Exception as e:
raise status.TestFailure(
"Fetching expected result failed for %s topology" % self.topology_name, e)
@@ -133,8 +133,8 @@
else:
failure = status.TestFailure("Actual result did not match expected result")
# lambda required below to remove the unicode 'u' from the output
- logging.info("Actual result ---------- \n" + str(map(lambda x: str(x), actual_results)))
- logging.info("Expected result ---------- \n" + str(map(lambda x: str(x), expected_results)))
+ logging.info("Actual result ---------- \n" + str([str(x) for x in actual_results]))
+ logging.info("Expected result ---------- \n" + str([str(x) for x in expected_results]))
raise failure
class AtLeastOnceResultsChecker(ExactlyOnceResultsChecker):
@@ -161,9 +161,9 @@
failure = status.TestFailure("Actual result did not match expected result")
# lambda required below to remove the unicode 'u' from the output
logging.info("Actual value frequencies ---------- \n" + ', '.join(
- map(lambda (k, v): "%s(%s)" % (str(k), v), actual_counts.iteritems())))
+ ["%s(%s)" % (str(k_v[0]), k_v[1]) for k_v in iter(actual_counts.items())]))
logging.info("Expected value frequencies ---------- \n" + ', '.join(
- map(lambda (k, v): "%s(%s)" % (str(k), v), expected_counts.iteritems())))
+ ["%s(%s)" % (str(k_v1[0]), k_v1[1]) for k_v1 in iter(expected_counts.items())]))
raise failure
def _frequency_dict(values):
@@ -301,11 +301,11 @@
initial_topologies = test_topologies
if test_pattern:
pattern = re.compile(test_pattern)
- test_topologies = filter(lambda x: pattern.match(x['topologyName']), test_topologies)
+ test_topologies = [x for x in test_topologies if pattern.match(x['topologyName'])]
if len(test_topologies) == 0:
logging.error("Test filter '%s' did not match any configured test names:\n%s",
- test_pattern, '\n'.join(map(lambda x: x['topologyName'], initial_topologies)))
+ test_pattern, '\n'.join([x['topologyName'] for x in initial_topologies]))
sys.exit(1)
return test_topologies
diff --git a/integration_test/src/python/topology_test_runner/BUILD b/integration_test/src/python/topology_test_runner/BUILD
index 20c412b..e12ba45 100644
--- a/integration_test/src/python/topology_test_runner/BUILD
+++ b/integration_test/src/python/topology_test_runner/BUILD
@@ -9,7 +9,10 @@
resources = [
"resources/test.json",
],
- reqs = ["argparse==1.4.0"],
+ reqs = [
+ "argparse==1.4.0",
+ "future==0.18.2"
+ ],
deps = [
"//heron/common/src/python:common-py",
"//heronpy/proto:proto-py",
diff --git a/integration_test/src/python/topology_test_runner/main.py b/integration_test/src/python/topology_test_runner/main.py
index 08fefea..2ba8184 100644
--- a/integration_test/src/python/topology_test_runner/main.py
+++ b/integration_test/src/python/topology_test_runner/main.py
@@ -24,7 +24,7 @@
import sys
import time
import uuid
-from httplib import HTTPConnection
+from http.client import HTTPConnection
from threading import Lock, Thread
from ..common import status
@@ -198,7 +198,7 @@
raise status.TestFailure("Fail to get actual results of instance states for topology %s"
% self.topology_name)
- if '_' not in expected_result[0].keys()[0]:
+ if '_' not in list(expected_result[0].keys())[0]:
actual_result = self._parse_instance_id(actual_result)
return self._compare_state(sorted(expected_result), sorted(actual_result))
@@ -210,8 +210,8 @@
else:
failure = status.TestFailure("Actual result did not match expected result")
# lambda required below to remove the unicode 'u' from the output
- logging.info("Actual result ---------- \n" + str(map(lambda x: str(x), actual_results)))
- logging.info("Expected result ---------- \n" + str(map(lambda x: str(x), expected_results)))
+ logging.info("Actual result ---------- \n" + str([str(x) for x in actual_results]))
+ logging.info("Expected result ---------- \n" + str([str(x) for x in expected_results]))
raise failure
def _parse_instance_id(self, input):
@@ -345,11 +345,11 @@
initial_topologies = test_topologies
if test_pattern:
pattern = re.compile(test_pattern)
- test_topologies = filter(lambda x: pattern.match(x['topologyName']), test_topologies)
+ test_topologies = [x for x in test_topologies if pattern.match(x['topologyName'])]
if len(test_topologies) == 0:
logging.error("Test filter '%s' did not match any configured test names:\n%s",
- test_pattern, '\n'.join(map(lambda x: x['topologyName'], initial_topologies)))
+ test_pattern, '\n'.join([x['topologyName'] for x in initial_topologies]))
sys.exit(1)
return test_topologies
diff --git a/scripts/shutils/save-logs.py b/scripts/shutils/save-logs.py
index f101e63..c378936 100755
--- a/scripts/shutils/save-logs.py
+++ b/scripts/shutils/save-logs.py
@@ -30,7 +30,7 @@
with open(filename, "rb") as f:
fm = mmap.mmap(f.fileno(), 0, mmap.MAP_SHARED, mmap.PROT_READ)
try:
- for i in xrange(size - 1, -1, -1):
+ for i in range(size - 1, -1, -1):
if fm[i] == '\n':
n -= 1
if n == -1:
@@ -40,7 +40,7 @@
fm.close()
def main(file, cmd):
- print "%s writing to: %s" % (cmd, file)
+ print("%s writing to: %s" % (cmd, file))
with open(file, "w") as out:
count = 0
process = subprocess.Popen(cmd,
@@ -64,16 +64,16 @@
errcode = process.wait()
diff = datetime.now() - start
sys.stdout.write("\r%d seconds %d log lines"%(diff.seconds, count))
- print "\n %s finished with errcode: %d" % (cmd, errcode)
+ print("\n %s finished with errcode: %d" % (cmd, errcode))
if errcode != 0:
lines = tail(file, 1000)
- print '\n'.join(lines)
+ print('\n'.join(lines))
sys.exit(errcode)
return errcode
if __name__ == "__main__":
if sys.argv < 1:
- print "Usage: %s [file info]" % sys.argv[0]
+ print("Usage: %s [file info]" % sys.argv[0])
sys.exit(1)
file = sys.argv[1]
cmd = sys.argv[2:]
diff --git a/third_party/python/cpplint/cpplint.py b/third_party/python/cpplint/cpplint.py
index 74c75bc..c30a3bc 100755
--- a/third_party/python/cpplint/cpplint.py
+++ b/third_party/python/cpplint/cpplint.py
@@ -585,7 +585,7 @@
# False positives include C-style multi-line comments and multi-line strings
# but those have always been troublesome for cpplint.
_ALT_TOKEN_REPLACEMENT_PATTERN = re.compile(
- r'[ =()](' + ('|'.join(_ALT_TOKEN_REPLACEMENT.keys())) + r')(?=[ (]|$)')
+ r'[ =()](' + ('|'.join(list(_ALT_TOKEN_REPLACEMENT.keys()))) + r')(?=[ (]|$)')
# These constants define types of headers for use with
@@ -6247,9 +6247,9 @@
if filename != '-' and file_extension not in GetAllExtensions():
# bazel 0.5.1> uses four distinct generated files that gives a warning
# we suppress the warning for these files
- bazel_gen_files = set([
+ bazel_gen_files = set([
"external/local_config_cc/libtool",
- "external/local_config_cc/make_hashed_objlist.py",
+ "external/local_config_cc/make_hashed_objlist.py",
"external/local_config_cc/wrapped_ar",
"external/local_config_cc/wrapped_clang",
"external/local_config_cc/xcrunwrapper.sh",
