sdks/python/apache_beam/io/aws/s3io_test.py

#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements.  See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License.  You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""Tests for S3 client."""
# pytype: skip-file

import logging
import os
import random
import time
import unittest

from apache_beam.io.aws import s3io
from apache_beam.io.aws.clients.s3 import fake_client
from apache_beam.io.aws.clients.s3 import messages
from apache_beam.options import pipeline_options


class TestS3PathParser(unittest.TestCase):

  BAD_S3_PATHS = [
      's3://',
      's3://bucket',
      's3:///name',
      's3:///',
      's3:/blah/bucket/name',
  ]

  def test_s3_path(self):
    self.assertEqual(s3io.parse_s3_path('s3://bucket/name'), ('bucket', 'name'))
    self.assertEqual(
        s3io.parse_s3_path('s3://bucket/name/sub'), ('bucket', 'name/sub'))

  def test_bad_s3_path(self):
    for path in self.BAD_S3_PATHS:
      self.assertRaises(ValueError, s3io.parse_s3_path, path)
    self.assertRaises(ValueError, s3io.parse_s3_path, 's3://bucket/')

  def test_s3_path_object_optional(self):
    self.assertEqual(
        s3io.parse_s3_path('s3://bucket/name', object_optional=True),
        ('bucket', 'name'))
    self.assertEqual(
        s3io.parse_s3_path('s3://bucket/', object_optional=True),
        ('bucket', ''))

  def test_bad_s3_path_object_optional(self):
    for path in self.BAD_S3_PATHS:
      self.assertRaises(ValueError, s3io.parse_s3_path, path, True)


class TestS3IO(unittest.TestCase):
  def _insert_random_file(self, client, path, size):
    bucket, name = s3io.parse_s3_path(path)
    contents = os.urandom(size)
    fakeFile = fake_client.FakeFile(bucket, name, contents)

    if self.USE_MOCK:
      self.client.add_file(fakeFile)

    else:
      f = self.aws.open(path, 'w')
      f.write(contents)
      f.close()

    return fakeFile

  def setUp(self):

    # These tests can be run locally against a mock S3 client, or as integration
    # tests against the real S3 client.
    self.USE_MOCK = True

    # If you're running integration tests with S3, set this variable to be an
    # s3 path that you have access to where test data can be written. If you're
    # just running tests against the mock, this can be any s3 path. It should
    # end with a '/'.
    self.TEST_DATA_PATH = 's3://random-data-sets/beam_tests/'

    if self.USE_MOCK:
      self.client = fake_client.FakeS3Client()
      test_data_bucket, _ = s3io.parse_s3_path(self.TEST_DATA_PATH)
      self.client.known_buckets.add(test_data_bucket)
      self.aws = s3io.S3IO(self.client)

    else:
      self.aws = s3io.S3IO(options=pipeline_options.S3Options())
      self.client = self.aws.client

  def test_size(self):
    file_name = self.TEST_DATA_PATH + 'dummy_file'
    file_size = 1234

    self._insert_random_file(self.client, file_name, file_size)
    self.assertTrue(self.aws.exists(file_name))
    self.assertEqual(1234, self.aws.size(file_name))

    # Clean up
    self.aws.delete(file_name)

  def test_last_updated(self):
    file_name = self.TEST_DATA_PATH + 'dummy_file'
    file_size = 1234

    self._insert_random_file(self.client, file_name, file_size)
    self.assertTrue(self.aws.exists(file_name))
    # The time difference should be tiny for the mock client.
    # A loose tolerance is for the consideration of real s3 client.
    tolerance = 5 * 60  # 5 mins
    result = self.aws.last_updated(file_name)
    self.assertAlmostEqual(result, time.time(), delta=tolerance)

    # Clean up
    self.aws.delete(file_name)

  def test_checksum(self):
    file_name = self.TEST_DATA_PATH + 'checksum'
    file_size = 1024
    file_ = self._insert_random_file(self.client, file_name, file_size)

    original_etag = self.aws.checksum(file_name)

    self.aws.delete(file_name)

    with self.aws.open(file_name, 'w') as f:
      f.write(file_.contents)

    rewritten_etag = self.aws.checksum(file_name)

    self.assertEqual(original_etag, rewritten_etag)
    self.assertEqual(len(original_etag), 36)
    self.assertTrue(original_etag.endswith('-1"'))

    # Clean up
    self.aws.delete(file_name)

  def test_file_status(self):
    file_name = self.TEST_DATA_PATH + 'metadata'
    file_size = 1024
    self._insert_random_file(self.client, file_name, file_size)
    file_checksum = self.aws.checksum(file_name)
    file_timestamp = self.aws.last_updated(file_name)

    file_status = self.aws._status(file_name)

    self.assertEqual(file_status['size'], file_size)
    self.assertEqual(file_status['checksum'], file_checksum)
    self.assertEqual(file_status['last_updated'], file_timestamp)

    # Clean up
    self.aws.delete(file_name)

  def test_copy(self):
    src_file_name = self.TEST_DATA_PATH + 'source'
    dest_file_name = self.TEST_DATA_PATH + 'dest'
    file_size = 1024
    self._insert_random_file(self.client, src_file_name, file_size)

    self.assertTrue(
        src_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))
    self.assertFalse(
        dest_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))

    self.aws.copy(src_file_name, dest_file_name)

    self.assertTrue(
        src_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))
    self.assertTrue(
        dest_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))

    # Clean up
    self.aws.delete_files([src_file_name, dest_file_name])

    # Test copy of non-existent files.
    with self.assertRaises(messages.S3ClientError) as err:
      self.aws.copy(
          self.TEST_DATA_PATH + 'non-existent',
          self.TEST_DATA_PATH + 'non-existent-destination')

    self.assertTrue('Not Found' in err.exception.message)

  def test_copy_paths(self):
    from_name_pattern = self.TEST_DATA_PATH + 'copy_me_%d'
    to_name_pattern = self.TEST_DATA_PATH + 'destination_%d'
    file_size = 1024
    num_files = 10

    src_dest_pairs = [(from_name_pattern % i, to_name_pattern % i)
                      for i in range(num_files)]

    result = self.aws.copy_paths(src_dest_pairs)

    self.assertTrue(result)
    for i, (src, dest, exception) in enumerate(result):
      self.assertEqual(src, from_name_pattern % i)
      self.assertEqual(dest, to_name_pattern % i)
      self.assertTrue(isinstance(exception, messages.S3ClientError))
      self.assertEqual(exception.code, 404)
      self.assertFalse(self.aws.exists(from_name_pattern % i))
      self.assertFalse(self.aws.exists(to_name_pattern % i))

    # Insert some files.
    for i in range(num_files):
      self._insert_random_file(self.client, from_name_pattern % i, file_size)

    # Check files inserted properly.
    for i in range(num_files):
      self.assertTrue(self.aws.exists(from_name_pattern % i))

    # Execute batch copy.
    result = self.aws.copy_paths(src_dest_pairs)

    # Check files copied properly.
    for i in range(num_files):
      self.assertTrue(self.aws.exists(from_name_pattern % i))
      self.assertTrue(self.aws.exists(to_name_pattern % i))

    # Check results
    for i, (src, dest, exception) in enumerate(result):
      self.assertEqual(src_dest_pairs[i], (src, dest))
      self.assertEqual(exception, None)

    # Clean up
    all_files = set().union(*[set(pair) for pair in src_dest_pairs])
    self.aws.delete_files(all_files)

  def test_copy_paths_error(self):
    n_real_files = 3

    # Create some files
    from_path = self.TEST_DATA_PATH + 'copy_paths/'
    files = [from_path + '%d' % i for i in range(n_real_files)]
    to_path = self.TEST_DATA_PATH + 'destination/'
    destinations = [to_path + '%d' % i for i in range(n_real_files)]
    for file_ in files:
      self._insert_random_file(self.client, file_, 1024)

    # Add nonexistent files to the sources and destinations
    sources = files + [
        from_path + 'X',
        from_path + 'fake_directory_1/',
        from_path + 'fake_directory_2/'
    ]
    destinations += [
        to_path + 'X',
        to_path + 'fake_directory_1/',
        to_path + 'fake_directory_2'
    ]
    result = self.aws.copy_paths(list(zip(sources, destinations)))

    # The copy_paths function of class S3IO does not return one single
    # result when copying a directory. Instead, it returns the results
    # of copying every file in the source directory.
    self.assertEqual(len(result), len(sources) - 1)

    for _, _, err in result[:n_real_files]:
      self.assertTrue(err is None)

    for _, _, err in result[n_real_files:]:
      self.assertIsInstance(err, messages.S3ClientError)

    # For the same reason of copy_paths function of S3IO above
    # skip this assert.
    #self.assertEqual(result[-3][2].code, 404)
    self.assertEqual(result[-2][2].code, 404)
    self.assertEqual(result[-1][2].code, 400)

    # Clean up
    self.aws.delete_files(files)
    self.aws.delete_files(destinations)

  def test_copy_tree(self):
    src_dir_name = self.TEST_DATA_PATH + 'source/'
    dest_dir_name = self.TEST_DATA_PATH + 'dest/'
    file_size = 1024
    paths = ['a', 'b/c', 'b/d']
    for path in paths:
      src_file_name = src_dir_name + path
      dest_file_name = dest_dir_name + path
      self._insert_random_file(self.client, src_file_name, file_size)
      self.assertTrue(
          src_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))
      self.assertFalse(
          dest_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))

    results = self.aws.copy_tree(src_dir_name, dest_dir_name)

    for src_file_name, dest_file_name, err in results:

      self.assertTrue(src_dir_name in src_file_name)
      self.assertTrue(dest_dir_name in dest_file_name)
      self.assertIsNone(err)

      self.assertTrue(
          src_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))
      self.assertTrue(
          dest_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))

    # Clean up
    for path in paths:
      src_file_name = src_dir_name + path
      dest_file_name = dest_dir_name + path
      self.aws.delete_files([src_file_name, dest_file_name])

  def test_rename(self):
    src_file_name = self.TEST_DATA_PATH + 'source'
    dest_file_name = self.TEST_DATA_PATH + 'dest'
    file_size = 1024

    self._insert_random_file(self.client, src_file_name, file_size)

    self.assertTrue(
        src_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))
    self.assertFalse(
        dest_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))

    self.aws.rename(src_file_name, dest_file_name)

    self.assertFalse(
        src_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))
    self.assertTrue(
        dest_file_name in dict(self.aws.list_files(self.TEST_DATA_PATH)))

    # Clean up
    self.aws.delete_files([src_file_name, dest_file_name])

  def test_rename_files(self):
    from_name_pattern = self.TEST_DATA_PATH + 'to_rename_%d'
    to_name_pattern = self.TEST_DATA_PATH + 'been_renamed_%d'
    file_size = 1024
    num_files = 10

    src_dest_pairs = [(from_name_pattern % i, to_name_pattern % i)
                      for i in range(num_files)]

    result = self.aws.rename_files(src_dest_pairs)

    self.assertTrue(result)
    for i, (src, dest, exception) in enumerate(result):
      self.assertEqual(src, from_name_pattern % i)
      self.assertEqual(dest, to_name_pattern % i)
      self.assertTrue(isinstance(exception, messages.S3ClientError))
      self.assertEqual(exception.code, 404)
      self.assertFalse(self.aws.exists(from_name_pattern % i))
      self.assertFalse(self.aws.exists(to_name_pattern % i))

    # Insert some files.
    for i in range(num_files):
      self._insert_random_file(self.client, from_name_pattern % i, file_size)

    # Check files inserted properly.
    for i in range(num_files):
      self.assertTrue(self.aws.exists(from_name_pattern % i))
      self.assertFalse(self.aws.exists(to_name_pattern % i))

    # Execute batch rename.
    self.aws.rename_files(src_dest_pairs)

    # Check files were renamed properly.
    for i in range(num_files):
      self.assertFalse(self.aws.exists(from_name_pattern % i))
      self.assertTrue(self.aws.exists(to_name_pattern % i))

    # Clean up
    all_files = set().union(*[set(pair) for pair in src_dest_pairs])
    self.aws.delete_files(all_files)

  def test_rename_files_with_errors(self):
    real_prefix = self.TEST_DATA_PATH + 'rename_batch_%s'
    fake_prefix = 's3://fake-bucket-68ae4b0ef7b9/rename_batch_%s'
    src_dest_pairs = [(prefix % 'src', prefix % 'dest')
                      for prefix in (real_prefix, fake_prefix)]

    # Create the file in the real bucket
    self._insert_random_file(self.client, real_prefix % 'src', 1024)

    # Execute batch rename
    result = self.aws.rename_files(src_dest_pairs)

    # First is the file in the real bucket, which shouldn't throw an error
    self.assertEqual(result[0][0], src_dest_pairs[0][0])
    self.assertEqual(result[0][1], src_dest_pairs[0][1])
    self.assertIsNone(result[0][2])

    # Second is the file in the fake bucket, which should throw a 404
    self.assertEqual(result[1][0], src_dest_pairs[1][0])
    self.assertEqual(result[1][1], src_dest_pairs[1][1])
    self.assertEqual(result[1][2].code, 404)

    # Clean up
    self.aws.delete(real_prefix % 'dest')

  def test_rename_files_with_errors_directory(self):

    # Make file
    dir_name = self.TEST_DATA_PATH + 'rename_dir/'
    file_name = dir_name + 'file'
    self._insert_random_file(self.client, file_name, 1024)

    self.assertTrue(self.aws.exists(file_name))

    with self.assertRaises(ValueError):
      self.aws.rename_files([(file_name, self.TEST_DATA_PATH + 'dir_dest/')])

    # Clean up
    self.aws.delete(file_name)

  def test_delete_paths(self):
    # Make files
    prefix = self.TEST_DATA_PATH + 'delete_paths/'
    file_names = [prefix + 'a', prefix + 'b/c']
    for file_name in file_names:
      self._insert_random_file(self.client, file_name, 1024)

    self.assertTrue(self.aws.exists(file_names[0]))
    self.assertTrue(self.aws.exists(file_names[1]))

    # Delete paths
    paths = [prefix + 'a', prefix + 'b/']
    self.aws.delete_paths(paths)

    self.assertFalse(self.aws.exists(file_names[0]))
    self.assertFalse(self.aws.exists(file_names[1]))

  def test_delete(self):
    file_name = self.TEST_DATA_PATH + 'delete_file'
    file_size = 1024

    # Test deletion of non-existent file (shouldn't raise any error)
    self.aws.delete(file_name)

    # Create the file and check that it was created
    self._insert_random_file(self.aws.client, file_name, file_size)
    files = dict(self.aws.list_files(self.TEST_DATA_PATH))
    self.assertTrue(file_name in files)

    # Delete the file and check that it was deleted
    self.aws.delete(file_name)
    self.assertFalse(self.aws.exists(file_name))

  def test_delete_files(self, *unused_args):
    file_name_pattern = self.TEST_DATA_PATH + 'delete_batch/%d'
    file_size = 1024
    num_files = 5

    # Test deletion of non-existent files.
    result = self.aws.delete_files(
        [file_name_pattern % i for i in range(num_files)])
    self.assertTrue(result)
    for i, (file_name, exception) in enumerate(result):
      self.assertEqual(file_name, file_name_pattern % i)
      self.assertEqual(exception, None)
      self.assertFalse(self.aws.exists(file_name_pattern % i))

    # Insert some files.
    for i in range(num_files):
      self._insert_random_file(self.client, file_name_pattern % i, file_size)

    # Check files inserted properly.
    for i in range(num_files):
      self.assertTrue(self.aws.exists(file_name_pattern % i))

    # Execute batch delete.
    self.aws.delete_files([file_name_pattern % i for i in range(num_files)])

    # Check files deleted properly.
    for i in range(num_files):
      self.assertFalse(self.aws.exists(file_name_pattern % i))

  def test_delete_files_with_errors(self, *unused_args):
    real_file = self.TEST_DATA_PATH + 'delete_batch/file'
    fake_file = 's3://fake-bucket-68ae4b0ef7b9/delete_batch/file'
    filenames = [real_file, fake_file]

    result = self.aws.delete_files(filenames)

    # First is the file in the real bucket, which shouldn't throw an error
    self.assertEqual(result[0][0], filenames[0])
    self.assertIsNone(result[0][1])

    # Second is the file in the fake bucket, which should throw a 404
    self.assertEqual(result[1][0], filenames[1])
    self.assertEqual(result[1][1].code, 404)

  def test_delete_tree(self):

    root_path = self.TEST_DATA_PATH + 'delete_tree/'
    leaf_paths = ['a', 'b/c', 'b/d', 'b/d/e']
    paths = [root_path + leaf for leaf in leaf_paths]

    # Create file tree
    file_size = 1024
    for path in paths:
      self._insert_random_file(self.client, path, file_size)

    # Check that the files exist
    for path in paths:
      self.assertTrue(self.aws.exists(path))

    # Delete the tree
    self.aws.delete_tree(root_path)

    # Check that the files have been deleted
    for path in paths:
      self.assertFalse(self.aws.exists(path))

  def test_exists(self):
    file_name = self.TEST_DATA_PATH + 'exists'
    file_size = 1024

    self.assertFalse(self.aws.exists(file_name))

    self._insert_random_file(self.aws.client, file_name, file_size)

    self.assertTrue(self.aws.exists(file_name))

    # Clean up
    self.aws.delete(file_name)

    self.assertFalse(self.aws.exists(file_name))

  def test_file_mode(self):
    file_name = self.TEST_DATA_PATH + 'jerry/pigpen/bobby'
    with self.aws.open(file_name, 'w') as f:
      assert f.mode == 'w'
    with self.aws.open(file_name, 'r') as f:
      assert f.mode == 'r'

    # Clean up
    self.aws.delete(file_name)

  def test_full_file_read(self):
    file_name = self.TEST_DATA_PATH + 'jerry/pigpen/phil'
    file_size = 1024

    f = self._insert_random_file(self.aws.client, file_name, file_size)
    contents = f.contents

    f = self.aws.open(file_name)
    self.assertEqual(f.mode, 'r')
    f.seek(0, os.SEEK_END)
    self.assertEqual(f.tell(), file_size)
    self.assertEqual(f.read(), b'')
    f.seek(0)
    self.assertEqual(f.read(), contents)

    # Clean up
    self.aws.delete(file_name)

  def test_file_write(self):
    file_name = self.TEST_DATA_PATH + 'write_file'
    file_size = 8 * 1024 * 1024 + 2000
    contents = os.urandom(file_size)
    f = self.aws.open(file_name, 'w')
    self.assertEqual(f.mode, 'w')
    f.write(contents[0:1000])
    f.write(contents[1000:1024 * 1024])
    f.write(contents[1024 * 1024:])
    f.close()
    new_f = self.aws.open(file_name, 'r')
    new_f_contents = new_f.read()
    self.assertEqual(new_f_contents, contents)

    # Clean up
    self.aws.delete(file_name)

  def test_file_mime_type(self):
    if self.USE_MOCK:
      self.skipTest("The boto3_client mock doesn't support mime_types")

    mime_type = 'example/example'
    file_name = self.TEST_DATA_PATH + 'write_file'
    f = self.aws.open(file_name, 'w', mime_type=mime_type)
    f.write(b'a string of binary text')
    f.close()

    bucket, key = s3io.parse_s3_path(file_name)
    metadata = self.client.get_object_metadata(messages.GetRequest(bucket, key))

    self.assertEqual(mime_type, metadata.mime_type)

    # Clean up
    self.aws.delete(file_name)

  def test_file_random_seek(self):
    file_name = self.TEST_DATA_PATH + 'write_seek_file'
    file_size = 5 * 1024 * 1024 - 100
    contents = os.urandom(file_size)
    with self.aws.open(file_name, 'w') as wf:
      wf.write(contents)

    f = self.aws.open(file_name)
    random.seed(0)

    for _ in range(0, 10):
      a = random.randint(0, file_size - 1)
      b = random.randint(0, file_size - 1)
      start, end = min(a, b), max(a, b)
      f.seek(start)

      self.assertEqual(f.tell(), start)

      self.assertEqual(f.read(end - start + 1), contents[start:end + 1])
      self.assertEqual(f.tell(), end + 1)

    # Clean up
    self.aws.delete(file_name)

  def test_file_flush(self):
    file_name = self.TEST_DATA_PATH + 'flush_file'
    file_size = 5 * 1024 * 1024 + 2000
    contents = os.urandom(file_size)
    f = self.aws.open(file_name, 'w')
    self.assertEqual(f.mode, 'w')
    f.write(contents[0:1000])
    f.flush()
    f.write(contents[1000:1024 * 1024])
    f.flush()
    f.flush()  # Should be a NOOP.
    f.write(contents[1024 * 1024:])
    f.close(
    )  # This should al`read`y call the equivalent of flush() in its body
    new_f = self.aws.open(file_name, 'r')
    new_f_contents = new_f.read()
    self.assertEqual(new_f_contents, contents)

    # Clean up
    self.aws.delete(file_name)

  def test_file_iterator(self):
    file_name = self.TEST_DATA_PATH + 'iterate_file'
    lines = []
    line_count = 10
    for _ in range(line_count):
      line_length = random.randint(100, 500)
      line = os.urandom(line_length).replace(b'\n', b' ') + b'\n'
      lines.append(line)

    contents = b''.join(lines)

    with self.aws.open(file_name, 'w') as wf:
      wf.write(contents)

    f = self.aws.open(file_name)

    read_lines = 0
    for line in f:
      read_lines += 1

    self.assertEqual(read_lines, line_count)

    # Clean up
    self.aws.delete(file_name)

  def test_file_read_line(self):
    file_name = self.TEST_DATA_PATH + 'read_line_file'
    lines = []

    # Set a small buffer size to exercise refilling the buffer.
    # First line is carefully crafted so the newline falls as the last character
    # of the buffer to exercise this code path.
    read_buffer_size = 1099
    lines.append(b'x' * 1023 + b'\n')

    for _ in range(1, 1000):
      line_length = random.randint(100, 500)
      line = os.urandom(line_length).replace(b'\n', b' ') + b'\n'
      lines.append(line)
    contents = b''.join(lines)

    file_size = len(contents)

    with self.aws.open(file_name, 'wb') as wf:
      wf.write(contents)

    f = self.aws.open(file_name, 'rb', read_buffer_size=read_buffer_size)

    # Test read of first two lines.
    f.seek(0)
    self.assertEqual(f.readline(), lines[0])
    self.assertEqual(f.tell(), len(lines[0]))
    self.assertEqual(f.readline(), lines[1])

    # Test read at line boundary.
    f.seek(file_size - len(lines[-1]) - 1)
    self.assertEqual(f.readline(), b'\n')

    # Test read at end of file.
    f.seek(file_size)
    self.assertEqual(f.readline(), b'')

    # Test reads at random positions.
    random.seed(0)
    for _ in range(0, 10):
      start = random.randint(0, file_size - 1)
      line_index = 0
      # Find line corresponding to start index.
      chars_left = start
      while True:
        next_line_length = len(lines[line_index])
        if chars_left - next_line_length < 0:
          break
        chars_left -= next_line_length
        line_index += 1
      f.seek(start)
      self.assertEqual(f.readline(), lines[line_index][chars_left:])

    # Clean up
    self.aws.delete(file_name)

  def test_file_close(self):
    file_name = self.TEST_DATA_PATH + 'close_file'
    file_size = 5 * 1024 * 1024 + 2000
    contents = os.urandom(file_size)
    f = self.aws.open(file_name, 'w')
    self.assertEqual(f.mode, 'w')
    f.write(contents)
    f.close()
    f.close()  # This should not crash.

    with self.aws.open(file_name, 'r') as f:
      read_contents = f.read()

    self.assertEqual(read_contents, contents)

    # Clean up
    self.aws.delete(file_name)

  def test_context_manager(self):
    # Test writing with a context manager.
    file_name = self.TEST_DATA_PATH + 'context_manager_file'
    file_size = 1024
    contents = os.urandom(file_size)
    with self.aws.open(file_name, 'w') as f:
      f.write(contents)

    with self.aws.open(file_name, 'r') as f:
      self.assertEqual(f.read(), contents)

    # Clean up
    self.aws.delete(file_name)

  def test_list_files(self):

    objects = [
        ('jerry/pigpen/phil', 5),
        ('jerry/pigpen/bobby', 3),
        ('jerry/billy/bobby', 4),
    ]

    for (object_name, size) in objects:
      file_name = self.TEST_DATA_PATH + object_name
      self._insert_random_file(self.aws.client, file_name, size)

    test_cases = [
        (
            self.TEST_DATA_PATH + 'j',
            [
                ('jerry/pigpen/phil', 5),
                ('jerry/pigpen/bobby', 3),
                ('jerry/billy/bobby', 4),
            ]),
        (
            self.TEST_DATA_PATH + 'jerry/',
            [
                ('jerry/pigpen/phil', 5),
                ('jerry/pigpen/bobby', 3),
                ('jerry/billy/bobby', 4),
            ]),
        (
            self.TEST_DATA_PATH + 'jerry/pigpen/phil', [
                ('jerry/pigpen/phil', 5),
            ]),
    ]

    for file_pattern, expected_object_names in test_cases:
      expected_file_names = [(self.TEST_DATA_PATH + object_name, size)
                             for (object_name, size) in expected_object_names]
      self.assertEqual(
          set(dict(self.aws.list_files(file_pattern)).items()),
          set(expected_file_names))

    # Clean up
    for (object_name, size) in objects:
      self.aws.delete(self.TEST_DATA_PATH + object_name)

  def test_midsize_file(self):
    file_name = self.TEST_DATA_PATH + 'midsized'
    file_size = 6 * 1024 * 1024
    self._insert_random_file(self.aws.client, file_name, file_size)
    with self.aws.open(file_name, 'r') as f:
      self.assertEqual(len(f.read()), file_size)
    self.aws.delete(file_name)

  def test_zerosize_file(self):
    file_name = self.TEST_DATA_PATH + 'zerosized'
    file_size = 0
    self._insert_random_file(self.aws.client, file_name, file_size)
    with self.aws.open(file_name, 'r') as f:
      self.assertEqual(len(f.read()), file_size)
    self.aws.delete(file_name)


if __name__ == '__main__':
  logging.getLogger().setLevel(logging.INFO)
  unittest.main()