| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| # coding: utf-8 |
| # pylint: disable= |
| """Dataset container.""" |
| __all__ = ['MNIST', 'FashionMNIST', 'CIFAR10', 'CIFAR100', |
| 'ImageRecordDataset', 'ImageFolderDataset', 'ImageListDataset'] |
| |
| import os |
| import gzip |
| import tarfile |
| import struct |
| import warnings |
| import numpy as np |
| |
| from .. import dataset |
| from ...utils import download, check_sha1, _get_repo_file_url |
| from .... import ndarray as nd, image, recordio, base |
| from .... import numpy as _mx_np # pylint: disable=reimported |
| from ....util import is_np_array, default_array |
| from ....base import numeric_types |
| |
| |
| class MNIST(dataset._DownloadedDataset): |
| """MNIST handwritten digits dataset from http://yann.lecun.com/exdb/mnist |
| |
| Each sample is an image (in 3D NDArray) with shape (28, 28, 1). |
| |
| Parameters |
| ---------- |
| root : str, default $MXNET_HOME/datasets/mnist |
| Path to temp folder for storing data. |
| train : bool, default True |
| Whether to load the training or testing set. |
| transform : function, default None |
| DEPRECATED FUNCTION ARGUMENTS. |
| A user defined callback that transforms each sample. For example:: |
| |
| transform=lambda data, label: (data.astype(np.float32)/255, label) |
| |
| """ |
| def __init__(self, root=os.path.join(base.data_dir(), 'datasets', 'mnist'), |
| train=True, transform=None): |
| self._train = train |
| self._train_data = ('train-images-idx3-ubyte.gz', |
| '6c95f4b05d2bf285e1bfb0e7960c31bd3b3f8a7d') |
| self._train_label = ('train-labels-idx1-ubyte.gz', |
| '2a80914081dc54586dbdf242f9805a6b8d2a15fc') |
| self._test_data = ('t10k-images-idx3-ubyte.gz', |
| 'c3a25af1f52dad7f726cce8cacb138654b760d48') |
| self._test_label = ('t10k-labels-idx1-ubyte.gz', |
| '763e7fa3757d93b0cdec073cef058b2004252c17') |
| self._namespace = 'mnist' |
| super(MNIST, self).__init__(root, transform) |
| |
| def _get_data(self): |
| if self._train: |
| data, label = self._train_data, self._train_label |
| else: |
| data, label = self._test_data, self._test_label |
| |
| namespace = 'gluon/dataset/'+self._namespace |
| data_file = download(_get_repo_file_url(namespace, data[0]), |
| path=self._root, |
| sha1_hash=data[1]) |
| label_file = download(_get_repo_file_url(namespace, label[0]), |
| path=self._root, |
| sha1_hash=label[1]) |
| |
| with gzip.open(label_file, 'rb') as fin: |
| struct.unpack(">II", fin.read(8)) |
| label = np.frombuffer(fin.read(), dtype=np.uint8).astype(np.int32) |
| if is_np_array(): |
| label = _mx_np.array(label, dtype=label.dtype) |
| |
| with gzip.open(data_file, 'rb') as fin: |
| struct.unpack(">IIII", fin.read(16)) |
| data = np.frombuffer(fin.read(), dtype=np.uint8) |
| data = data.reshape(len(label), 28, 28, 1) |
| |
| array_fn = _mx_np.array if is_np_array() else nd.array |
| self._data = array_fn(data, dtype=data.dtype) |
| self._label = label |
| |
| |
| class FashionMNIST(MNIST): |
| """A dataset of Zalando's article images consisting of fashion products, |
| a drop-in replacement of the original MNIST dataset from |
| https://github.com/zalandoresearch/fashion-mnist |
| |
| Each sample is an image (in 3D NDArray) with shape (28, 28, 1). |
| |
| Parameters |
| ---------- |
| root : str, default $MXNET_HOME/datasets/fashion-mnist' |
| Path to temp folder for storing data. |
| train : bool, default True |
| Whether to load the training or testing set. |
| transform : function, default None |
| DEPRECATED FUNCTION ARGUMENTS. |
| A user defined callback that transforms each sample. For example:: |
| |
| transform=lambda data, label: (data.astype(np.float32)/255, label) |
| |
| """ |
| def __init__(self, root=os.path.join(base.data_dir(), 'datasets', 'fashion-mnist'), |
| train=True, transform=None): |
| self._train = train |
| self._train_data = ('train-images-idx3-ubyte.gz', |
| '0cf37b0d40ed5169c6b3aba31069a9770ac9043d') |
| self._train_label = ('train-labels-idx1-ubyte.gz', |
| '236021d52f1e40852b06a4c3008d8de8aef1e40b') |
| self._test_data = ('t10k-images-idx3-ubyte.gz', |
| '626ed6a7c06dd17c0eec72fa3be1740f146a2863') |
| self._test_label = ('t10k-labels-idx1-ubyte.gz', |
| '17f9ab60e7257a1620f4ad76bbbaf857c3920701') |
| self._namespace = 'fashion-mnist' |
| super(MNIST, self).__init__(root, transform) # pylint: disable=bad-super-call |
| |
| |
| class CIFAR10(dataset._DownloadedDataset): |
| """CIFAR10 image classification dataset from https://www.cs.toronto.edu/~kriz/cifar.html |
| |
| Each sample is an image (in 3D NDArray) with shape (32, 32, 3). |
| |
| Parameters |
| ---------- |
| root : str, default $MXNET_HOME/datasets/cifar10 |
| Path to temp folder for storing data. |
| train : bool, default True |
| Whether to load the training or testing set. |
| transform : function, default None |
| DEPRECATED FUNCTION ARGUMENTS. |
| A user defined callback that transforms each sample. For example:: |
| |
| transform=lambda data, label: (data.astype(np.float32)/255, label) |
| |
| """ |
| def __init__(self, root=os.path.join(base.data_dir(), 'datasets', 'cifar10'), |
| train=True, transform=None): |
| self._train = train |
| self._archive_file = ('cifar-10-binary.tar.gz', 'fab780a1e191a7eda0f345501ccd62d20f7ed891') |
| self._train_data = [('data_batch_1.bin', 'aadd24acce27caa71bf4b10992e9e7b2d74c2540'), |
| ('data_batch_2.bin', 'c0ba65cce70568cd57b4e03e9ac8d2a5367c1795'), |
| ('data_batch_3.bin', '1dd00a74ab1d17a6e7d73e185b69dbf31242f295'), |
| ('data_batch_4.bin', 'aab85764eb3584312d3c7f65fd2fd016e36a258e'), |
| ('data_batch_5.bin', '26e2849e66a845b7f1e4614ae70f4889ae604628')] |
| self._test_data = [('test_batch.bin', '67eb016db431130d61cd03c7ad570b013799c88c')] |
| self._namespace = 'cifar10' |
| super(CIFAR10, self).__init__(root, transform) |
| |
| def _read_batch(self, filename): |
| with open(filename, 'rb') as fin: |
| data = np.frombuffer(fin.read(), dtype=np.uint8).reshape(-1, 3072+1) |
| |
| return data[:, 1:].reshape(-1, 3, 32, 32).transpose(0, 2, 3, 1), \ |
| data[:, 0].astype(np.int32) |
| |
| def _get_data(self): |
| if any(not os.path.exists(path) or not check_sha1(path, sha1) |
| for path, sha1 in ((os.path.join(self._root, name), sha1) |
| for name, sha1 in self._train_data + self._test_data)): |
| namespace = 'gluon/dataset/'+self._namespace |
| filename = download(_get_repo_file_url(namespace, self._archive_file[0]), |
| path=self._root, |
| sha1_hash=self._archive_file[1]) |
| |
| with tarfile.open(filename) as tar: |
| tar.extractall(self._root) |
| |
| if self._train: |
| data_files = self._train_data |
| else: |
| data_files = self._test_data |
| data, label = zip(*(self._read_batch(os.path.join(self._root, name)) |
| for name, _ in data_files)) |
| data = np.concatenate(data) |
| label = np.concatenate(label) |
| |
| array_fn = _mx_np.array if is_np_array() else nd.array |
| self._data = array_fn(data, dtype=data.dtype) |
| self._label = array_fn(label, dtype=label.dtype) if is_np_array() else label |
| |
| |
| class CIFAR100(CIFAR10): |
| """CIFAR100 image classification dataset from https://www.cs.toronto.edu/~kriz/cifar.html |
| |
| Each sample is an image (in 3D NDArray) with shape (32, 32, 3). |
| |
| Parameters |
| ---------- |
| root : str, default $MXNET_HOME/datasets/cifar100 |
| Path to temp folder for storing data. |
| fine_label : bool, default False |
| Whether to load the fine-grained (100 classes) or coarse-grained (20 super-classes) labels. |
| train : bool, default True |
| Whether to load the training or testing set. |
| transform : function, default None |
| DEPRECATED FUNCTION ARGUMENTS. |
| A user defined callback that transforms each sample. For example:: |
| |
| transform=lambda data, label: (data.astype(np.float32)/255, label) |
| |
| """ |
| def __init__(self, root=os.path.join(base.data_dir(), 'datasets', 'cifar100'), |
| fine_label=False, train=True, transform=None): |
| self._train = train |
| self._archive_file = ('cifar-100-binary.tar.gz', 'a0bb982c76b83111308126cc779a992fa506b90b') |
| self._train_data = [('train.bin', 'e207cd2e05b73b1393c74c7f5e7bea451d63e08e')] |
| self._test_data = [('test.bin', '8fb6623e830365ff53cf14adec797474f5478006')] |
| self._fine_label = fine_label |
| self._namespace = 'cifar100' |
| super(CIFAR10, self).__init__(root, transform) # pylint: disable=bad-super-call |
| |
| def _read_batch(self, filename): |
| with open(filename, 'rb') as fin: |
| data = np.frombuffer(fin.read(), dtype=np.uint8).reshape(-1, 3072+2) |
| |
| return data[:, 2:].reshape(-1, 3, 32, 32).transpose(0, 2, 3, 1), \ |
| data[:, 0+self._fine_label].astype(np.int32) |
| |
| |
| class ImageRecordDataset(dataset.RecordFileDataset): |
| """A dataset wrapping over a RecordIO file containing images. |
| |
| Each sample is an image and its corresponding label. |
| |
| Parameters |
| ---------- |
| filename : str |
| Path to rec file. |
| flag : {0, 1}, default 1 |
| If 0, always convert images to greyscale. \ |
| If 1, always convert images to colored (RGB). |
| transform : function, default None |
| DEPRECATED FUNCTION ARGUMENTS. |
| A user defined callback that transforms each sample. For example:: |
| |
| transform=lambda data, label: (data.astype(np.float32)/255, label) |
| |
| """ |
| def __init__(self, filename, flag=1, transform=None): |
| super(ImageRecordDataset, self).__init__(filename) |
| if transform is not None: |
| raise DeprecationWarning( |
| 'Directly apply transform to dataset is deprecated. ' |
| 'Please use dataset.transform() or dataset.transform_first() instead...') |
| self._flag = flag |
| self._transform = transform |
| |
| def __getitem__(self, idx): |
| record = super(ImageRecordDataset, self).__getitem__(idx) |
| header, img = recordio.unpack(record) |
| if self._transform is not None: |
| return self._transform(image.imdecode(img, self._flag), header.label) |
| return image.imdecode(img, self._flag), header.label |
| |
| def __mx_handle__(self): |
| from .._internal import ImageRecordFileDataset as _ImageRecordFileDataset |
| return _ImageRecordFileDataset(rec_file=self.filename, idx_file=self.idx_file, |
| flag=self._flag) |
| |
| |
| class ImageFolderDataset(dataset.Dataset): |
| """A dataset for loading image files stored in a folder structure. |
| |
| like:: |
| |
| root/car/0001.jpg |
| root/car/xxxa.jpg |
| root/car/yyyb.jpg |
| root/bus/123.jpg |
| root/bus/023.jpg |
| root/bus/wwww.jpg |
| |
| Parameters |
| ---------- |
| root : str |
| Path to root directory. |
| flag : {0, 1}, default 1 |
| If 0, always convert loaded images to greyscale (1 channel). |
| If 1, always convert loaded images to colored (3 channels). |
| transform : callable, default None |
| DEPRECATED FUNCTION ARGUMENTS. |
| A function that takes data and label and transforms them:: |
| |
| transform = lambda data, label: (data.astype(np.float32)/255, label) |
| |
| Attributes |
| ---------- |
| synsets : list |
| List of class names. `synsets[i]` is the name for the integer label `i` |
| items : list of tuples |
| List of all images in (filename, label) pairs. |
| """ |
| def __init__(self, root, flag=1, transform=None): |
| self._root = os.path.expanduser(root) |
| self._flag = flag |
| if transform is not None: |
| raise DeprecationWarning( |
| 'Directly apply transform to dataset is deprecated. ' |
| 'Please use dataset.transform() or dataset.transform_first() instead...') |
| self._transform = transform |
| self._exts = ['.jpg', '.jpeg', '.png'] |
| self._list_images(self._root) |
| self._handle = None |
| |
| def _list_images(self, root): |
| self.synsets = [] |
| self.items = [] |
| |
| for folder in sorted(os.listdir(root)): |
| path = os.path.join(root, folder) |
| if not os.path.isdir(path): |
| warnings.warn(f'Ignoring {path}, which is not a directory.', stacklevel=3) |
| continue |
| label = len(self.synsets) |
| self.synsets.append(folder) |
| for filename in sorted(os.listdir(path)): |
| filename = os.path.join(path, filename) |
| ext = os.path.splitext(filename)[1] |
| if ext.lower() not in self._exts: |
| warnings.warn(f'Ignoring {filename} of type {ext}. Only support {", ".join(self._exts)}') |
| continue |
| self.items.append((filename, label)) |
| |
| def __getitem__(self, idx): |
| img = image.imread(self.items[idx][0], self._flag) |
| label = self.items[idx][1] |
| if self._transform is not None: |
| return self._transform(img, label) |
| return img, label |
| |
| def __len__(self): |
| return len(self.items) |
| |
| def __mx_handle__(self): |
| if self._handle is None: |
| from .._internal import ImageSequenceDataset, NDArrayDataset, GroupDataset |
| path_sep = '|' |
| im_names = path_sep.join([x[0] for x in self.items]) |
| label = default_array([x[1] for x in self.items]) |
| self._handle = GroupDataset(datasets=( |
| ImageSequenceDataset(img_list=im_names, path_sep=path_sep, flag=self._flag), |
| NDArrayDataset(arr=label))) |
| return self._handle |
| |
| |
| class ImageListDataset(dataset.Dataset): |
| """A dataset for loading image files specified by a list of entries. |
| |
| like:: |
| |
| # if written to text file *.lst |
| 0\t0\troot/car/0001.jpg |
| 1\t0\troot/car/xxxa.jpg |
| 2\t0\troot/car/yyyb.jpg |
| 3\t1\troot/bus/123.jpg |
| 4\t1\troot/bus/023.jpg |
| 5\t1\troot/bus/wwww.jpg |
| |
| # if as a pure list, each item is a list [imagelabel: float or list of float, imgpath] |
| [[0, root/car/0001.jpg] |
| [0, root/car/xxxa.jpg] |
| [0, root/car/yyyb.jpg] |
| [1, root/bus/123.jpg] |
| [1, root/bus/023.jpg] |
| [1, root/bus/wwww.jpg]] |
| |
| Parameters |
| ---------- |
| root : str |
| Path to root directory. |
| imglist : str or list |
| Specify the path of imglist file or a list directly |
| flag : {0, 1}, default 1 |
| If 0, always convert loaded images to greyscale (1 channel). |
| If 1, always convert loaded images to colored (3 channels). |
| |
| Attributes |
| ---------- |
| items : list of tuples |
| List of all images in (filename, label) pairs. |
| """ |
| def __init__(self, root='.', imglist=None, flag=1): |
| self._root = os.path.expanduser(root) |
| self._flag = flag |
| self._imglist = {} |
| self._imgkeys = [] |
| self._handle = None |
| array_fn = _mx_np.array if is_np_array() else nd.array |
| if isinstance(imglist, str): |
| # read from file |
| fname = os.path.join(self._root, imglist) |
| with open(fname, 'rt') as fin: |
| for line in iter(fin.readline, ''): |
| line = line.strip().split('\t') |
| label = array_fn(line[1:-1]) |
| key = int(line[0]) |
| self._imglist[key] = (label, os.path.join(self._root, line[-1])) |
| self._imgkeys.append(key) |
| elif isinstance(imglist, list): |
| index = 1 |
| for img in imglist: |
| key = str(index) |
| index += 1 |
| if len(img) > 2: |
| label = array_fn(img[:-1]) |
| elif isinstance(img[0], numeric_types): |
| label = array_fn([img[0]]) |
| else: |
| label = array_fn(img[0]) |
| assert isinstance(img[-1], str) |
| self._imglist[key] = (label, os.path.join(self._root, img[-1])) |
| self._imgkeys.append(key) |
| else: |
| raise ValueError( |
| "imglist must be filename or list of valid entries, given {}".format( |
| type(imglist))) |
| |
| def __getitem__(self, idx): |
| key = self._imgkeys[idx] |
| img = image.imread(self._imglist[key][1], self._flag) |
| label = self._imglist[key][0] |
| return img, label |
| |
| def __len__(self): |
| return len(self._imgkeys) |
| |
| def __mx_handle__(self): |
| if self._handle is None: |
| from .._internal import ImageSequenceDataset, NDArrayDataset, GroupDataset |
| path_sep = '|' |
| im_names = path_sep.join([self._imglist[x][1] for x in self._imgkeys]) |
| label = default_array(np.array([self._imglist[x][0].asnumpy() for x in self._imgkeys])) |
| self._handle = GroupDataset(datasets=( |
| ImageSequenceDataset(img_list=im_names, path_sep=path_sep, flag=self._flag), |
| NDArrayDataset(arr=label))) |
| return self._handle |
