|
|
|
@ -2166,6 +2166,8 @@ class MapDataset(DatasetOp):
|
|
|
|
|
new_op.operations = self.operations
|
|
|
|
|
new_op.dataset_size = self.dataset_size
|
|
|
|
|
new_op.callbacks = self.callbacks
|
|
|
|
|
if hasattr(self, "__total_batch__"):
|
|
|
|
|
new_op.__total_batch__ = self.__total_batch__
|
|
|
|
|
return new_op
|
|
|
|
|
|
|
|
|
|
# Iterator bootstrap will be called on iterator construction.
|
|
|
|
@ -3640,6 +3642,8 @@ class GeneratorDataset(MappableDataset):
|
|
|
|
|
new_op.num_samples = copy.deepcopy(self.num_samples, memodict)
|
|
|
|
|
new_op.dataset_size = self.dataset_size
|
|
|
|
|
new_op.sampler = copy.deepcopy(self.sampler)
|
|
|
|
|
if hasattr(self, "__total_batch__"):
|
|
|
|
|
new_op.__total_batch__ = self.__total_batch__
|
|
|
|
|
if new_op.sampler is not None and hasattr(self.source, "__getitem__"):
|
|
|
|
|
if isinstance(new_op.sampler, (samplers.SequentialSampler, samplers.DistributedSampler,
|
|
|
|
|
samplers.RandomSampler, samplers.SubsetRandomSampler,
|
|
|
|
@ -5705,10 +5709,11 @@ class NumpySlicesDataset(GeneratorDataset):
|
|
|
|
|
|
|
|
|
|
Args:
|
|
|
|
|
data (Union[list, tuple, dict]) Input of given data. Supported data types include: list, tuple, dict and other
|
|
|
|
|
NumPy formats. Input data will be sliced along the first dimension and generate additional rows.
|
|
|
|
|
Large data is not recommended to be loaded in this way as data is loading into memory.
|
|
|
|
|
NumPy formats. Input data will be sliced along the first dimension and generate additional rows, if input is
|
|
|
|
|
list, there will be one column in each row, otherwise there tends to be multi columns. Large data is not
|
|
|
|
|
recommended to be loaded in this way as data is loading into memory.
|
|
|
|
|
column_names (list[str], optional): List of column names of the dataset (default=None). If column_names is not
|
|
|
|
|
provided, when data is dict, column_names will be its keys, otherwise it will be like column_1, column_2 ...
|
|
|
|
|
provided, when data is dict, column_names will be its keys, otherwise it will be like column_0, column_1 ...
|
|
|
|
|
num_samples (int, optional): The number of samples to be included in the dataset (default=None, all images).
|
|
|
|
|
num_parallel_workers (int, optional): Number of subprocesses used to fetch the dataset in parallel (default=1).
|
|
|
|
|
shuffle (bool, optional): Whether or not to perform shuffle on the dataset. Random accessible input is required.
|
|
|
|
|