|
| 1 | +import os |
| 2 | + |
| 3 | +import albumentations as A |
| 4 | +import numpy as np |
| 5 | +import torch |
| 6 | +import torchvision.transforms as T |
| 7 | +from albumentations.pytorch import ToTensorV2 |
| 8 | +from PIL import Image |
| 9 | +from timm.data.constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD |
| 10 | +from timm.data.transforms import str_to_pil_interp |
| 11 | +from torch.utils.data import Dataset |
| 12 | +from torchvision import datasets |
| 13 | + |
| 14 | + |
| 15 | +def build_cls_dataset(config, logger): |
| 16 | + train_transforms = T.Compose( |
| 17 | + [ |
| 18 | + T.Lambda(lambda img: img.convert("RGB") if img.mode != "RGB" else img), |
| 19 | + T.Resize( |
| 20 | + (config.data.img_size, config.data.img_size), |
| 21 | + interpolation=str_to_pil_interp(config.data.interpolation), |
| 22 | + ), |
| 23 | + # T.RandomHorizontalFlip(), |
| 24 | + # A.RandomRotate90(p=0.5), |
| 25 | + # A.HorizontalFlip(p=0.5), |
| 26 | + # A.VerticalFlip(p=0.5), |
| 27 | + T.ToTensor(), |
| 28 | + T.Normalize( |
| 29 | + mean=torch.tensor(IMAGENET_DEFAULT_MEAN), |
| 30 | + std=torch.tensor(IMAGENET_DEFAULT_STD), |
| 31 | + ), |
| 32 | + ] |
| 33 | + ) |
| 34 | + val_transforms = T.Compose( |
| 35 | + [ |
| 36 | + T.Resize( |
| 37 | + (config.data.img_size, config.data.img_size), |
| 38 | + interpolation=str_to_pil_interp(config.data.interpolation), |
| 39 | + ), |
| 40 | + T.ToTensor(), |
| 41 | + T.Normalize( |
| 42 | + mean=torch.tensor(IMAGENET_DEFAULT_MEAN), |
| 43 | + std=torch.tensor(IMAGENET_DEFAULT_STD), |
| 44 | + ), |
| 45 | + ] |
| 46 | + ) |
| 47 | + if config.data.type == "cls_imagenet": |
| 48 | + data_path = config.data.path |
| 49 | + dataset_train = datasets.ImageFolder( |
| 50 | + os.path.join(data_path.root, data_path.split.train), |
| 51 | + transform=train_transforms, |
| 52 | + ) |
| 53 | + dataset_val = datasets.ImageFolder( |
| 54 | + os.path.join(data_path.root, data_path.split.val), transform=val_transforms |
| 55 | + ) |
| 56 | + dataset_test = datasets.ImageFolder( |
| 57 | + os.path.join(data_path.root, data_path.split.test), transform=val_transforms |
| 58 | + ) |
| 59 | + else: |
| 60 | + raise NotImplementedError("We only support ImageNet Now.") |
| 61 | + |
| 62 | + logger.info( |
| 63 | + f"Build [Cls] dataset: train images = {len(dataset_train)}, val images = {len(dataset_val)}, test images = {len(dataset_test)}" |
| 64 | + ) |
| 65 | + return dataset_train, dataset_val, dataset_test |
| 66 | + |
| 67 | + |
| 68 | +def build_seg_dataset(config, logger): |
| 69 | + train_transforms = A.Compose( |
| 70 | + [ |
| 71 | + A.Resize(width=config.data.img_size, height=config.data.img_size), |
| 72 | + A.RandomRotate90(p=0.5), |
| 73 | + A.HorizontalFlip(p=0.5), |
| 74 | + A.VerticalFlip(p=0.5), |
| 75 | + # A.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, p=0.5), |
| 76 | + A.ToFloat(max_value=255), |
| 77 | + A.Normalize( |
| 78 | + mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), max_pixel_value=1 |
| 79 | + ), |
| 80 | + ToTensorV2(), |
| 81 | + ] |
| 82 | + ) |
| 83 | + val_transforms = A.Compose( |
| 84 | + [ |
| 85 | + A.Resize(width=config.data.img_size, height=config.data.img_size), |
| 86 | + A.ToFloat(max_value=255), |
| 87 | + A.Normalize( |
| 88 | + mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225), max_pixel_value=1 |
| 89 | + ), |
| 90 | + ToTensorV2(), |
| 91 | + ] |
| 92 | + ) |
| 93 | + Dataset_class = eval(config.data.type + "Dataset") |
| 94 | + |
| 95 | + dataset_train = Dataset_class(config.data, "train", train_transforms) |
| 96 | + dataset_val = Dataset_class(config.data, "val", val_transforms) |
| 97 | + dataset_test = Dataset_class(config.data, "test", val_transforms) |
| 98 | + logger.info( |
| 99 | + f"Build [Seg] dataset: train images = {len(dataset_train)}, val images = {len(dataset_val)}, test images = {len(dataset_test)}" |
| 100 | + ) |
| 101 | + |
| 102 | + return dataset_train, dataset_val, dataset_test |
| 103 | + |
| 104 | + |
| 105 | +class SegBaseDataset(Dataset): |
| 106 | + def __init__(self, DataConfig, stage, transforms=None): |
| 107 | + super().__init__() |
| 108 | + data_folder = os.path.join(DataConfig.path.root, DataConfig.path.split[stage]) |
| 109 | + self.num_classes = DataConfig.num_classes |
| 110 | + self.update_datalist(data_folder) |
| 111 | + self.transforms = transforms |
| 112 | + |
| 113 | + def __getitem__(self, index): |
| 114 | + image_file = self.image_list[index] |
| 115 | + mask_file = self.mask_list[index] |
| 116 | + image = np.array(Image.open(image_file).convert("RGB")) |
| 117 | + if self.num_classes == 2: |
| 118 | + mask = np.array(Image.open(mask_file).convert("1")).astype(int) |
| 119 | + else: |
| 120 | + mask = np.array(Image.open(mask_file)).astype(int) |
| 121 | + if self.transforms is not None: |
| 122 | + image_mask = self.transforms(image=image, mask=mask) |
| 123 | + image_mask["img_path"] = image_file |
| 124 | + image_mask["mask_path"] = mask_file |
| 125 | + return image_mask |
| 126 | + |
| 127 | + def update_datalist(self, folder): |
| 128 | + image_path = os.path.join(folder, "image") |
| 129 | + mask_path = os.path.join(folder, "mask") |
| 130 | + # find all file in the folder and subfolder |
| 131 | + filenames = [] |
| 132 | + for root, dirs, files in os.walk(image_path): |
| 133 | + for file in files: |
| 134 | + filenames.append(os.path.join(root, file)) |
| 135 | + |
| 136 | + # filenames = os.listdir(image_path) |
| 137 | + self.image_list = filenames |
| 138 | + self.mask_list = [i.replace(image_path, mask_path) for i in filenames] |
| 139 | + |
| 140 | + def __len__(self): |
| 141 | + return len(self.image_list) |
| 142 | + |
| 143 | + |
| 144 | +class SegVocDataset(Dataset): |
| 145 | + def __init__(self, DataConfig, stage, transforms=None): |
| 146 | + super().__init__() |
| 147 | + self.update_datalist(DataConfig.path.root, stage, DataConfig.path.image_type) |
| 148 | + self.transforms = transforms |
| 149 | + |
| 150 | + def __getitem__(self, index): |
| 151 | + image_file = self.image_list[index] |
| 152 | + mask_file = self.mask_list[index] |
| 153 | + image = np.array(Image.open(image_file).convert("RGB")) |
| 154 | + mask = np.array(Image.open(mask_file).convert("1")).astype(int) |
| 155 | + if self.transforms is not None: |
| 156 | + image_mask = self.transforms( |
| 157 | + image=image, mask=mask, img_path=image_file, mask_path=mask_file |
| 158 | + ) |
| 159 | + return image_mask |
| 160 | + |
| 161 | + def update_datalist(self, root, stage, image_type): |
| 162 | + filenames = np.loadtxt( |
| 163 | + os.path.join(root, "ImageSets", stage + ".txt"), dtype=str |
| 164 | + ) |
| 165 | + image_filenames = [i + "." + image_type for i in filenames] |
| 166 | + mask_filenames = [i + ".png" for i in filenames] |
| 167 | + self.image_list = [os.path.join(root, "JPEGImages", i) for i in image_filenames] |
| 168 | + self.mask_list = [ |
| 169 | + os.path.join(root, "SegmentationClass", i) for i in mask_filenames |
| 170 | + ] |
| 171 | + |
| 172 | + def __len__(self): |
| 173 | + return len(self.image_list) |
0 commit comments