dataset_statistics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5ff82a10",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import glob\n",
    "import torch\n",
    "import numpy as np\n",
    "import multiprocessing as mp\n",
    "from skimage import io\n",
    "from skimage.color import rgb2gray\n",
    "from tqdm import tqdm\n",
    "\n",
    "# mp.set_start_method('fork', force=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a78fb460",
   "metadata": {},
   "outputs": [],
   "source": [
    "def _backbone_mp_image_stas(this_img_path, image_mean_ls, image_std_ls, image_gray_mean_ls, image_gray_std_ls, mp_progress, task_size, mp_lock):\n",
    "    \"\"\"\n",
    "    Multiprocessng backbone function to get image statistics.\n",
    "    This is to alleviate the IO-bound performance bottleneck.\n",
    "    \"\"\"\n",
    "    image = io.imread(this_img_path)\n",
    "    if len(image.shape) == 3 and image.shape[-1] == 4:\n",
    "        # RGBA to RGB for Cesium dataset\n",
    "        image = image[:, :, :3]\n",
    "    \n",
    "    # Warning: the order of each entry might not match the overall input argument list because of inter-thread synchronization overhead.\n",
    "    image_mean_ls.append(np.mean(image.reshape(-1, 3), axis=0))\n",
    "    image_std_ls.append(np.std(image.reshape(-1, 3), axis=0))\n",
    "\n",
    "    image_gray = rgb2gray(image)\n",
    "    image_gray_mean_ls.append(np.mean(image_gray))\n",
    "    image_gray_std_ls.append(np.std(image_gray))\n",
    "    \n",
    "    with mp_lock:\n",
    "        mp_progress.value += 1\n",
    "        print(\"\\rProgress: {:d} / {:d}\".format(mp_progress.value, task_size), end=\" \", flush=True)\n",
    "        \n",
    "        \n",
    "def _backbone_mp_label_stas(this_label_path, label_mean_ls, label_std_ls, mp_progress, task_size, mp_lock, surface_normal=False):\n",
    "    \"\"\"\n",
    "    Multiprocessng backbone function to get 3D label statistics.\n",
    "    This is to alleviate the IO-bound performance bottleneck.\n",
    "    \"\"\"\n",
    "    label = torch.load(this_label_path)  # the .dat file\n",
    "    if len(label.shape) == 2:\n",
    "        # data in [H, W] such as depth\n",
    "        label = label.unsqueeze(0)  # [1, H, W]\n",
    "    elif len(label.shape) == 3:\n",
    "        # data in [X, H, W] such as coord (X=3)\n",
    "        pass\n",
    "    else:\n",
    "        raise NotImplementedError\n",
    "\n",
    "    # Warning: the order of each entry might not match the overall input argument list because of inter-thread synchronization overhead.\n",
    "    label = label.reshape(label.size(0), -1)  # [X, N]\n",
    "    valid_label = label[0, :] != -1  # [N], mask\n",
    "    label = label[:, valid_label]  # [X', N]\n",
    "    \n",
    "    if surface_normal:\n",
    "        label = xyz2ae(label.unsqueeze(0)).squeeze(0) # [2, N]\n",
    "    this_mean = label.reshape(label.size(0), -1).mean(dim=1).numpy()  # [X] numpy array\n",
    "    this_std = label.reshape(label.size(0), -1).std(dim=1).numpy()  # [X] numpy array\n",
    "    label_mean_ls.append(this_mean)\n",
    "    label_std_ls.append(this_std)\n",
    "    \n",
    "    with mp_lock:\n",
    "        mp_progress.value += 1\n",
    "        print(\"\\rProgress: {:d} / {:d}\".format(mp_progress.value, task_size), end=\" \", flush=True)\n",
    "\n",
    "        \n",
    "def xyz2ae(xyz: torch.Tensor) -> torch.Tensor:\n",
    "    \"\"\"\n",
    "    Turn normalized direction vector into azimuth and elevation.\n",
    "    @param xyz  [B, 3, *] tensor of normalized direction vector.\n",
    "    @return:    [B, 2, *] tensor of azimuth and elevation in radian.\n",
    "    \"\"\"\n",
    "\n",
    "    # azimuth = arctan2(y, x), range [-pi, pi]\n",
    "    azimuth = torch.atan2(xyz[:, 1], xyz[:, 0])  # [B, *]\n",
    "\n",
    "    # elevation = arctan2(z, sqrt(x**2 + y**2)), range [-pi, pi]\n",
    "    elevation = torch.atan2(xyz[:, 2], torch.norm(xyz[:, 0:2], dim=1, p=2))  # [B, *]\n",
    "\n",
    "    return torch.stack([azimuth, elevation], dim=1)  # [B, 2, *]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "3d033662",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Multiprocessing preparation...: 100%|████████████████████████████████████████████████████████████████████████████████████| 13500/13500 [00:00<00:00, 1682557.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 13500 / 13500                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        Dataset urbanscape statistics:\n",
      "RGB mean: [0.42451202 0.4374828  0.3836407 ], RGB std: [0.18234482 0.17014659 0.18538321]\n",
      "grayscale mean: 0.4308444925493635, grayscale std: 0.17239308289661812\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Obtain RGB and grayscale mean & std statistics respectively for urbanscape dataset.\n",
    "\"\"\"\n",
    "\n",
    "dataset_nms = ['urbanscape']\n",
    "PROJ_DIR = os.path.abspath(os.path.join(os.getcwd()))\n",
    "\n",
    "mp_manager = mp.Manager()\n",
    "mp_lock = mp_manager.Lock()\n",
    "mp_progress = mp_manager.Value('i', 0)\n",
    "image_mean_ls = mp_manager.list()\n",
    "image_std_ls = mp_manager.list()\n",
    "image_gray_mean_ls = mp_manager.list()\n",
    "image_gray_std_ls = mp_manager.list()\n",
    "\n",
    "\n",
    "for nm in dataset_nms:\n",
    "    train_sim_rgb_dir = os.path.join(PROJ_DIR, 'datasets', nm, 'train_sim', 'rgb')\n",
    "    train_sim_rgb_ls = sorted(glob.glob(os.path.join(train_sim_rgb_dir, '*.png')))\n",
    "\n",
    "    mp_args_ls = []\n",
    "    for i in tqdm(range(len(train_sim_rgb_ls)), desc=\"Multiprocessing preparation...\"):\n",
    "        this_img_path = train_sim_rgb_ls[i]\n",
    "        mp_args_ls.append((this_img_path, image_mean_ls, image_std_ls, image_gray_mean_ls, image_gray_std_ls, mp_progress, len(train_sim_rgb_ls), mp_lock))\n",
    "        \n",
    "    with mp.Pool() as pool:\n",
    "        pool.starmap(_backbone_mp_image_stas, mp_args_ls, chunksize=1)\n",
    "        \n",
    "    image_mean_ls = np.array(image_mean_ls) / 255.0\n",
    "    mean_overall = np.mean(image_mean_ls, axis=0)\n",
    "    \n",
    "    image_std_ls = np.array(image_std_ls) / 255.0\n",
    "    std_overall = np.mean(image_std_ls, axis=0)\n",
    "    \n",
    "    # skimage rgb2gray already stores the pixel in [0, 1] range, no need to divide by 255\n",
    "    image_gray_mean_ls = np.array(image_gray_mean_ls)\n",
    "    gray_mean_overall = np.mean(image_gray_mean_ls, axis=0)\n",
    "    \n",
    "    image_gray_std_ls = np.array(image_gray_std_ls)\n",
    "    gray_std_overall = np.mean(image_gray_std_ls, axis=0)\n",
    "    \n",
    "    print(\"Dataset {:s} statistics:\\nRGB mean: {}, RGB std: {}\".format(nm, mean_overall, std_overall))\n",
    "    print(\"grayscale mean: {}, grayscale std: {}\".format(gray_mean_overall, gray_std_overall))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e4869481-e8b6-4730-904e-c8b842dc727b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Multiprocessing preparation...: 100%|████████████████████████████████████████████████████████████████████████████████████| 13500/13500 [00:00<00:00, 1444135.38it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 13500 / 13500                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        Dataset urbanscape statistics:\n",
      "init mean: [-29.341667 184.16975   91.95693 ], init std: [46.37544  63.712765 45.36746 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Multiprocessing preparation...: 100%|█████████████████████████████████████████████████████████████████████████████████████| 13500/13500 [00:00<00:00, 199285.90it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 13500 / 13500                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        Dataset urbanscape statistics:\n",
      "depth mean: [136.23961], depth std: [43.091194]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Multiprocessing preparation...: 100%|████████████████████████████████████████████████████████████████████████████████████| 13500/13500 [00:00<00:00, 1595376.54it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 13500 / 13500                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        Dataset urbanscape statistics:\n",
      "normal mean: [-1.0454447 -0.985821 ], normal std: [1.2661825  0.26884857]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Obtain 3D label mean & std statistics respectively for urbanscape dataset.\n",
    "\"\"\"\n",
    "\n",
    "dataset_nms = ['urbanscape']\n",
    "PROJ_DIR = os.path.abspath(os.path.join(os.getcwd()))\n",
    "\n",
    "mp_manager = mp.Manager()\n",
    "mp_lock = mp_manager.Lock()\n",
    "mp_progress = mp_manager.Value('i', 0)\n",
    "label_mean_ls = mp_manager.list()\n",
    "label_std_ls = mp_manager.list()\n",
    "\n",
    "label_nms = ['init', 'depth', 'normal']\n",
    "\n",
    "for label_nm in label_nms:\n",
    "    for nm in dataset_nms:\n",
    "        with mp_lock:\n",
    "            mp_progress.value = 0\n",
    "            label_mean_ls = mp_manager.list()\n",
    "            label_std_ls = mp_manager.list()\n",
    "        train_sim_label_dir = os.path.join(PROJ_DIR, 'datasets', nm, 'train_sim', label_nm)\n",
    "        train_sim_label_ls = sorted(glob.glob(os.path.join(train_sim_label_dir, '*.dat')))\n",
    "\n",
    "        mp_args_ls = []\n",
    "        for i in tqdm(range(len(train_sim_label_ls)), desc=\"Multiprocessing preparation...\"):\n",
    "            this_label_path = train_sim_label_ls[i]\n",
    "            mp_args_ls.append((this_label_path, label_mean_ls, label_std_ls, mp_progress, len(train_sim_label_ls), mp_lock, label_nm == 'normal'))\n",
    "\n",
    "        with mp.Pool() as pool:\n",
    "            pool.starmap(_backbone_mp_label_stas, mp_args_ls, chunksize=1)\n",
    "\n",
    "        label_mean_ls = np.array(label_mean_ls)  # [N, X]\n",
    "        mean_overall = np.mean(label_mean_ls, axis=0)  # [X]\n",
    "\n",
    "        label_std_ls = np.array(label_std_ls)  # [N, X]\n",
    "        std_overall = np.mean(label_std_ls, axis=0)  # [X]\n",
    "\n",
    "        print(\"Dataset {:s} statistics:\\n{:s} mean: {}, {:s} std: {}\".format(nm, label_nm, mean_overall, label_nm, std_overall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "ce2c5589",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Multiprocessing preparation...: 100%|████████████████████████████████████████████████████████████████████████████████████| 27000/27000 [00:00<00:00, 1479800.96it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 27000 / 27000                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    Dataset naturescape statistics:\n",
      "RGB mean: [0.36358474 0.43309037 0.29565111], RGB std: [0.13830253 0.1457159  0.11469524]\n",
      "grayscale mean: 0.4084110506147726, grayscale std: 0.1403811662465024\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Obtain RGB and grayscale mean & std statistics respectively for naturescape dataset.\n",
    "\"\"\"\n",
    "\n",
    "dataset_nms = ['naturescape']\n",
    "PROJ_DIR = os.path.abspath(os.path.join(os.getcwd()))\n",
    "\n",
    "mp_manager = mp.Manager()\n",
    "mp_lock = mp_manager.Lock()\n",
    "mp_progress = mp_manager.Value('i', 0)\n",
    "image_mean_ls = mp_manager.list()\n",
    "image_std_ls = mp_manager.list()\n",
    "image_gray_mean_ls = mp_manager.list()\n",
    "image_gray_std_ls = mp_manager.list()\n",
    "\n",
    "\n",
    "for nm in dataset_nms:\n",
    "    train_sim_rgb_dir = os.path.join(PROJ_DIR, 'datasets', nm, 'train_sim', 'rgb')\n",
    "    train_sim_rgb_ls = sorted(glob.glob(os.path.join(train_sim_rgb_dir, '*.png')))\n",
    "\n",
    "    mp_args_ls = []\n",
    "    for i in tqdm(range(len(train_sim_rgb_ls)), desc=\"Multiprocessing preparation...\"):\n",
    "        this_img_path = train_sim_rgb_ls[i]\n",
    "        mp_args_ls.append((this_img_path, image_mean_ls, image_std_ls, image_gray_mean_ls, image_gray_std_ls, mp_progress, len(train_sim_rgb_ls), mp_lock))\n",
    "        \n",
    "    with mp.Pool() as pool:\n",
    "        pool.starmap(_backbone_mp_image_stas, mp_args_ls, chunksize=1)\n",
    "        \n",
    "    image_mean_ls = np.array(image_mean_ls) / 255.0\n",
    "    mean_overall = np.mean(image_mean_ls, axis=0)\n",
    "    \n",
    "    image_std_ls = np.array(image_std_ls) / 255.0\n",
    "    std_overall = np.mean(image_std_ls, axis=0)\n",
    "    \n",
    "    # skimage rgb2gray already stores the pixel in [0, 1] range, no need to divide by 255\n",
    "    image_gray_mean_ls = np.array(image_gray_mean_ls)\n",
    "    gray_mean_overall = np.mean(image_gray_mean_ls, axis=0)\n",
    "    \n",
    "    image_gray_std_ls = np.array(image_gray_std_ls)\n",
    "    gray_std_overall = np.mean(image_gray_std_ls, axis=0)\n",
    "    \n",
    "    print(\"Dataset {:s} statistics:\\nRGB mean: {}, RGB std: {}\".format(nm, mean_overall, std_overall))\n",
    "    print(\"grayscale mean: {}, grayscale std: {}\".format(gray_mean_overall, gray_std_overall))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "029329e8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Multiprocessing preparation...: 100%|████████████████████████████████████████████████████████████████████████████████████| 27000/27000 [00:00<00:00, 1557655.23it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 27000 / 27000                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    Dataset naturescape statistics:\n",
      "init mean: [-455.9359   417.50745  520.3076 ], init std: [ 61.656967 100.36582   86.24486 ]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Multiprocessing preparation...: 100%|████████████████████████████████████████████████████████████████████████████████████| 27000/27000 [00:00<00:00, 1579202.74it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 27000 / 27000                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    Dataset naturescape statistics:\n",
      "depth mean: [241.47272], depth std: [58.832935]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Obtain 3D label mean & std statistics respectively for naturescape dataset.\n",
    "\"\"\"\n",
    "\n",
    "dataset_nms = ['naturescape']\n",
    "PROJ_DIR = os.path.abspath(os.path.join(os.getcwd()))\n",
    "\n",
    "mp_manager = mp.Manager()\n",
    "mp_lock = mp_manager.Lock()\n",
    "mp_progress = mp_manager.Value('i', 0)\n",
    "label_mean_ls = mp_manager.list()\n",
    "label_std_ls = mp_manager.list()\n",
    "\n",
    "label_nms = ['init', 'depth']\n",
    "\n",
    "for label_nm in label_nms:\n",
    "    for nm in dataset_nms:\n",
    "        with mp_lock:\n",
    "            mp_progress.value = 0\n",
    "            label_mean_ls = mp_manager.list()\n",
    "            label_std_ls = mp_manager.list()\n",
    "        train_sim_label_dir = os.path.join(PROJ_DIR, 'datasets', nm, 'train_sim', label_nm)\n",
    "        train_sim_label_ls = sorted(glob.glob(os.path.join(train_sim_label_dir, '*.dat')))\n",
    "\n",
    "        mp_args_ls = []\n",
    "        for i in tqdm(range(len(train_sim_label_ls)), desc=\"Multiprocessing preparation...\"):\n",
    "            this_label_path = train_sim_label_ls[i]\n",
    "            mp_args_ls.append((this_label_path, label_mean_ls, label_std_ls, mp_progress, len(train_sim_label_ls), mp_lock, label_nm == 'normal'))\n",
    "\n",
    "        with mp.Pool() as pool:\n",
    "            pool.starmap(_backbone_mp_label_stas, mp_args_ls, chunksize=1)\n",
    "\n",
    "        label_mean_ls = np.array(label_mean_ls)  # [N, X]\n",
    "        mean_overall = np.mean(label_mean_ls, axis=0)  # [X]\n",
    "\n",
    "        label_std_ls = np.array(label_std_ls)  # [N, X]\n",
    "        std_overall = np.mean(label_std_ls, axis=0)  # [X]\n",
    "\n",
    "        print(\"Dataset {:s} statistics:\\n{:s} mean: {}, {:s} std: {}\".format(nm, label_nm, mean_overall, label_nm, std_overall))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "35602960-c2b8-49fe-8799-b8b4e283ceff",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Multiprocessing preparation...: 100%|█████████████████████████████████████████████████████████████████████████████████████| 27000/27000 [00:00<00:00, 199465.97it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress: 27000 / 27000                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    Dataset naturescape statistics:\n",
      "normal mean: [-0.79427886 -0.9986142 ], normal std: [1.2973572  0.26808947]\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "Obtain 3D label mean & std statistics respectively for naturescape dataset.\n",
    "\"\"\"\n",
    "# separate normal statistics to be bug free\n",
    "dataset_nms = ['naturescape']\n",
    "PROJ_DIR = os.path.abspath(os.path.join(os.getcwd()))\n",
    "\n",
    "mp_manager = mp.Manager()\n",
    "mp_lock = mp_manager.Lock()\n",
    "mp_progress = mp_manager.Value('i', 0)\n",
    "label_mean_ls = mp_manager.list()\n",
    "label_std_ls = mp_manager.list()\n",
    "\n",
    "label_nms = ['normal']\n",
    "\n",
    "for label_nm in label_nms:\n",
    "    for nm in dataset_nms:\n",
    "        with mp_lock:\n",
    "            mp_progress.value = 0\n",
    "            label_mean_ls = mp_manager.list()\n",
    "            label_std_ls = mp_manager.list()\n",
    "        train_sim_label_dir = os.path.join(PROJ_DIR, 'datasets', nm, 'train_sim', label_nm)\n",
    "        train_sim_label_ls = sorted(glob.glob(os.path.join(train_sim_label_dir, '*.dat')))\n",
    "\n",
    "        mp_args_ls = []\n",
    "        for i in tqdm(range(len(train_sim_label_ls)), desc=\"Multiprocessing preparation...\"):\n",
    "            this_label_path = train_sim_label_ls[i]\n",
    "            mp_args_ls.append((this_label_path, label_mean_ls, label_std_ls, mp_progress, len(train_sim_label_ls), mp_lock, label_nm == 'normal'))\n",
    "\n",
    "        with mp.Pool() as pool:\n",
    "            pool.starmap(_backbone_mp_label_stas, mp_args_ls, chunksize=1)\n",
    "\n",
    "        label_mean_ls = np.array(label_mean_ls)  # [N, X]\n",
    "        mean_overall = np.mean(label_mean_ls, axis=0)  # [X]\n",
    "\n",
    "        label_std_ls = np.array(label_std_ls)  # [N, X]\n",
    "        std_overall = np.mean(label_std_ls, axis=0)  # [X]\n",
    "\n",
    "        print(\"Dataset {:s} statistics:\\n{:s} mean: {}, {:s} std: {}\".format(nm, label_nm, mean_overall, label_nm, std_overall))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}