{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import visualization\n",
    "import os\n",
    "from gta_math import points_to_homo, ndc_to_view, construct_proj_matrix, view_to_world, construct_view_matrix, ndcs_to_pixels, grid_to_ndc_pcl_linear_view\n",
    "from visualization import load_depth\n",
    "import progressbar\n",
    "from pointcloud_to_voxelmap import pointclouds_to_voxelmap\n",
    "from joblib import Parallel, delayed\n",
    "from configparser import ConfigParser\n",
    "from PIL import Image\n",
    "import pickle\n",
    "from voxelmaps import camera_to_pointcloud, load_scene_db_data, get_main_image_name, scene_to_pointcloud, scene_to_voxelmap, NoMainImageException, scene_to_voxelmap_with_map, get_main_image, ndc_pcl_to_grid_linear_view, generate_frustum_points\n",
    "import voxelmaps\n",
    "import time\n",
    "import csv\n",
    "import random\n",
    "import gta_math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ini_file = \"gta-postprocessing.ini\"\n",
    "visualization.multi_page = False\n",
    "visualization.ini_file = ini_file\n",
    "visualization.use_cache = False\n",
    "\n",
    "conn = visualization.get_connection()\n",
    "cur = conn.cursor()\n",
    "\n",
    "CONFIG = ConfigParser()\n",
    "CONFIG.read(ini_file)\n",
    "in_directory = CONFIG[\"Images\"][\"Tiff\"]\n",
    "out_directory = CONFIG[\"Images\"][\"MlDatasetVoxel\"]\n",
    "out_root_directory = r'D:\\\\'\n",
    "\n",
    "\n",
    "# setting linear samping of voxelmap in view space\n",
    "linear_view_sampling = True # otherwise, it is linear in NDC depth -> hyperbolic in view\n",
    "gta_math.PROJECTING = True\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 8438 scenes\n"
     ]
    }
   ],
   "source": [
    "run_id = 6\n",
    "\n",
    "cur.execute(\"\"\"SELECT DISTINCT scene_id \\\n",
    "  FROM snapshots \\\n",
    "  WHERE run_id = {} \\\n",
    "  \"\"\".format(run_id))\n",
    "\n",
    "scenes = []\n",
    "for row in cur:\n",
    "    res = dict(row)\n",
    "    scenes.append(res)\n",
    "\n",
    "image_names = {}\n",
    "\n",
    "print('There are {} scenes'.format(len(scenes)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### functions for each scene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import time\n",
    "voxelmaps.MAX_DISTANCE = 30 \n",
    "# I will create frustum up to 25 meters, so I take distance to 30 meters\n",
    "# in meters, meaning I care only for point up to 25 meters, \n",
    "# in depth for 100 bins, it corresponds to 25 centimers big voxels\n",
    "\n",
    "points_cache = {}\n",
    "\n",
    "def get_base_name(name):\n",
    "    return os.path.basename(os.path.splitext(name)[0])\n",
    "\n",
    "\n",
    "def convert_tiff(in_directory, out_directory, out_name, name):\n",
    "    # throws OSError\n",
    "    out_format = 'jpg'\n",
    "    outfile = os.path.join(out_directory, \"{}.{}\".format(out_name, out_format))\n",
    "    if os.path.exists(outfile):\n",
    "        return outfile\n",
    "    infile = os.path.join(in_directory, name)\n",
    "    im = Image.open(infile)\n",
    "    im = im.convert(mode=\"RGB\")\n",
    "    # print(\"Generating new format for {} to new file {}\".format(name, out_name))\n",
    "    im.save(outfile)\n",
    "    return outfile\n",
    "\n",
    "def convert_scene_to_ndc_voxelmap(cameras):\n",
    "    # this method is just fucking slow, because of scene_to_voxelmap_with_map\n",
    "    # here I generate points cuboid which will be output of NN.\n",
    "    # these points will be transferred into camera view frustum\n",
    "    # for this frustum, I take voxel values of calculated voxelmap\n",
    "    # I use these voxel values as values for cuboid in NDC, the NN output\n",
    "    #start = time.time()\n",
    "\n",
    "    voxels, values, map_voxel_size, map_obj = scene_to_voxelmap_with_map(cameras, subsampling_size=0.12, main_camera_view=True)\n",
    "\n",
    "    #end = time.time()\n",
    "    #print('scene to whole voxelmap:', end - start)\n",
    "    #start = time.time()\n",
    "\n",
    "    cam = get_main_image(cameras)\n",
    "    proj_matrix = cam['proj_matrix']\n",
    "    view_matrix = cam['view_matrix']\n",
    "\n",
    "    #end = time.time()\n",
    "    #print('getting main cam and stuff:', end - start)\n",
    "    #start = time.time()\n",
    "\n",
    "    x_range = 160  # X output size of NN\n",
    "    y_range = 120  # Y output size of NN\n",
    "    z_range = 100  # Z output size of NN\n",
    "    z_meters_min = 1.5\n",
    "    z_meters_max = 25\n",
    "    positions, z_max, z_min = generate_frustum_points(proj_matrix, x_range, y_range, z_range, z_meters_min, z_meters_max, linear_view_sampling)\n",
    "\n",
    "    #end = time.time()\n",
    "    #print('generating frustum points:', end - start)\n",
    "    #start = time.time()\n",
    "\n",
    "    points_view = ndc_to_view(positions, proj_matrix)\n",
    "    points_world = view_to_world(points_view, view_matrix)\n",
    "\n",
    "    #end = time.time()\n",
    "    #print('obtaining world frustum:', end - start)\n",
    "    #start = time.time()\n",
    "\n",
    "    # here I find corresponding voxels for generated points, by obtaining voxelmap reference\n",
    "    #voxel_values = map_obj.get_voxels(points_world[0:3, :], np.zeros((points_world.shape[1], 1)))\n",
    "    # here in view, because I created voxelmap in view coords\n",
    "    voxel_values = map_obj.get_voxels(points_view[0:3, :], np.zeros((points_view.shape[1], 1)))\n",
    "    occupied_ndc_points = positions[:, voxel_values > 0]\n",
    "    invalid_ndc_points = positions[:, np.isnan(voxel_values)]\n",
    "\n",
    "    #end = time.time()\n",
    "    #print('obtaining voxelmap in NDC:', end - start)\n",
    "    #start = time.time()\n",
    "    # this converts ndc pointcloud to bool grid\n",
    "    occupied_ndc_grid = ndc_pcl_to_grid_linear_view(x_range, y_range, z_range, occupied_ndc_points, proj_matrix, z_meters_min, z_meters_max)\n",
    "    invalid_ndc_grid = ndc_pcl_to_grid_linear_view(x_range, y_range, z_range, invalid_ndc_points, proj_matrix, z_meters_min, z_meters_max)\n",
    "    \n",
    "    # merging obstacles and invalids together, having\n",
    "    # obstacle = 1\n",
    "    # free = 0\n",
    "    # unknown = -1\n",
    "    return occupied_ndc_grid - invalid_ndc_grid\n",
    "\n",
    "\n",
    "def convert_scene_to_img_and_voxelmap(in_directory, out_directory, scene_id):\n",
    "    if 'pbar' in globals() and 'counter' in globals():\n",
    "        global counter\n",
    "        counter += 1\n",
    "        pbar.update(counter)\n",
    "\n",
    "    # start = time.time()\n",
    "    cameras = load_scene_db_data(scene_id)\n",
    "    try:\n",
    "        image_name = get_main_image_name(cameras)\n",
    "        rgb_outfile = convert_tiff(in_directory, out_directory, image_name, image_name+'.tiff')\n",
    "        outfile = os.path.join(out_directory, \"{}.npy\".format(image_name))\n",
    "\n",
    "        image_names[scene_id] = (rgb_outfile, outfile)\n",
    "        if os.path.exists(outfile):\n",
    "            return\n",
    "        \n",
    "        #start = time.time()\n",
    "        occupied_ndc_positions = convert_scene_to_ndc_voxelmap(cameras)\n",
    "        #end = time.time()\n",
    "        #print('time to do whole scene to voxelmap:', end - start)\n",
    "        #start = time.time()\n",
    "        \n",
    "        #occupied_ndc_positions.tofile(outfile)\n",
    "        np.save(outfile, occupied_ndc_positions)\n",
    "        #end = time.time()\n",
    "        #print('time to save:', end - start)\n",
    "        \n",
    "    except (OSError, NoMainImageException) as e:\n",
    "        print('No main image for scene {}, skipping.'.format(scene_id))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### actually running the extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2% 176 |#                                                         |  89.1 B/sC:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:70: RuntimeWarning: invalid value encountered in greater\n",
      "  2% 203 |#                                                         |   2.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 592377fd-866a-4d81-bdb0-4e63ae584bab, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2% 214 |#                                                         |   1.5 B/sC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tifffile\\tifffile.py:2642: RuntimeWarning: py_decodelzw encountered unexpected end of stream\n",
      "  strip = decompress(strip)\n",
      "  2% 230 |#                                                         |   1.0 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene c70fe8d1-e628-483e-a27a-6a2c7b862857, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7% 646 |####                                                      |   0.3 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene e0a57943-f85b-4017-ac1d-6660a789972c, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7% 665 |####                                                      |   0.3 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 1e82b334-4f8f-4a62-b98f-c1964cab55d5, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8% 698 |####                                                      |   0.3 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 9215c874-450b-485a-9c53-f44b0b16e86d, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9% 775 |#####                                                     |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene a8cc8456-1bfb-4f17-867a-8259be7a7aef, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10% 851 |#####                                                     |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene a7fa77e4-8981-49ea-ab97-3a8ff39f85ab, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11% 994 |######                                                    |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 08056d94-ab3d-42cd-8000-f9828444e783, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12% 1033 |######                                                   |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 6f634b01-f5a0-4d65-b2fc-7df58b0d8904, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14% 1187 |########                                                 |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 745ddc7a-3d29-4d21-ac25-46e3351b269c, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14% 1194 |########                                                 |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 1825304b-591c-46c3-8361-8443cb11fd64, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15% 1307 |########                                                 |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene f5e84ca1-ad34-45c5-84c9-92981c124d93, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16% 1368 |#########                                                |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 0514fbe8-e421-4c83-b48a-c83d7404cac0, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16% 1402 |#########                                                |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 410ae84d-e9dc-44b2-b88d-07a397e1b073, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18% 1519 |##########                                               |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 1cec18a3-1847-4bfd-8cb5-6e24bf1a431b, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22% 1859 |############                                             |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 1d0939c2-dc9d-48bb-b088-4c42b219ad7b, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22% 1893 |############                                             |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene cd7334f6-d78f-4cc2-97fc-27971040af36, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25% 2158 |##############                                           |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 0b770141-6775-4f63-9254-cc32226f9eda, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29% 2503 |################                                         |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 7610c8e6-f262-4f3c-89bd-6414f701333c, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30% 2559 |#################                                        |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 92e1fb64-1f97-4e2c-a959-622ccb583ea9, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30% 2605 |#################                                        |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 47dc1c0f-958e-4165-9648-db4b651e97b5, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32% 2741 |##################                                       |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 4d85bc08-8dea-488d-af24-af165f9b25e4, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32% 2754 |##################                                       |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 946fe837-8f42-485e-a82f-00a19760f918, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34% 2922 |###################                                      |   0.2 B/sC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tifffile\\tifffile.py:2577: UserWarning: unpack: string size must be a multiple of element size\n",
      "  warnings.warn(\"unpack: %s\" % e)\n",
      " 35% 3025 |####################                                     |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene c72c1e21-629a-4d90-92c5-0ea06715fead, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38% 3256 |#####################                                    |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene a9707fcc-2468-4620-8be2-6fca9d9b8599, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38% 3263 |######################                                   |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 4e113039-6c55-47b8-8fb3-90ca54b0ae84, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39% 3331 |######################                                   |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene d06bf958-fcb7-4983-b50e-0fc00d8e674e, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44% 3780 |#########################                                |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene ded7f7fc-275b-4760-bcbe-3ee21511d741, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44% 3788 |#########################                                |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 99975c57-a080-4513-aea9-bb1c70dec8c3, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47% 3998 |###########################                              |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene aad2d202-7676-4c38-a589-23561ad417aa, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47% 4048 |###########################                              |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene f2b357b4-0ee8-4f85-8ed9-7df286c5a84d, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50% 4268 |############################                             |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 388309dc-568f-43ab-b2c1-86fb809cc570, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58% 4929 |#################################                        |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 7a7314b0-92a5-402f-a690-2425b48133e6, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58% 4931 |#################################                        |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene be0bb451-894a-40c5-8e86-f9e51b8e73d2, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58% 4948 |#################################                        |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene be512cbb-475d-4a86-b698-09cd0dc4594c, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58% 4963 |#################################                        |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene cdc4513d-105c-4f66-8682-0f5c3e466230, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59% 5060 |##################################                       |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 575da4d5-1d8c-4d07-8c6c-0002868990ea, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63% 5398 |####################################                     |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 73090cd6-266e-49d8-b7cf-4081685587a9, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64% 5471 |####################################                     |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 0e0860ca-6286-4e5f-82ae-50e020246266, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69% 5857 |#######################################                  |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 00b11f74-4351-44e6-a04f-ffa5b0aa9687, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69% 5897 |#######################################                  |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene d88feec6-34e8-423e-a6f8-1d6e8c47391f, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70% 5945 |########################################                 |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene a85ec9f0-b530-4453-9489-e24eba763962, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74% 6245 |##########################################               |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 40243c33-5ca4-4820-b00a-bfdc6dfd813a, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75% 6356 |##########################################               |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene ce832041-1b20-4039-ab02-f33bab06ae70, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75% 6378 |###########################################              |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 687c4f27-f9aa-4ba9-89cc-9ac7f137e836, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76% 6452 |###########################################              |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene fb21df65-9b64-454f-81c9-67615571d714, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79% 6698 |#############################################            |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 7e6c1ab3-0677-43ec-9c2d-875381c272a8, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80% 6824 |##############################################           |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene ccf852fa-0912-41d9-9da2-6f3cd9e3e478, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81% 6837 |##############################################           |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene ccf3c1a0-a285-4abd-967f-df1e1209d13f, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83% 7040 |###############################################          |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 3217930d-a70e-4cec-b118-49b0d1840d46, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84% 7107 |################################################         |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 6c43ce90-80e3-45aa-88dd-7dd5a7caf510, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84% 7125 |################################################         |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene c813e778-a55f-4cef-a641-c002a689ceca, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87% 7343 |#################################################        |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene e5a75b9d-2999-4723-8009-e72678ae8f84, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87% 7363 |#################################################        |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene eff48590-594a-46da-9ba4-d3f8d26eda2f, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87% 7421 |##################################################       |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 40589bb8-a219-4d97-b5f6-9b911893854b, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89% 7549 |##################################################       |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene ee3e35b7-ade0-498a-8776-1b4f9756a5ca, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90% 7608 |###################################################      |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene d476c128-026e-4e7c-a218-a0a31727100a, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90% 7617 |###################################################      |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 351c4d52-8e54-416c-9c98-8a92f3df6c02, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90% 7639 |###################################################      |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 63a1043b-33bf-41af-9000-ff98b27d062e, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90% 7657 |###################################################      |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene e2aa192e-7e95-4a90-ac2a-22649cee4954, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91% 7732 |####################################################     |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene f7e5a5ee-deb3-4431-b433-d19b3c8ed337, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% 8060 |######################################################   |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene be6291fa-8b17-4b7a-b185-95d3ebf96c23, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% 8071 |######################################################   |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 6eb364df-255f-47dd-b45f-608a6aaac744, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97% 8233 |#######################################################  |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene c0f6909d-532d-4c08-bca2-bc8a3eb7d534, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98% 8282 |#######################################################  |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene b638ad23-83f7-4be8-b0ec-39ae1298fced, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98% 8326 |######################################################## |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene c87ec003-9a91-4f0e-a71f-1379d7ed7c91, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99% 8361 |######################################################## |   0.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene cdc54e85-d9ff-4ccf-b97f-7f38865bb0dd, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100% 8438 |#########################################################|   0.2 B/s"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " None,\n",
       " ...]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "workers = 10\n",
    "\n",
    "widgets = [progressbar.Percentage(), ' ', progressbar.Counter(), ' ', progressbar.Bar(), ' ',\n",
    "           progressbar.FileTransferSpeed()]\n",
    "\n",
    "pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(scenes)).start()\n",
    "counter = 0\n",
    "\n",
    "Parallel(n_jobs=workers, backend='threading')(delayed(convert_scene_to_img_and_voxelmap)(in_directory, out_directory, i['scene_id']) for i in scenes)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generating filenames list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filenames = []\n",
    "for scene_id, (rgb_outfile, outfile) in image_names.items():\n",
    "    rgb_name = os.path.relpath(os.path.join(out_directory, rgb_outfile), start=out_root_directory)\n",
    "    rgb_name = rgb_name.replace('\\\\', '/')\n",
    "    voxelmap_name = os.path.relpath(os.path.join(out_directory, outfile), start=out_root_directory)\n",
    "    voxelmap_name = voxelmap_name.replace('\\\\', '/')\n",
    "    filenames.append([rgb_name, voxelmap_name])\n",
    "\n",
    "    # r je abych nemusel psát zpětná lomítka\n",
    "with open(os.path.join(out_root_directory, 'whole-voxel-gta.csv'), mode='w+', newline='') as f:\n",
    "    csv.writer(f).writerows(filenames)\n",
    "\n",
    "random.shuffle(filenames)\n",
    "\n",
    "train_ratio = 0.8\n",
    "train_threshold = int(train_ratio * len(image_names))\n",
    "train_filenames = filenames[:train_threshold]\n",
    "test_filenames = filenames[train_threshold:]\n",
    "\n",
    "with open(os.path.join(out_root_directory, 'train-voxel-gta.csv'), mode='w+', newline='') as f:\n",
    "    csv.writer(f).writerows(train_filenames)\n",
    "with open(os.path.join(out_root_directory, 'test-voxel-gta.csv'), mode='w+', newline='') as f:\n",
    "    csv.writer(f).writerows(test_filenames)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### running extraction for one scene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "convert_scene_to_img_and_voxelmap(in_directory, out_directory, scenes[20]['scene_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# just some timing experiments\n",
    "for i in range(5):\n",
    "    convert_scene_to_img_and_voxelmap(in_directory, out_directory, scenes[i]['scene_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prev = None\n",
    "for i in range(5):\n",
    "    cameras = load_scene_db_data(scenes[i]['scene_id'])\n",
    "    cam = get_main_image(cameras)\n",
    "    proj_matrix = cam['proj_matrix']\n",
    "    if prev is not None:\n",
    "        print(np.array_equal(prev, proj_matrix))\n",
    "    prev = proj_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "prev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-1,  0,  1], dtype=int8)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "loaded_values = np.load(r'D:\\ml-datasets-voxel\\2018-03-07--16-38-30--192.npy')\n",
    "np.unique(loaded_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
