{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import visualization\n",
    "import os\n",
    "from gta_math import points_to_homo, ndc_to_view, construct_proj_matrix, view_to_world, construct_view_matrix\n",
    "from visualization import load_depth, save_pointcloud_csv\n",
    "import progressbar\n",
    "from pointcloud_to_voxelmap import pointclouds_to_voxelmap\n",
    "from joblib import Parallel, delayed\n",
    "from configparser import ConfigParser\n",
    "from PIL import Image\n",
    "import pickle\n",
    "from voxelmaps import camera_to_pointcloud, load_scene_db_data, get_main_image_name, scene_to_pointcloud, scene_to_voxelmap\n",
    "import voxelmaps"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ini_file = \"gta-postprocessing.ini\"\n",
    "visualization.multi_page = False\n",
    "visualization.ini_file = ini_file\n",
    "\n",
    "conn = visualization.get_connection_pooled()\n",
    "cur = conn.cursor()\n",
    "\n",
    "CONFIG = ConfigParser()\n",
    "CONFIG.read(ini_file)\n",
    "in_directory = CONFIG[\"Images\"][\"Tiff\"]\n",
    "out_directory = CONFIG[\"Images\"][\"MlDatasetVoxel\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 8438 scenes\n"
     ]
    }
   ],
   "source": [
    "run_id = 6\n",
    "\n",
    "cur.execute(\"\"\"SELECT DISTINCT scene_id \\\n",
    "  FROM snapshots \\\n",
    "  WHERE run_id = {} \\\n",
    "  \"\"\".format(run_id))\n",
    "\n",
    "scenes = []\n",
    "for row in cur:\n",
    "    res = dict(row)\n",
    "    scenes.append(res)\n",
    "\n",
    "print('There are {} scenes'.format(len(scenes)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### functions for each scene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# import time\n",
    "voxelmaps.MAX_DISTANCE = 50 # in meters, meaning I care only for point up to 20 meters\n",
    "\n",
    "def get_base_name(name):\n",
    "    return os.path.basename(os.path.splitext(name)[0])\n",
    "\n",
    "\n",
    "def convert_tiff(in_directory, out_directory, out_name, name):\n",
    "    out_format = 'jpg'\n",
    "    outfile = os.path.join(out_directory, \"{}.{}\".format(out_name, out_format))\n",
    "    if os.path.exists(outfile):\n",
    "        return\n",
    "    try:\n",
    "        infile = os.path.join(in_directory, name)\n",
    "        im = Image.open(infile)\n",
    "        im = im.convert(mode=\"RGB\")\n",
    "        # print(\"Generating new format for {} to new file {}\".format(name, out_name))\n",
    "        im.save(outfile)\n",
    "    except OSError:\n",
    "        # print(\"Skipping invalid file {}\".format(name))\n",
    "        return\n",
    "\n",
    "\n",
    "def convert_scene_to_img_and_voxelmap(in_directory, out_directory, scene_id):\n",
    "    if 'pbar' in globals() and 'counter' in globals():\n",
    "        global counter\n",
    "        counter += 1\n",
    "        pbar.update(counter)\n",
    "\n",
    "    # start = time.time()\n",
    "    cameras = load_scene_db_data(scene_id)\n",
    "    try:\n",
    "        image_name = get_main_image_name(cameras)\n",
    "        convert_tiff(in_directory, out_directory, image_name, image_name+'.tiff')\n",
    "        outfile = os.path.join(out_directory, \"{}.rick\".format(image_name))\n",
    "\n",
    "        if os.path.exists(outfile):\n",
    "            return\n",
    "        \n",
    "        voxelmap = scene_to_voxelmap(cameras)\n",
    "        with open(outfile, 'wb+') as f:\n",
    "            pickle.dump(voxelmap, f)\n",
    "    except NoMainImageException:\n",
    "        print('No main image for scene {}, skipping.'.format(scene_id))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### actually running the extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3% 278 |#                                                         | 274.6 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 388309dc-568f-43ab-b2c1-86fb809cc570, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5% 489 |###                                                       | 296.6 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 592377fd-866a-4d81-bdb0-4e63ae584bab, skipping.\n",
      "No main image for scene c70fe8d1-e628-483e-a27a-6a2c7b862857, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16% 1351 |#########                                                | 329.5 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene e0a57943-f85b-4017-ac1d-6660a789972c, skipping.\n",
      "No main image for scene 1e82b334-4f8f-4a62-b98f-c1964cab55d5, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17% 1461 |#########                                                | 331.0 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 9215c874-450b-485a-9c53-f44b0b16e86d, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18% 1602 |##########                                               | 332.0 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene a8cc8456-1bfb-4f17-867a-8259be7a7aef, skipping.\n",
      "No main image for scene 7a7314b0-92a5-402f-a690-2425b48133e6, skipping.\n",
      "No main image for scene be0bb451-894a-40c5-8e86-f9e51b8e73d2, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20% 1709 |###########                                              | 332.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene be512cbb-475d-4a86-b698-09cd0dc4594c, skipping.\n",
      "No main image for scene cdc4513d-105c-4f66-8682-0f5c3e466230, skipping.\n",
      "No main image for scene a7fa77e4-8981-49ea-ab97-3a8ff39f85ab, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22% 1923 |############                                             | 334.6 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 575da4d5-1d8c-4d07-8c6c-0002868990ea, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                               \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 08056d94-ab3d-42cd-8000-f9828444e783, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25% 2136 |##############                                           | 334.3 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 6f634b01-f5a0-4d65-b2fc-7df58b0d8904, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28% 2428 |################                                         | 335.0 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 1825304b-591c-46c3-8361-8443cb11fd64, skipping.\n",
      "No main image for scene 745ddc7a-3d29-4d21-ac25-46e3351b269c, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30% 2581 |#################                                        | 336.4 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 73090cd6-266e-49d8-b7cf-4081685587a9, skipping.\n",
      "No main image for scene f5e84ca1-ad34-45c5-84c9-92981c124d93, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32% 2736 |##################                                       | 338.6 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 0e0860ca-6286-4e5f-82ae-50e020246266, skipping.\n",
      "No main image for scene 0514fbe8-e421-4c83-b48a-c83d7404cac0, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33% 2850 |###################                                      | 339.1 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 410ae84d-e9dc-44b2-b88d-07a397e1b073, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36% 3099 |####################                                     | 340.6 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 1cec18a3-1847-4bfd-8cb5-6e24bf1a431b, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41% 3481 |#######################                                  | 341.4 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 00b11f74-4351-44e6-a04f-ffa5b0aa9687, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42% 3555 |########################                                 | 339.8 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene d88feec6-34e8-423e-a6f8-1d6e8c47391f, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43% 3629 |########################                                 | 340.0 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene a85ec9f0-b530-4453-9489-e24eba763962, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45% 3841 |#########################                                | 340.4 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 1d0939c2-dc9d-48bb-b088-4c42b219ad7b, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46% 3907 |##########################                               | 340.1 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene cd7334f6-d78f-4cc2-97fc-27971040af36, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49% 4182 |############################                             | 339.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 40243c33-5ca4-4820-b00a-bfdc6dfd813a, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52% 4416 |#############################                            | 338.0 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene ce832041-1b20-4039-ab02-f33bab06ae70, skipping.\n",
      "No main image for scene 687c4f27-f9aa-4ba9-89cc-9ac7f137e836, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53% 4488 |##############################                           | 338.6 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 0b770141-6775-4f63-9254-cc32226f9eda, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54% 4623 |###############################                          | 338.5 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene fb21df65-9b64-454f-81c9-67615571d714, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59% 5062 |##################################                       | 340.1 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 7e6c1ab3-0677-43ec-9c2d-875381c272a8, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61% 5167 |##################################                       | 339.7 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 7610c8e6-f262-4f3c-89bd-6414f701333c, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62% 5302 |###################################                      | 340.0 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene ccf852fa-0912-41d9-9da2-6f3cd9e3e478, skipping.\n",
      "No main image for scene 92e1fb64-1f97-4e2c-a959-622ccb583ea9, skipping.\n",
      "No main image for scene ccf3c1a0-a285-4abd-967f-df1e1209d13f, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63% 5377 |####################################                     | 340.1 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 47dc1c0f-958e-4165-9648-db4b651e97b5, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67% 5661 |######################################                   | 340.9 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 4d85bc08-8dea-488d-af24-af165f9b25e4, skipping.\n",
      "No main image for scene 946fe837-8f42-485e-a82f-00a19760f918, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67% 5725 |######################################                   | 340.4 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 3217930d-a70e-4cec-b118-49b0d1840d46, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69% 5840 |#######################################                  | 340.7 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 6c43ce90-80e3-45aa-88dd-7dd5a7caf510, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70% 5950 |########################################                 | 340.9 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene c813e778-a55f-4cef-a641-c002a689ceca, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74% 6268 |##########################################               | 341.3 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene c72c1e21-629a-4d90-92c5-0ea06715fead, skipping.\n",
      "No main image for scene e5a75b9d-2999-4723-8009-e72678ae8f84, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75% 6331 |##########################################               | 340.9 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene eff48590-594a-46da-9ba4-d3f8d26eda2f, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76% 6442 |###########################################              | 341.2 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 40589bb8-a219-4d97-b5f6-9b911893854b, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79% 6697 |#############################################            | 341.7 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene ee3e35b7-ade0-498a-8776-1b4f9756a5ca, skipping.\n",
      "No main image for scene 4e113039-6c55-47b8-8fb3-90ca54b0ae84, skipping."
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80% 6762 |#############################################            | 341.6 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "No main image for scene a9707fcc-2468-4620-8be2-6fca9d9b8599, skipping.\n",
      "No main image for scene d476c128-026e-4e7c-a218-a0a31727100a, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81% 6867 |##############################################           | 341.6 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 351c4d52-8e54-416c-9c98-8a92f3df6c02, skipping.\n",
      "No main image for scene d06bf958-fcb7-4983-b50e-0fc00d8e674e, skipping.\n",
      "No main image for scene 63a1043b-33bf-41af-9000-ff98b27d062e, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82% 6939 |##############################################           | 341.7 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene e2aa192e-7e95-4a90-ac2a-22649cee4954, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84% 7090 |###############################################          | 341.7 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene f7e5a5ee-deb3-4431-b433-d19b3c8ed337, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89% 7587 |###################################################      |   2.9 B/sC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tifffile\\tifffile.py:2642: RuntimeWarning: py_decodelzw encountered unexpected end of stream\n",
      "  strip = decompress(strip)\n",
      " 89% 7588 |###################################################      |   2.9 B/sC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tifffile\\tifffile.py:2577: UserWarning: unpack: string size must be a multiple of element size\n",
      "  warnings.warn(\"unpack: %s\" % e)\n",
      " 91% 7694 |###################################################      |   2.3 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene be6291fa-8b17-4b7a-b185-95d3ebf96c23, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91% 7698 |####################################################     |   2.3 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene ded7f7fc-275b-4760-bcbe-3ee21511d741, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91% 7712 |####################################################     |   2.3 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 99975c57-a080-4513-aea9-bb1c70dec8c3, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91% 7714 |####################################################     |   2.3 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene 6eb364df-255f-47dd-b45f-608a6aaac744, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% 8024 |######################################################   |   1.5 B/s"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No main image for scene c0f6909d-532d-4c08-bca2-bc8a3eb7d534, skipping.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95% 8057 |######################################################   |   1.4 B/s"
     ]
    }
   ],
   "source": [
    "workers = 10\n",
    "\n",
    "widgets = [progressbar.Percentage(), ' ', progressbar.Counter(), ' ', progressbar.Bar(), ' ',\n",
    "           progressbar.FileTransferSpeed()]\n",
    "\n",
    "pbar = progressbar.ProgressBar(widgets=widgets, maxval=len(scenes)).start()\n",
    "counter = 0\n",
    "\n",
    "Parallel(n_jobs=workers, backend='threading')(delayed(convert_scene_to_img_and_voxelmap)(in_directory, out_directory, i['scene_id']) for i in scenes)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generating filenames list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filenames = []\n",
    "for i, (name, _) in enumerate(records):\n",
    "    rgb_name = \"{}.{}\".format(get_base_name(name), rgb_format)\n",
    "    rgb_name = os.path.relpath(os.path.join(out_directory, rgb_name), start=out_root_directory)\n",
    "    rgb_name = rgb_name.replace('\\\\', '/')\n",
    "    depth_name = \"{}.{}\".format(get_base_name(name), depth_format)\n",
    "    depth_name = os.path.relpath(os.path.join(out_directory, depth_name), start=out_root_directory)\n",
    "    depth_name = depth_name.replace('\\\\', '/')\n",
    "    filenames.append([rgb_name, depth_name])\n",
    "\n",
    "    # r je abych nemusel psát zpětná lomítka\n",
    "with open(os.path.join(out_root_directory, 'whole-voxel-gta.csv'), mode='w+', newline='') as f:\n",
    "    csv.writer(f).writerows(filenames)\n",
    "\n",
    "random.shuffle(filenames)\n",
    "\n",
    "train_ratio = 0.8\n",
    "train_threshold = int(train_ratio * len(filenames))\n",
    "train_filenames = filenames[:train_threshold]\n",
    "test_filenames = filenames[train_threshold:]\n",
    "\n",
    "with open(os.path.join(out_root_directory, 'train-voxel-gta.csv'), mode='w+', newline='') as f:\n",
    "    csv.writer(f).writerows(train_filenames)\n",
    "with open(os.path.join(out_root_directory, 'test-voxel-gta.csv'), mode='w+', newline='') as f:\n",
    "    csv.writer(f).writerows(test_filenames)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### running extraction for one scene"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "convert_scene_to_img_and_voxelmap(in_directory, out_directory, scenes[0]['scene_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# this is to visualize voxelmap as a pointcloud\n",
    "def voxelmap_to_pointcloud_csv(path_in, path_out):\n",
    "    with open(path_in, 'rb') as f:\n",
    "        voxels, values, voxel_size = pickle.load(f)\n",
    "    voxels = voxels[:, values >= 0]\n",
    "    save_pointcloud_csv(voxels.T, path_out)\n",
    "\n",
    "\n",
    "def scene_to_pointcloud_csv(scene_id, path_out):\n",
    "    if 'pbar' in globals() and 'counter' in globals():\n",
    "        global counter\n",
    "        counter += 1\n",
    "        pbar.update(counter)\n",
    "\n",
    "    cameras = load_scene_db_data(scene_id)\n",
    "    pointclouds, cam_positions = scene_to_pointcloud(cameras)\n",
    "    big_pointcloud = np.zeros((3, 0))\n",
    "    for pc in pointclouds:\n",
    "        big_pointcloud = np.append(big_pointcloud, pc, axis=1)\n",
    "    save_pointcloud_csv(big_pointcloud.T, path_out)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "voxelmap_to_pointcloud_csv(r'D:\\ml-datasets-voxel\\2018-03-07--15-19-54--505.rick', r'D:\\ml-datasets-voxel\\30m-2018-03-07--15-19-54--505.csv')\n",
    "scene_to_pointcloud_csv(scenes[0]['scene_id'], r'D:\\ml-datasets-voxel\\pc-2018-03-07--15-19-54--505.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    " 23% 1983 |#############                                            | 146.5 B/s\n",
    "\n",
    "No main image for scene 08056d94-ab3d-42cd-8000-f9828444e783, skipping.\n",
    "\n",
    " 24% 2049 |#############                                            |   5.8 B/s\n",
    "\n",
    "No main image for scene 6f634b01-f5a0-4d65-b2fc-7df58b0d8904, skipping.\n",
    "\n",
    " 24% 2086 |##############                                           |   3.4 B/sC:\\ProgramData\\Anaconda3\\lib\\site-packages\\tifffile\\tifffile.py:2642: RuntimeWarning: py_decodelzw encountered unexpected end of stream\n",
    "  strip = decompress(strip)\n",
    " 25% 2139 |##############                                           |   2.2 B/s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
