{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "from PIL import Image, ImageFile\n",
    "from skimage import io\n",
    "import matplotlib.pyplot as plt\n",
    "import psycopg2\n",
    "import tifffile\n",
    "from scipy import misc\n",
    "from tifffile import TiffFile\n",
    "from tifffile.tifffile import TIFF_DECOMPESSORS\n",
    "from configparser import ConfigParser\n",
    "from batch_tiff_to_jpeg import tiff_to_jpg\n",
    "from psycopg2.extras import DictCursor\n",
    "from visualization import get_connection, get_gta_image_jpg_dir\n",
    "import progressbar\n",
    "import random\n",
    "from os import path\n",
    "import math\n",
    "import humanize\n",
    "JPEG_DIR = 'D:/GTAV_extraction_output/rgb-jpeg'\n",
    "JPEG_DIR_2 = 'Y:/GTA-jpg'\n",
    "JPEG_DIRS = [JPEG_DIR, JPEG_DIR_2]\n",
    "TIFF_DIR = 'D:/GTAV_extraction_output'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# threaded connection pooling\n",
    "from psycopg2.pool import PersistentConnectionPool\n",
    "from psycopg2.extensions import ISOLATION_LEVEL_AUTOCOMMIT\n",
    "\n",
    "conn_pool = None\n",
    "\n",
    "def get_connection_pooled():\n",
    "    \"\"\"\n",
    "    :rtype: connection\n",
    "    \"\"\"\n",
    "    global conn_pool\n",
    "    if conn_pool is None:\n",
    "        CONFIG = ConfigParser()\n",
    "        CONFIG.read(\"gta-postprocessing.ini\")\n",
    "        minimum = 1\n",
    "        maximum = 28\n",
    "        conn_pool = PersistentConnectionPool(minimum, maximum, CONFIG[\"Postgres\"][\"db\"], cursor_factory=DictCursor)\n",
    "    conn = conn_pool.getconn()\n",
    "    conn.set_isolation_level(ISOLATION_LEVEL_AUTOCOMMIT)\n",
    "    return conn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### selecting images metadata, grouping by position"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = get_connection()\n",
    "cur = conn.cursor()\n",
    "\n",
    "# for each position, select first time and last time car was in this position\n",
    "cur.execute(\"SELECT DISTINCT COUNT(camera_pos) AS count FROM snapshots\")\n",
    "\n",
    "position_count = cur.fetchone()['count']\n",
    "# cur = conn.cursor('server_cursor_{}'.format(random.randint(0, 1e9)))\n",
    "cur = conn.cursor()\n",
    "print(cur.itersize)\n",
    "\n",
    "cur.execute(\"\"\"SELECT \\\n",
    "        ARRAY[st_x(camera_pos), st_y(camera_pos), st_z(camera_pos)] as position, \\\n",
    "    camera_pos, min(timestamp) AS minimum, max(timestamp) AS maximum, \\\n",
    "    min(imagepath), min(proj_matrix), \\\n",
    "    count(camera_pos) as count \\\n",
    "    FROM snapshots \\\n",
    "    GROUP BY camera_pos \\\n",
    "    ORDER BY min(timestamp) \\\n",
    "    \"\"\")\n",
    "\n",
    "positions = {}\n",
    "print('positions_total: {}'.format(position_count))\n",
    "\n",
    "widgets = [progressbar.Percentage(), ' ', progressbar.Counter(), ' ', progressbar.Bar(), ' ', progressbar.FileTransferSpeed()]\n",
    "pbar = progressbar.ProgressBar(widgets=widgets, max_value = cur.rowcount).start()\n",
    "\n",
    "for i, row in enumerate(cur):\n",
    "    pbar.update(i)\n",
    "    positions[tuple(row['position'])] = {\n",
    "        'min': row['minimum'],\n",
    "        'max': row['maximum'],\n",
    "        'time': row['maximum'] - row['minimum'],\n",
    "        'position': row['position'],\n",
    "        'count': row['count'],\n",
    "        'index': i,\n",
    "    }\n",
    "pbar.finish()\n",
    "\n",
    "print('total {} positions'.format(len(positions)))\n",
    "# timespans = {key:pos['max'] - pos['min'] for key, pos in positions.items()}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### selecting images metadata, grouping by projection matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "conn = get_connection()\n",
    "cur = conn.cursor()\n",
    "\n",
    "# for each position, select first time and last time car was in this position\n",
    "cur.execute(\"SELECT DISTINCT COUNT(camera_pos) AS count FROM snapshots\")\n",
    "\n",
    "position_count = cur.fetchone()['count']\n",
    "# cur = conn.cursor('server_cursor_{}'.format(random.randint(0, 1e9)))\n",
    "cur = conn.cursor()\n",
    "print(cur.itersize)\n",
    "\n",
    "# somehow, grouping by proj matrix shows me same images, but grouping by camera pos does not\n",
    "cur.execute(\"\"\"SELECT \\\n",
    "        min(ARRAY[st_x(camera_pos), st_y(camera_pos), st_z(camera_pos)]) as position, \\\n",
    "    min(camera_pos), min(timestamp) AS minimum, max(timestamp) AS maximum, \\\n",
    "    min(imagepath), \\\n",
    "    proj_matrix, \\\n",
    "    count(proj_matrix) as count \\\n",
    "    FROM snapshots \\\n",
    "    GROUP BY proj_matrix \\\n",
    "    ORDER BY min(timestamp) \\\n",
    "    \"\"\")\n",
    "\n",
    "projections = {}\n",
    "print('positions_total: {}'.format(position_count))\n",
    "\n",
    "widgets = [progressbar.Percentage(), ' ', progressbar.Counter(), ' ', progressbar.Bar(), ' ', progressbar.FileTransferSpeed()]\n",
    "pbar = progressbar.ProgressBar(widgets=widgets, max_value = cur.rowcount).start()\n",
    "\n",
    "for i, row in enumerate(cur):\n",
    "    pbar.update(i)\n",
    "#    print(tuple(tuple(x) for x in row['proj_matrix']))\n",
    "    projections[tuple(tuple(x) for x in row['proj_matrix'])] = {\n",
    "        'min': row['minimum'],\n",
    "        'max': row['maximum'],\n",
    "        'time': row['maximum'] - row['minimum'],\n",
    "        'position': row['position'],\n",
    "        'proj_matrix': row['proj_matrix'],\n",
    "        'count': row['count'],\n",
    "        'index': i,\n",
    "    }\n",
    "pbar.finish()\n",
    "\n",
    "print('total {} projections'.format(len(projections)))\n",
    "# timespans = {key:pos['max'] - pos['min'] for key, pos in positions.items()}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### display images per position"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "offset = 1\n",
    "stride = 1\n",
    "# sorted_data = sorted(positions.items(), key=lambda x:x[1]['index'])\n",
    "sorted_data = sorted(positions.items(), key=lambda x: -x[1]['count'])\n",
    "for position, data in sorted_data[offset::stride]:\n",
    "    i = data['index']\n",
    "    cur = conn.cursor()\n",
    "    print(position)\n",
    "# fucking python has only 1e-12 float position, which is not enough for postgres\n",
    "    cur.execute(\"\"\"SELECT \\\n",
    "        imagepath, \\\n",
    "        ARRAY[st_x(camera_pos), st_y(camera_pos)::double precision, st_z(camera_pos)::double precision] as position, \\\n",
    "        camera_pos \\\n",
    "    FROM snapshots \\\n",
    "      WHERE st_distance(camera_pos, ST_GeomFromText('POINT({} {} {})')) < 1e-9 \\\n",
    "    \"\"\".format(position[0], position[1], position[2]))\n",
    "\n",
    "    print(\"i: {}, time: {}, same position images: {}\".format(i, data['min'], cur.rowcount))\n",
    "    for row in cur:\n",
    "        # basename = 'info-' + row['imagepath']\n",
    "        basename = row['imagepath']\n",
    "        img_path = path.join(JPEG_DIR, basename + '-0.jpg')\n",
    "        if not path.exists(img_path):\n",
    "            print('image not found')\n",
    "            continue\n",
    "        plt.figure(figsize=(15,15))\n",
    "        img = Image.open(img_path)\n",
    "        plt.imshow(img)\n",
    "        plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### display images per projection matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "offset = 1\n",
    "stride = 1\n",
    "# sorted_data = sorted(projections.items(), key=lambda x:x[1]['index'])\n",
    "sorted_data = sorted(projections.items(), key=lambda x: -x[1]['count'])\n",
    "for position, data in sorted_data[offset::stride]:\n",
    "    i = data['index']\n",
    "    cur = conn.cursor()\n",
    "    print(projection)\n",
    "# fucking python has only 1e-12 float position, which is not enough for postgres\n",
    "    cur.execute(\"\"\"SELECT \\\n",
    "        imagepath, \\\n",
    "        ARRAY[st_x(camera_pos), st_y(camera_pos)::double precision, st_z(camera_pos)::double precision] as position, \\\n",
    "        camera_pos \\\n",
    "    FROM snapshots \\\n",
    "      WHERE proj_matrix::NUMERIC[] = ARRAY{} \\\n",
    "    \"\"\".format(data['proj_matrix']))\n",
    "\n",
    "    print(\"i: {}, time: {}, same proj_matrix images: {}\".format(i, data['min'], cur.rowcount))\n",
    "    for row in cur:\n",
    "        # basename = 'info-' + row['imagepath']\n",
    "        basename = row['imagepath']\n",
    "        img_path = path.join(JPEG_DIR, basename + '-0.jpg')\n",
    "        if not path.exists(img_path):\n",
    "            print('image not found')\n",
    "            continue\n",
    "        plt.figure(figsize=(15,15))\n",
    "        img = Image.open(img_path)\n",
    "        plt.imshow(img)\n",
    "        plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from decimal import *\n",
    "print(float(\"431.63467407226560000\"))\n",
    "print(Decimal('431.63467407226560000'))\n",
    "print(float(\"-55309.14581298830000000\"))\n",
    "              55309.1458129883\n",
    "# float('431.63467')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "sorted(positions.items(), key=lambda x: x[1]['min'])\n",
    "sorted(projections.items(), key=lambda x: -x[1]['count'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### showing proj_image histogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = [x['index'] for key, x in positions.items()]\n",
    "values = [x['count'] for key, x in positions.items()]\n",
    "plt.plot(keys, values)\n",
    "plt.title('count of images per position')\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "keys = [x['index'] for key, x in projections.items()]\n",
    "values = [x['count'] for key, x in projections.items()]\n",
    "plt.plot(keys, values)\n",
    "plt.title('count of images per projection')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = [x['index'] for key, x in positions.items()]\n",
    "values = [math.log(x['count']) for key, x in positions.items()]\n",
    "plt.plot(keys, values)\n",
    "plt.title('logarithmed count of images per position')\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "keys = [x['index'] for key, x in projections.items()]\n",
    "values = [math.log(x['count']) for key, x in projections.items()]\n",
    "plt.plot(keys, values)\n",
    "plt.title('logarithmed count of images per projection')\n",
    "plt.grid(True)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "keys = [x['index'] for key, x in positions.items()]\n",
    "values = [(x['max'] - x['min']).seconds / 60 for key, x in positions.items()]\n",
    "plt.plot(keys, values)\n",
    "plt.title('minutes in same position')\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "keys = [x['index'] for key, x in projections.items()]\n",
    "values = [(x['max'] - x['min']).seconds / 60 for key, x in projections.items()]\n",
    "plt.plot(keys, values)\n",
    "plt.title('minutes in same projection')\n",
    "plt.grid(True)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### filtering function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lag_interval_min = 13\n",
    "treshold = 2000\n",
    "\n",
    "def filter_lagged(data):\n",
    "    return {key: x for key, x in data.items() if (((x['max'] - x['min']).seconds / 60) > lag_interval_min) and x['count'] >= treshold}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### showing number of glitched by positions and projections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered = filter_lagged(positions)\n",
    "keys = [x['index'] for key, x in filtered.items()]\n",
    "values = [x['count'] for key, x in filtered.items()]\n",
    "plt.scatter(keys, values)\n",
    "plt.grid(True)\n",
    "plt.show()\n",
    "\n",
    "filtered = filter_lagged(projections)\n",
    "keys = [x['index'] for key, x in filtered.items()]\n",
    "values = [x['count'] for key, x in filtered.items()]\n",
    "plt.scatter(keys, values)\n",
    "plt.grid(True)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### computing size of projected to be deleted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filtered = filter_lagged(projections)\n",
    "total_size = 0\n",
    "\n",
    "widgets = [progressbar.Percentage(), ' ', progressbar.Counter(), ' ', progressbar.Bar(), ' ', progressbar.FileTransferSpeed()]\n",
    "pbar = progressbar.ProgressBar(widgets=widgets, max_value = len(filtered)).start()\n",
    "j = 0\n",
    "for projection, data in filtered.items():\n",
    "    j += 1\n",
    "    pbar.update(j)\n",
    "    i = data['index']\n",
    "    cur = conn.cursor()\n",
    "    cur.execute(\"\"\"SELECT \\\n",
    "        imagepath, \\\n",
    "        ARRAY[st_x(camera_pos), st_y(camera_pos)::double precision, st_z(camera_pos)::double precision] as position, \\\n",
    "        camera_pos \\\n",
    "    FROM snapshots \\\n",
    "      WHERE proj_matrix::NUMERIC[] = ARRAY{} \\\n",
    "    \"\"\".format(data['proj_matrix']))\n",
    "\n",
    "#    print(\"i: {}, from: {}, to: {}, same proj_matrix images: {}\".format(i, data['min'], data['max'], cur.rowcount))\n",
    "    pbar1 = progressbar.ProgressBar(widgets=widgets, max_value = cur.rowcount).start()\n",
    "    k = 0\n",
    "    for row in cur:\n",
    "        k += 1\n",
    "        pbar1.update(k)\n",
    "        # basename = 'info-' + row['imagepath']\n",
    "        basename = row['imagepath']\n",
    "        img_path = path.join(JPEG_DIR, basename + '-0.jpg')\n",
    "        if not path.exists(img_path):\n",
    "            continue\n",
    "        total_size += os.path.getsize(img_path)\n",
    "        img_path = path.join(TIFF_DIR, basename + '.tiff')\n",
    "        if not path.exists(img_path):\n",
    "            continue\n",
    "        total_size += os.path.getsize(img_path)\n",
    "        img_path = path.join(TIFF_DIR, basename + '-depth.tiff')\n",
    "        if not path.exists(img_path):\n",
    "            continue\n",
    "        total_size += os.path.getsize(img_path)\n",
    "        img_path = path.join(TIFF_DIR, basename + '-stencil.tiff')\n",
    "        if not path.exists(img_path):\n",
    "            continue\n",
    "        total_size += os.path.getsize(img_path)\n",
    "    pbar1.finish()\n",
    "pbar.finish()\n",
    "        \n",
    "print('total size is {}'.format(humanize.naturalsize(total_size)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "total_size\n",
    "filtered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "filtered = filter_lagged(projections)\n",
    "for projection, data in sorted(filtered.items(), key=lambda x:x[1]['index']):\n",
    "    i = data['index']\n",
    "    cur = conn.cursor()\n",
    "    print(projection)\n",
    "# fucking python has only 1e-12 float position, which is not enough for postgres\n",
    "    cur.execute(\"\"\"SELECT \\\n",
    "        imagepath, \\\n",
    "        ARRAY[st_x(camera_pos), st_y(camera_pos)::double precision, st_z(camera_pos)::double precision] as position, \\\n",
    "        camera_pos \\\n",
    "    FROM snapshots \\\n",
    "      WHERE proj_matrix::NUMERIC[] = ARRAY{} \\\n",
    "      LIMIT 10 \\\n",
    "    \"\"\".format(data['proj_matrix']))\n",
    "\n",
    "    print(\"i: {}, time: {}, same proj_matrix images: {}\".format(i, data['min'], cur.rowcount))\n",
    "    for row in cur:\n",
    "        # basename = 'info-' + row['imagepath']\n",
    "        basename = row['imagepath']\n",
    "        img_path = path.join(JPEG_DIR, basename + '-0.jpg')\n",
    "        if not path.exists(img_path):\n",
    "            print('image not found with name {}'.format(img_path))\n",
    "            continue\n",
    "        plt.figure(figsize=(15,15))\n",
    "        img = Image.open(img_path)\n",
    "        plt.imshow(img)\n",
    "        plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## actually deleting things"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from joblib import Parallel, delayed\n",
    "from psycopg2.pool import PoolError\n",
    "import time\n",
    "\n",
    "filtered = filter_lagged(projections)\n",
    "# jpeg_names = ['info-{}-0.jpg']\n",
    "# tiff_names = ['info-{}.tiff', 'info-{}-depth.tiff', 'info-{}-stencil.tiff']\n",
    "jpeg_names = ['{}-0.jpg']\n",
    "tiff_names = ['{}.tiff', '{}-depth.tiff', '{}-stencil.tiff']\n",
    "\n",
    "\n",
    "def delete_image(row):\n",
    "    global counter\n",
    "    counter += 1\n",
    "    pbar.update(counter)\n",
    "    for DIR in JPEG_DIRS:\n",
    "        for name in jpeg_names:\n",
    "            img_path = path.join(DIR, name.format(row['imagepath']))\n",
    "            if not path.exists(img_path):\n",
    "                continue\n",
    "            os.remove(img_path)\n",
    "    for name in tiff_names:\n",
    "        img_path = path.join(TIFF_DIR, name.format(row['imagepath']))\n",
    "        if not path.exists(img_path):\n",
    "            continue\n",
    "        os.remove(img_path)\n",
    "    # n_start = time.time()\n",
    "    while True:\n",
    "        try:\n",
    "            conn2 = get_connection_pooled()\n",
    "            break\n",
    "        except PoolError:\n",
    "            print('waiting for free connection')\n",
    "            time.sleep(0.5)\n",
    "            \n",
    "    cur2 = conn2.cursor()\n",
    "    cur2.execute('DELETE FROM snapshots WHERE snapshot_id = {}'.format(row['snapshot_id']))\n",
    "    # n_end = time.time()\n",
    "    # print('sql time to execute: {}'.format(n_end - n_start))\n",
    "\n",
    "\n",
    "for projection, data in sorted(filtered.items(), key=lambda x:x[1]['index']):\n",
    "    i = data['index']\n",
    "    cur = conn.cursor()\n",
    "# fucking python has only 1e-12 float position, which is not enough for postgres\n",
    "    cur.execute(\"\"\"SELECT \\\n",
    "        snapshot_id, \\\n",
    "        imagepath, \\\n",
    "        ARRAY[st_x(camera_pos), st_y(camera_pos)::double precision, st_z(camera_pos)::double precision] as position, \\\n",
    "        camera_pos \\\n",
    "    FROM snapshots \\\n",
    "      WHERE proj_matrix::NUMERIC[] = ARRAY{} \\\n",
    "    \"\"\".format(data['proj_matrix']))\n",
    "    \n",
    "    if cur.rowcount < treshold:\n",
    "        continue\n",
    "    \n",
    "    rows = cur.fetchall()\n",
    "    cur.close()\n",
    "            \n",
    "    if conn_pool is not None:\n",
    "        conn_pool.closeall()\n",
    "    conn_pool = None\n",
    "\n",
    "    print(\"i: {}, time: {}, same proj_matrix images: {}, going to delete them\"\n",
    "          .format(i, data['min'], len(rows)))\n",
    "    pbar = progressbar.ProgressBar(widgets=widgets, max_value = cur.rowcount).start()\n",
    "    counter = 0\n",
    "    workers = 20\n",
    "    Parallel(n_jobs=workers, backend='threading')(delayed(delete_image)(row) for row in rows)\n",
    "    # for row in rows:\n",
    "    #     delete_image(row)\n",
    "    pbar.finish()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "[r['snapshot_id'] for r in rows]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "row = rows[10]\n",
    "print(row)\n",
    "for DIR in JPEG_DIRS:\n",
    "    for name in jpeg_names:\n",
    "        img_path = path.join(DIR, name.format(row['imagepath']))\n",
    "        if not path.exists(img_path):\n",
    "            print('{} not exists'.format(img_path))\n",
    "            continue\n",
    "        os.remove(img_path)\n",
    "for name in tiff_names:\n",
    "    img_path = path.join(TIFF_DIR, name.format(row['imagepath']))\n",
    "    if not path.exists(img_path):\n",
    "        print('{} not exists'.format(img_path))\n",
    "        continue\n",
    "    os.remove(img_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### dumping filtered values to delete"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "filtered = filter_lagged(projections)\n",
    "list(filtered.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_keys = sorted(projections, key=lambda x: -projections[x]['count'])\n",
    "projections[sorted_keys[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
