{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3119ecc5-cc95-4977-9296-e2e59183341a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import planetary_computer\n",
    "import itertools\n",
    "import dask.dataframe as dd\n",
    "\n",
    "cc = planetary_computer.get_container_client(\"pcstacitems\", \"items\")\n",
    "\n",
    "blobs = list(cc.list_blobs(\"sentinel-2-l2a.parquet/\"))\n",
    "\n",
    "def key(blob):\n",
    "    return blob.name.split(\"/\")[1].split(\"_\")[0]\n",
    "\n",
    "keep_blobs = []\n",
    "for k, v in itertools.groupby(sorted(blobs, key=key), key=key):\n",
    "    v = list(v)\n",
    "    blob = max(v, key=lambda x: x.last_modified)\n",
    "    keep_blobs.append(blob)\n",
    "    \n",
    "uris = [f\"az://items/{blob.name}\" for blob in keep_blobs]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "31660c9b-6158-41aa-b4b8-30f0e56cd11a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "445"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(uris)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b893b6a9-9f6e-460e-9e07-7bace08638e0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>stac_version</th>\n",
       "      <th>stac_extensions</th>\n",
       "      <th>id</th>\n",
       "      <th>geometry</th>\n",
       "      <th>bbox</th>\n",
       "      <th>links</th>\n",
       "      <th>assets</th>\n",
       "      <th>collection</th>\n",
       "      <th>datetime</th>\n",
       "      <th>...</th>\n",
       "      <th>s2:cloud_shadow_percentage</th>\n",
       "      <th>s2:nodata_pixel_percentage</th>\n",
       "      <th>s2:unclassified_percentage</th>\n",
       "      <th>s2:dark_features_percentage</th>\n",
       "      <th>s2:not_vegetated_percentage</th>\n",
       "      <th>s2:degraded_msi_data_percentage</th>\n",
       "      <th>s2:high_proba_clouds_percentage</th>\n",
       "      <th>s2:reflectance_conversion_factor</th>\n",
       "      <th>s2:medium_proba_clouds_percentage</th>\n",
       "      <th>s2:saturated_defective_pixel_percentage</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Feature</td>\n",
       "      <td>1.0.0</td>\n",
       "      <td>[https://stac-extensions.github.io/eo/v1.0.0/s...</td>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T35XQA_2021041...</td>\n",
       "      <td>b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x1e\\x00...</td>\n",
       "      <td>[32.782321671748434, 71.80556197526644, 36.189...</td>\n",
       "      <td>[{'href': 'https://planetarycomputer.microsoft...</td>\n",
       "      <td>{'AOT': {'gsd': 10.0, 'href': 'https://sentine...</td>\n",
       "      <td>sentinel-2-l2a</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>20.722227</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>92.546540</td>\n",
       "      <td>0.967449</td>\n",
       "      <td>4.807670</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Feature</td>\n",
       "      <td>1.0.0</td>\n",
       "      <td>[https://stac-extensions.github.io/eo/v1.0.0/s...</td>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T32TMM_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0e\\x00...</td>\n",
       "      <td>[8.479220766509275, 41.46069640920803, 9.11869...</td>\n",
       "      <td>[{'href': 'https://planetarycomputer.microsoft...</td>\n",
       "      <td>{'AOT': {'gsd': 10.0, 'href': 'https://sentine...</td>\n",
       "      <td>sentinel-2-l2a</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.009120</td>\n",
       "      <td>63.654864</td>\n",
       "      <td>0.265107</td>\n",
       "      <td>0.215319</td>\n",
       "      <td>13.662070</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.048035</td>\n",
       "      <td>0.967449</td>\n",
       "      <td>0.051376</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Feature</td>\n",
       "      <td>1.0.0</td>\n",
       "      <td>[https://stac-extensions.github.io/eo/v1.0.0/s...</td>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T32TMN_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...</td>\n",
       "      <td>[8.765144061005621, 42.36232117367103, 9.12043...</td>\n",
       "      <td>[{'href': 'https://planetarycomputer.microsoft...</td>\n",
       "      <td>{'AOT': {'gsd': 10.0, 'href': 'https://sentine...</td>\n",
       "      <td>sentinel-2-l2a</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001858</td>\n",
       "      <td>85.355914</td>\n",
       "      <td>0.437180</td>\n",
       "      <td>0.223688</td>\n",
       "      <td>19.642879</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.011238</td>\n",
       "      <td>0.967449</td>\n",
       "      <td>0.022928</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Feature</td>\n",
       "      <td>1.0.0</td>\n",
       "      <td>[https://stac-extensions.github.io/eo/v1.0.0/s...</td>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T36WWC_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\t\\x00\\x...</td>\n",
       "      <td>[32.99946824095416, 69.68910488955622, 34.0680...</td>\n",
       "      <td>[{'href': 'https://planetarycomputer.microsoft...</td>\n",
       "      <td>{'AOT': {'gsd': 10.0, 'href': 'https://sentine...</td>\n",
       "      <td>sentinel-2-l2a</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000432</td>\n",
       "      <td>88.475966</td>\n",
       "      <td>0.323290</td>\n",
       "      <td>0.006248</td>\n",
       "      <td>0.008810</td>\n",
       "      <td>0.0</td>\n",
       "      <td>65.812266</td>\n",
       "      <td>0.967449</td>\n",
       "      <td>19.050561</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Feature</td>\n",
       "      <td>1.0.0</td>\n",
       "      <td>[https://stac-extensions.github.io/eo/v1.0.0/s...</td>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T36WWD_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\r\\x00\\x...</td>\n",
       "      <td>[32.999443871575316, 70.21094250231056, 35.756...</td>\n",
       "      <td>[{'href': 'https://planetarycomputer.microsoft...</td>\n",
       "      <td>{'AOT': {'gsd': 10.0, 'href': 'https://sentine...</td>\n",
       "      <td>sentinel-2-l2a</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>39.018765</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001485</td>\n",
       "      <td>0.0</td>\n",
       "      <td>97.629422</td>\n",
       "      <td>0.967449</td>\n",
       "      <td>1.861097</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      type stac_version                                    stac_extensions  \\\n",
       "0  Feature        1.0.0  [https://stac-extensions.github.io/eo/v1.0.0/s...   \n",
       "1  Feature        1.0.0  [https://stac-extensions.github.io/eo/v1.0.0/s...   \n",
       "2  Feature        1.0.0  [https://stac-extensions.github.io/eo/v1.0.0/s...   \n",
       "3  Feature        1.0.0  [https://stac-extensions.github.io/eo/v1.0.0/s...   \n",
       "4  Feature        1.0.0  [https://stac-extensions.github.io/eo/v1.0.0/s...   \n",
       "\n",
       "                                                  id  \\\n",
       "0  S2A_MSIL2A_20150704T101006_R022_T35XQA_2021041...   \n",
       "1  S2A_MSIL2A_20150704T101006_R022_T32TMM_2021041...   \n",
       "2  S2A_MSIL2A_20150704T101006_R022_T32TMN_2021041...   \n",
       "3  S2A_MSIL2A_20150704T101006_R022_T36WWC_2021041...   \n",
       "4  S2A_MSIL2A_20150704T101006_R022_T36WWD_2021041...   \n",
       "\n",
       "                                            geometry  \\\n",
       "0  b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x1e\\x00...   \n",
       "1  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0e\\x00...   \n",
       "2  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...   \n",
       "3  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\t\\x00\\x...   \n",
       "4  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\r\\x00\\x...   \n",
       "\n",
       "                                                bbox  \\\n",
       "0  [32.782321671748434, 71.80556197526644, 36.189...   \n",
       "1  [8.479220766509275, 41.46069640920803, 9.11869...   \n",
       "2  [8.765144061005621, 42.36232117367103, 9.12043...   \n",
       "3  [32.99946824095416, 69.68910488955622, 34.0680...   \n",
       "4  [32.999443871575316, 70.21094250231056, 35.756...   \n",
       "\n",
       "                                               links  \\\n",
       "0  [{'href': 'https://planetarycomputer.microsoft...   \n",
       "1  [{'href': 'https://planetarycomputer.microsoft...   \n",
       "2  [{'href': 'https://planetarycomputer.microsoft...   \n",
       "3  [{'href': 'https://planetarycomputer.microsoft...   \n",
       "4  [{'href': 'https://planetarycomputer.microsoft...   \n",
       "\n",
       "                                              assets      collection  \\\n",
       "0  {'AOT': {'gsd': 10.0, 'href': 'https://sentine...  sentinel-2-l2a   \n",
       "1  {'AOT': {'gsd': 10.0, 'href': 'https://sentine...  sentinel-2-l2a   \n",
       "2  {'AOT': {'gsd': 10.0, 'href': 'https://sentine...  sentinel-2-l2a   \n",
       "3  {'AOT': {'gsd': 10.0, 'href': 'https://sentine...  sentinel-2-l2a   \n",
       "4  {'AOT': {'gsd': 10.0, 'href': 'https://sentine...  sentinel-2-l2a   \n",
       "\n",
       "                          datetime  ... s2:cloud_shadow_percentage  \\\n",
       "0 2015-07-04 10:10:06.027000+00:00  ...                   0.000000   \n",
       "1 2015-07-04 10:10:06.027000+00:00  ...                   0.009120   \n",
       "2 2015-07-04 10:10:06.027000+00:00  ...                   0.001858   \n",
       "3 2015-07-04 10:10:06.027000+00:00  ...                   0.000432   \n",
       "4 2015-07-04 10:10:06.027000+00:00  ...                   0.000000   \n",
       "\n",
       "   s2:nodata_pixel_percentage s2:unclassified_percentage  \\\n",
       "0                   20.722227                   0.000000   \n",
       "1                   63.654864                   0.265107   \n",
       "2                   85.355914                   0.437180   \n",
       "3                   88.475966                   0.323290   \n",
       "4                   39.018765                   0.000000   \n",
       "\n",
       "  s2:dark_features_percentage s2:not_vegetated_percentage  \\\n",
       "0                    0.000000                    0.000000   \n",
       "1                    0.215319                   13.662070   \n",
       "2                    0.223688                   19.642879   \n",
       "3                    0.006248                    0.008810   \n",
       "4                    0.000000                    0.001485   \n",
       "\n",
       "  s2:degraded_msi_data_percentage  s2:high_proba_clouds_percentage  \\\n",
       "0                             0.0                        92.546540   \n",
       "1                             0.0                         0.048035   \n",
       "2                             0.0                         0.011238   \n",
       "3                             0.0                        65.812266   \n",
       "4                             0.0                        97.629422   \n",
       "\n",
       "  s2:reflectance_conversion_factor s2:medium_proba_clouds_percentage  \\\n",
       "0                         0.967449                          4.807670   \n",
       "1                         0.967449                          0.051376   \n",
       "2                         0.967449                          0.022928   \n",
       "3                         0.967449                         19.050561   \n",
       "4                         0.967449                          1.861097   \n",
       "\n",
       "  s2:saturated_defective_pixel_percentage  \n",
       "0                                     0.0  \n",
       "1                                     0.0  \n",
       "2                                     0.0  \n",
       "3                                     0.0  \n",
       "4                                     0.0  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = dd.read_parquet(uris, storage_options={\"account_name\": \"pcstacitems\", \"credential\": planetary_computer.sas.get_token(\"pcstacitems\", \"items\").token})\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9ef20f08-a13b-4da6-a8b1-1e7762efe250",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df = df[['id', 'geometry', 'datetime', 'eo:cloud_cover']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a9927a04-a3d7-47ef-bd47-db92c0c45157",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>geometry</th>\n",
       "      <th>datetime</th>\n",
       "      <th>eo:cloud_cover</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T35XQA_2021041...</td>\n",
       "      <td>b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x1e\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>97.851394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T32TMM_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0e\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.177088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T32TMN_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.034959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T36WWC_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\t\\x00\\x...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>88.028410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T36WWD_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\r\\x00\\x...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>99.703225</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  id  \\\n",
       "0  S2A_MSIL2A_20150704T101006_R022_T35XQA_2021041...   \n",
       "1  S2A_MSIL2A_20150704T101006_R022_T32TMM_2021041...   \n",
       "2  S2A_MSIL2A_20150704T101006_R022_T32TMN_2021041...   \n",
       "3  S2A_MSIL2A_20150704T101006_R022_T36WWC_2021041...   \n",
       "4  S2A_MSIL2A_20150704T101006_R022_T36WWD_2021041...   \n",
       "\n",
       "                                            geometry  \\\n",
       "0  b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x1e\\x00...   \n",
       "1  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0e\\x00...   \n",
       "2  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...   \n",
       "3  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\t\\x00\\x...   \n",
       "4  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\r\\x00\\x...   \n",
       "\n",
       "                          datetime  eo:cloud_cover  \n",
       "0 2015-07-04 10:10:06.027000+00:00       97.851394  \n",
       "1 2015-07-04 10:10:06.027000+00:00        0.177088  \n",
       "2 2015-07-04 10:10:06.027000+00:00        0.034959  \n",
       "3 2015-07-04 10:10:06.027000+00:00       88.028410  \n",
       "4 2015-07-04 10:10:06.027000+00:00       99.703225  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "abe0d57e-2af2-423b-8ba4-ec3fe18ac87e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "mask = (df[\"eo:cloud_cover\"] < 20)\n",
    "df = df[mask]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "efafc7eb-ffe3-41d6-98a5-88abdd1c51ef",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>geometry</th>\n",
       "      <th>datetime</th>\n",
       "      <th>eo:cloud_cover</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T32TMM_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0e\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.177088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T32TMN_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.034959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T31RGL_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x05\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>1.548134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T31RGM_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x05\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.002084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T31RGN_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  id  \\\n",
       "1  S2A_MSIL2A_20150704T101006_R022_T32TMM_2021041...   \n",
       "2  S2A_MSIL2A_20150704T101006_R022_T32TMN_2021041...   \n",
       "7  S2A_MSIL2A_20150704T101006_R022_T31RGL_2021041...   \n",
       "8  S2A_MSIL2A_20150704T101006_R022_T31RGM_2021041...   \n",
       "9  S2A_MSIL2A_20150704T101006_R022_T31RGN_2021041...   \n",
       "\n",
       "                                            geometry  \\\n",
       "1  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0e\\x00...   \n",
       "2  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...   \n",
       "7  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x05\\x00...   \n",
       "8  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x05\\x00...   \n",
       "9  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...   \n",
       "\n",
       "                          datetime  eo:cloud_cover  \n",
       "1 2015-07-04 10:10:06.027000+00:00        0.177088  \n",
       "2 2015-07-04 10:10:06.027000+00:00        0.034959  \n",
       "7 2015-07-04 10:10:06.027000+00:00        1.548134  \n",
       "8 2015-07-04 10:10:06.027000+00:00        0.002084  \n",
       "9 2015-07-04 10:10:06.027000+00:00        0.000000  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5d19a3a0-0c6f-42e6-a100-27252131a8fb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "df[\"datetime\"] = dd.to_datetime(df[\"datetime\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "015f6a84-f09b-4665-87c3-a0d358286e31",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>geometry</th>\n",
       "      <th>datetime</th>\n",
       "      <th>eo:cloud_cover</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T32TMM_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0e\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.177088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T32TMN_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.034959</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T31RGL_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x05\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>1.548134</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T31RGM_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x05\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.002084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>S2A_MSIL2A_20150704T101006_R022_T31RGN_2021041...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...</td>\n",
       "      <td>2015-07-04 10:10:06.027000+00:00</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  id  \\\n",
       "1  S2A_MSIL2A_20150704T101006_R022_T32TMM_2021041...   \n",
       "2  S2A_MSIL2A_20150704T101006_R022_T32TMN_2021041...   \n",
       "7  S2A_MSIL2A_20150704T101006_R022_T31RGL_2021041...   \n",
       "8  S2A_MSIL2A_20150704T101006_R022_T31RGM_2021041...   \n",
       "9  S2A_MSIL2A_20150704T101006_R022_T31RGN_2021041...   \n",
       "\n",
       "                                            geometry  \\\n",
       "1  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0e\\x00...   \n",
       "2  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...   \n",
       "7  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x05\\x00...   \n",
       "8  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x05\\x00...   \n",
       "9  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0c\\x00...   \n",
       "\n",
       "                          datetime  eo:cloud_cover  \n",
       "1 2015-07-04 10:10:06.027000+00:00        0.177088  \n",
       "2 2015-07-04 10:10:06.027000+00:00        0.034959  \n",
       "7 2015-07-04 10:10:06.027000+00:00        1.548134  \n",
       "8 2015-07-04 10:10:06.027000+00:00        0.002084  \n",
       "9 2015-07-04 10:10:06.027000+00:00        0.000000  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3188a232-a59e-4906-a56d-2d8ff40c73f3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "filtered_df = df[df['datetime'] > '2021-01-01']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a9d2c6d2-0ad6-4cd4-8495-5a8cf3780bf4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "filtered_df = filtered_df.compute()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "169afd0b-5070-4530-80c4-126feb999aba",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3859047, 4)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7db179f3-9c69-45a5-b9c1-c40e8be7026e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>geometry</th>\n",
       "      <th>datetime</th>\n",
       "      <th>eo:cloud_cover</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26779</th>\n",
       "      <td>S2A_MSIL2A_20210104T094411_R036_T32RQN_2021011...</td>\n",
       "      <td>b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0b\\x00...</td>\n",
       "      <td>2021-01-04 09:44:11.024000+00:00</td>\n",
       "      <td>2.850817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26780</th>\n",
       "      <td>S2A_MSIL2A_20210104T094411_R036_T32RQP_2021010...</td>\n",
       "      <td>b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0b\\x00...</td>\n",
       "      <td>2021-01-04 09:44:11.024000+00:00</td>\n",
       "      <td>0.248822</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26785</th>\n",
       "      <td>S2A_MSIL2A_20210104T094411_R036_T33SXS_2021010...</td>\n",
       "      <td>b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0b\\x00...</td>\n",
       "      <td>2021-01-04 09:44:11.024000+00:00</td>\n",
       "      <td>7.399789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26786</th>\n",
       "      <td>S2A_MSIL2A_20210104T094411_R036_T33SXT_2021011...</td>\n",
       "      <td>b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0b\\x00...</td>\n",
       "      <td>2021-01-04 09:44:11.024000+00:00</td>\n",
       "      <td>3.925122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26787</th>\n",
       "      <td>S2A_MSIL2A_20210104T094411_R036_T33SXU_2021010...</td>\n",
       "      <td>b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x07\\x00...</td>\n",
       "      <td>2021-01-04 09:44:11.024000+00:00</td>\n",
       "      <td>1.881726</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      id  \\\n",
       "26779  S2A_MSIL2A_20210104T094411_R036_T32RQN_2021011...   \n",
       "26780  S2A_MSIL2A_20210104T094411_R036_T32RQP_2021010...   \n",
       "26785  S2A_MSIL2A_20210104T094411_R036_T33SXS_2021010...   \n",
       "26786  S2A_MSIL2A_20210104T094411_R036_T33SXT_2021011...   \n",
       "26787  S2A_MSIL2A_20210104T094411_R036_T33SXU_2021010...   \n",
       "\n",
       "                                                geometry  \\\n",
       "26779  b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0b\\x00...   \n",
       "26780  b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0b\\x00...   \n",
       "26785  b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0b\\x00...   \n",
       "26786  b\"\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x0b\\x00...   \n",
       "26787  b'\\x01\\x03\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x07\\x00...   \n",
       "\n",
       "                              datetime  eo:cloud_cover  \n",
       "26779 2021-01-04 09:44:11.024000+00:00        2.850817  \n",
       "26780 2021-01-04 09:44:11.024000+00:00        0.248822  \n",
       "26785 2021-01-04 09:44:11.024000+00:00        7.399789  \n",
       "26786 2021-01-04 09:44:11.024000+00:00        3.925122  \n",
       "26787 2021-01-04 09:44:11.024000+00:00        1.881726  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filtered_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "186075aa-d958-4a42-95b8-15ecd4b30e87",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "filtered_df.to_parquet(\"s2l2a_2_5_2024_clouds_lt_20_date_gt_2021_01_01.parquet\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}