Create utils.py

utils.py

@@ -0,0 +1,46 @@
+import pandas as pd
+from pathlib import Path
+from fsspec.parquet import open_parquet_file 
+import pyarrow.parquet as pq
+import pandas as pd
+from io import BytesIO
+from PIL import Image
+import holoviews as hv
+
+REPOSITORY = "Major-TOM"
+DATASETS = ['Core-S2L2A', 'Core-S2L1C']
+
+DATA_PATH = Path(__file__).parent/"data"
+
+
+def _meta_data_url(dataset='Core-S2L2A', repository=REPOSITORY):
+    return f'https://huggingface.co/datasets/{repository}/{dataset}/resolve/main/metadata.parquet'
+
+def _meta_data_path(dataset='Core-S2L2A', repository=REPOSITORY):
+    DATA_PATH.mkdir(parents=True, exist_ok=True)
+    return DATA_PATH/f"{dataset}_metadata.parquet"
+
+def get_meta_data(dataset='Core-S2L2A', repository=REPOSITORY):
+    path = _meta_data_path(dataset=dataset)
+    if not path.exists():
+        data = pd.read_parquet(_meta_data_url(dataset=dataset))
+        data.to_parquet(path)
+    data = pd.read_parquet(path)
+    
+    data["centre_easting"], data["centre_northing"] = hv.util.transform.lon_lat_to_easting_northing(data["centre_lon"], data["centre_lat"])
+
+    return data
+
+def get_image(row):
+    parquet_url = row["parquet_url"]
+    parquet_row = row["parquet_row"]
+    print(parquet_url)
+    print(parquet_row)
+    with open_parquet_file(parquet_url,columns = ["thumbnail"]) as f:
+        with pq.ParquetFile(f) as pf:
+            first_row_group = pf.read_row_group(parquet_row, columns=['thumbnail'])
+
+    stream = BytesIO(first_row_group['thumbnail'][0].as_py())
+    image = Image.open(stream)
+    return image
+