Skip to content

Commit

Permalink
But make it 3.8 friendly.
Browse files Browse the repository at this point in the history
  • Loading branch information
delucchi-cmu committed Oct 25, 2023
1 parent 8bef9ee commit 7201d25
Show file tree
Hide file tree
Showing 14 changed files with 52 additions and 52 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ def __init__(
pixels: PixelInputTypes,
join_pixels: JoinPixelInputTypes,
catalog_path=None,
storage_options: Union[Dict[Any, Any] | None] = None,
storage_options: Union[Dict[Any, Any], None] = None,
) -> None:
if not catalog_info.catalog_type == CatalogType.ASSOCIATION:
raise ValueError("Catalog info `catalog_type` must be 'association'")
Expand Down Expand Up @@ -59,7 +59,7 @@ def _get_partition_join_info_from_pixels(

@classmethod
def _read_args(
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
) -> Tuple[CatalogInfoClass, PixelInputTypes, JoinPixelInputTypes]: # type: ignore[override]
args = super()._read_args(catalog_base_dir, storage_options=storage_options)
partition_join_info_file = paths.get_partition_join_info_pointer(catalog_base_dir)
Expand All @@ -70,7 +70,7 @@ def _read_args(

@classmethod
def _check_files_exist(
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
):
super()._check_files_exist(catalog_base_dir, storage_options=storage_options)
partition_join_info_file = paths.get_partition_join_info_pointer(catalog_base_dir)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ def _check_column_names(self):

@classmethod
def read_from_file(
cls, partition_join_info_file: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, partition_join_info_file: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
) -> Self:
"""Read partition join info from a `partition_join_info.csv` file to create an object
Expand Down
2 changes: 1 addition & 1 deletion src/hipscat/catalog/catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def __init__(
catalog_info: CatalogInfoClass,
pixels: PixelInputTypes,
catalog_path: str = None,
storage_options: Union[Dict[Any, Any] | None] = None,
storage_options: Union[Dict[Any, Any], None] = None,
) -> None:
"""Initializes a Catalog
Expand Down
2 changes: 1 addition & 1 deletion src/hipscat/catalog/dataset/base_catalog_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def __str__(self):

@classmethod
def read_from_metadata_file(
cls, catalog_info_file: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_info_file: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
) -> Self:
"""Read catalog info from the `catalog_info.json` metadata file
Expand Down
2 changes: 1 addition & 1 deletion src/hipscat/catalog/dataset/catalog_info_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def create_catalog_info(keywords: dict, catalog_type: Optional[CatalogType] = No
return ci_class(**catalog_info_keywords)


def from_catalog_dir(catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None):
def from_catalog_dir(catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any], None] = None):
"""Generate a typed catalog info object from the type specified in the
catalog info file.
Expand Down
10 changes: 5 additions & 5 deletions src/hipscat/catalog/dataset/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def __init__(
self,
catalog_info: CatalogInfoClass,
catalog_path=None,
storage_options: Union[Dict[Any, Any] | None] = None,
storage_options: Union[Dict[Any, Any], None] = None,
) -> None:
"""Initializes a Dataset
Expand All @@ -44,7 +44,7 @@ def __init__(

@classmethod
def read_from_hipscat(
cls, catalog_path: str, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_path: str, storage_options: Union[Dict[Any, Any], None] = None
) -> Self:
"""Reads a HiPSCat Catalog from a HiPSCat directory
Expand All @@ -63,7 +63,7 @@ def read_from_hipscat(

@classmethod
def _read_args(
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
) -> Tuple[CatalogInfoClass]:
catalog_info_file = paths.get_catalog_info_pointer(catalog_base_dir)
catalog_info = cls.CatalogInfoClass.read_from_metadata_file(
Expand All @@ -73,13 +73,13 @@ def _read_args(

@classmethod
def _read_kwargs(
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
) -> dict:
return {"catalog_path": str(catalog_base_dir), "storage_options": storage_options}

@classmethod
def _check_files_exist(
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
):
if not file_io.does_file_or_directory_exist(catalog_base_dir, storage_options=storage_options):
raise FileNotFoundError(f"No directory exists at {str(catalog_base_dir)}")
Expand Down
6 changes: 3 additions & 3 deletions src/hipscat/catalog/healpix_dataset/healpix_dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ def __init__(
catalog_info: CatalogInfoClass,
pixels: PixelInputTypes,
catalog_path: str = None,
storage_options: Union[Dict[Any, Any] | None] = None,
storage_options: Union[Dict[Any, Any], None] = None,
) -> None:
"""Initializes a Catalog
Expand Down Expand Up @@ -79,7 +79,7 @@ def _get_pixel_tree_from_pixels(pixels: PixelInputTypes) -> PixelTree:

@classmethod
def _read_args(
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
) -> Tuple[CatalogInfoClass, PartitionInfo]:
args = super()._read_args(catalog_base_dir, storage_options=storage_options)
partition_info_file = paths.get_partition_info_pointer(catalog_base_dir)
Expand All @@ -88,7 +88,7 @@ def _read_args(

@classmethod
def _check_files_exist(
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_base_dir: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
):
super()._check_files_exist(catalog_base_dir, storage_options=storage_options)
partition_info_file = paths.get_partition_info_pointer(catalog_base_dir)
Expand Down
2 changes: 1 addition & 1 deletion src/hipscat/catalog/partition_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ def write_to_file(self, partition_info_file: FilePointer):

@classmethod
def read_from_file(
cls, partition_info_file: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
cls, partition_info_file: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
):
"""Read partition info from a `partition_info.csv` file to create an object
Expand Down
2 changes: 1 addition & 1 deletion src/hipscat/inspection/almanac.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ class Almanac:
"""

def __init__(
self, include_default_dir=True, dirs=None, storage_options: Union[Dict[Any, Any] | None] = None
self, include_default_dir=True, dirs=None, storage_options: Union[Dict[Any, Any], None] = None
):
"""Create new almanac."""
self.files = {}
Expand Down
8 changes: 4 additions & 4 deletions src/hipscat/inspection/almanac_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ class AlmanacInfo:
"""Container for parsed almanac information."""

file_path: str = ""
storage_options: Union[Dict[Any, Any] | None] = field(default_factory=dict)
storage_options: Union[Dict[Any, Any], None] = field(default_factory=dict)
namespace: str = ""
catalog_path: str = ""
catalog_name: str = ""
Expand Down Expand Up @@ -76,7 +76,7 @@ def get_default_dir() -> str:

@classmethod
def from_catalog_dir(
cls, catalog_base_dir: str, storage_options: Union[Dict[Any, Any] | None] = None
cls, catalog_base_dir: str, storage_options: Union[Dict[Any, Any], None] = None
) -> Self:
"""Create almanac information from the catalog information found at the target directory"""
catalog_info = catalog_info_factory.from_catalog_dir(
Expand All @@ -93,7 +93,7 @@ def from_catalog_dir(
return cls(**args)

@classmethod
def from_file(cls, file: str, storage_options: Union[Dict[Any, Any] | None] = None) -> Self:
def from_file(cls, file: str, storage_options: Union[Dict[Any, Any], None] = None) -> Self:
"""Create almanac information from an almanac file."""
_, fmt = os.path.splitext(file)
if fmt != ".yml":
Expand All @@ -106,7 +106,7 @@ def write_to_file(
directory=None,
default_dir=True,
fmt="yml",
storage_options: Union[Dict[Any, Any] | None] = None,
storage_options: Union[Dict[Any, Any], None] = None,
):
"""Write the almanac to an almanac file"""
if default_dir and directory:
Expand Down
2 changes: 1 addition & 1 deletion src/hipscat/inspection/visualize_catalog.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from hipscat.io import file_io, paths


def _read_point_map(catalog_base_dir, storage_options: Union[Dict[Any, Any] | None] = None):
def _read_point_map(catalog_base_dir, storage_options: Union[Dict[Any, Any], None] = None):
"""Read the object spatial distribution information from a healpix FITS file.
Args:
Expand Down
36 changes: 18 additions & 18 deletions src/hipscat/io/file_io/file_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@


def make_directory(
file_pointer: FilePointer, exist_ok: bool = False, storage_options: Union[Dict[Any, Any] | None] = None
file_pointer: FilePointer, exist_ok: bool = False, storage_options: Union[Dict[Any, Any], None] = None
):
"""Make a directory at a given file pointer
Expand All @@ -36,7 +36,7 @@ def make_directory(


def remove_directory(
file_pointer: FilePointer, ignore_errors=False, storage_options: Union[Dict[Any, Any] | None] = None
file_pointer: FilePointer, ignore_errors=False, storage_options: Union[Dict[Any, Any], None] = None
):
"""Remove a directory, and all contents, recursively.
Expand All @@ -61,7 +61,7 @@ def write_string_to_file(
file_pointer: FilePointer,
string: str,
encoding: str = "utf-8",
storage_options: Union[Dict[Any, Any] | None] = None,
storage_options: Union[Dict[Any, Any], None] = None,
):
"""Write a string to a text file
Expand All @@ -77,7 +77,7 @@ def write_string_to_file(


def load_text_file(
file_pointer: FilePointer, encoding: str = "utf-8", storage_options: Union[Dict[Any, Any] | None] = None
file_pointer: FilePointer, encoding: str = "utf-8", storage_options: Union[Dict[Any, Any], None] = None
):
"""Load a text file content to a list of strings.
Expand All @@ -96,7 +96,7 @@ def load_text_file(


def load_json_file(
file_pointer: FilePointer, encoding: str = "utf-8", storage_options: Union[Dict[Any, Any] | None] = None
file_pointer: FilePointer, encoding: str = "utf-8", storage_options: Union[Dict[Any, Any], None] = None
) -> dict:
"""Load a json file to a dictionary
Expand All @@ -117,7 +117,7 @@ def load_json_file(


def load_csv_to_pandas(
file_pointer: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None, **kwargs
file_pointer: FilePointer, storage_options: Union[Dict[Any, Any], None] = None, **kwargs
) -> pd.DataFrame:
"""Load a csv file to a pandas dataframe
Expand All @@ -132,7 +132,7 @@ def load_csv_to_pandas(


def load_parquet_to_pandas(
file_pointer: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None, **kwargs
file_pointer: FilePointer, storage_options: Union[Dict[Any, Any], None] = None, **kwargs
) -> pd.DataFrame:
"""Load a parquet file to a pandas dataframe
Expand All @@ -149,7 +149,7 @@ def load_parquet_to_pandas(
def write_dataframe_to_csv(
dataframe: pd.DataFrame,
file_pointer: FilePointer,
storage_options: Union[Dict[Any, Any] | None] = None,
storage_options: Union[Dict[Any, Any], None] = None,
**kwargs,
):
"""Write a pandas DataFrame to a CSV file
Expand All @@ -165,7 +165,7 @@ def write_dataframe_to_csv(


def write_dataframe_to_parquet(
dataframe: pd.DataFrame, file_pointer, storage_options: Union[Dict[Any, Any] | None] = None
dataframe: pd.DataFrame, file_pointer, storage_options: Union[Dict[Any, Any], None] = None
):
"""Write a pandas DataFrame to a parquet file
Expand All @@ -178,7 +178,7 @@ def write_dataframe_to_parquet(


def read_parquet_metadata(
file_pointer: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None, **kwargs
file_pointer: FilePointer, storage_options: Union[Dict[Any, Any], None] = None, **kwargs
) -> pq.FileMetaData:
"""Read FileMetaData from footer of a single Parquet file.
Expand All @@ -195,7 +195,7 @@ def read_parquet_metadata(
return parquet_file


def read_parquet_dataset(dir_pointer: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None):
def read_parquet_dataset(dir_pointer: FilePointer, storage_options: Union[Dict[Any, Any], None] = None):
"""Read parquet dataset from directory pointer.
Args:
Expand Down Expand Up @@ -224,7 +224,7 @@ def read_parquet_dataset(dir_pointer: FilePointer, storage_options: Union[Dict[A
return dataset


def read_parquet_file(file_pointer: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None):
def read_parquet_file(file_pointer: FilePointer, storage_options: Union[Dict[Any, Any], None] = None):
"""Read parquet file from file pointer.
Args:
Expand All @@ -239,7 +239,7 @@ def write_parquet_metadata(
schema: Any,
file_pointer: FilePointer,
metadata_collector: list | None = None,
storage_options: Union[Dict[Any, Any] | None] = None,
storage_options: Union[Dict[Any, Any], None] = None,
**kwargs,
):
"""Write a metadata only parquet file from a schema
Expand All @@ -260,7 +260,7 @@ def write_parquet_metadata(
)


def read_fits_image(map_file_pointer: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None):
def read_fits_image(map_file_pointer: FilePointer, storage_options: Union[Dict[Any, Any], None] = None):
"""Read the object spatial distribution information from a healpix FITS file.
Args:
Expand All @@ -280,7 +280,7 @@ def read_fits_image(map_file_pointer: FilePointer, storage_options: Union[Dict[A


def write_fits_image(
histogram: np.ndarray, map_file_pointer: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None
histogram: np.ndarray, map_file_pointer: FilePointer, storage_options: Union[Dict[Any, Any], None] = None
):
"""Write the object spatial distribution information to a healpix FITS file.
Expand All @@ -297,7 +297,7 @@ def write_fits_image(
_map_file.write(_tmp_file.read())


def read_yaml(file_handle: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None):
def read_yaml(file_handle: FilePointer, storage_options: Union[Dict[Any, Any], None] = None):
"""Reads yaml file from filesystem.
Args:
Expand All @@ -310,7 +310,7 @@ def read_yaml(file_handle: FilePointer, storage_options: Union[Dict[Any, Any] |
return metadata


def delete_file(file_handle: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None):
def delete_file(file_handle: FilePointer, storage_options: Union[Dict[Any, Any], None] = None):
"""Deletes file from filesystem.
Args:
Expand All @@ -322,7 +322,7 @@ def delete_file(file_handle: FilePointer, storage_options: Union[Dict[Any, Any]


def read_parquet_file_to_pandas(
file_pointer: FilePointer, storage_options: Union[Dict[Any, Any] | None] = None, **kwargs
file_pointer: FilePointer, storage_options: Union[Dict[Any, Any], None] = None, **kwargs
) -> pd.DataFrame:
"""Reads a parquet file to a pandas DataFrame
Expand Down
Loading

0 comments on commit 7201d25

Please sign in to comment.