Skip to content

Commit

Permalink
✨ Returns a list of added/deleted partitions.
Browse files Browse the repository at this point in the history
  • Loading branch information
fbriol committed Nov 17, 2023
1 parent 47e1fb7 commit 004e700
Showing 1 changed file with 13 additions and 3 deletions.
16 changes: 13 additions & 3 deletions zcollection/collection/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,7 +306,7 @@ def insert(
npartitions: int | None = None,
validate: bool = False,
**kwargs,
) -> None:
) -> Iterable[str]:
"""Insert a dataset into the collection.
Args:
Expand All @@ -333,6 +333,9 @@ def insert(
validate: Whether to validate dataset metadata before insertion
or not.
Returns:
A list of the inserted partitions.
Raises:
ValueError:
If the dataset does not match the definition of the collection.
Expand Down Expand Up @@ -407,13 +410,16 @@ def insert(
]
storage.execute_transaction(client, self.synchronizer, futures)

return (fs_utils.join_path(*((self.partition_properties.dir, ) + item))
for item, _ in partitions)

# pylint: disable=method-hidden
def drop_partitions(
self,
*,
filters: PartitionFilter = None,
timedelta: datetime.timedelta | None = None,
) -> None:
) -> Iterable[str]:
# pylint: disable=method-hidden
"""Drop the selected partitions.
Expand All @@ -424,6 +430,9 @@ def drop_partitions(
timedelta: Select the partitions created before the specified time
delta relative to the current time.
Returns:
A list of the dropped partitions.
Example:
>>> collection.drop_partitions(filters="year == 2019")
>>> collection.drop_partitions(
Expand All @@ -435,7 +444,7 @@ def drop_partitions(

# No partition selected, nothing to do.
if not folders:
return
return folders

def is_created_before(path: str, now: datetime.datetime,
timedelta: datetime.timedelta) -> bool:
Expand Down Expand Up @@ -463,6 +472,7 @@ def invalidate_cache(path) -> None:
self.fs.invalidate_cache(path)

tuple(map(invalidate_cache, folders))
return folders

# pylint: disable=method-hidden
def update(
Expand Down

0 comments on commit 004e700

Please sign in to comment.