Mirrors/the-algorithm-ml
2
0
Fork 0
mirror of https://github.com/twitter/the-algorithm-ml.git synced 2024-12-23 14:51:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Removed unused parameter from the get_readers() and removed unused import LocalFileSystem

Browse Source
This commit is contained in:
Faeez Shaikh 2023-04-01 16:14:59 -05:00
parent 78c3235eee
commit cb1d1190f1
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
reader/dataset.py
View File

@ -9,7 +9,7 @@ import functools
import random import random
from typing import Optional from typing import Optional
from fsspec.implementations.local import LocalFileSystem from fsspec.implementations.local
import pyarrow.dataset as pads import pyarrow.dataset as pads
import pyarrow as pa import pyarrow as pa
import pyarrow.parquet import pyarrow.parquet
@ -105,7 +105,7 @@ class Dataset(torch.utils.data.IterableDataset):
def dataloader(self, remote: bool = False): def dataloader(self, remote: bool = False):
if not remote: if not remote:
return map(self.pa_to_batch, self.to_batches()) return map(self.pa_to_batch, self.to_batches())
readers = get_readers(2) readers = get_readers()
return map(self.pa_to_batch, reader_utils.roundrobin(*readers)) return map(self.pa_to_batch, reader_utils.roundrobin(*readers))
@ -116,7 +116,7 @@ GRPC_OPTIONS = [
] ]
def get_readers(num_readers_per_worker: int): def get_readers():
addresses = env.get_flight_server_addresses() addresses = env.get_flight_server_addresses()
readers = [] readers = []