Mirrors/the-algorithm-ml
2
0
Fork 0
mirror of https://github.com/twitter/the-algorithm-ml.git synced 2024-12-23 14:51:49 +01:00
Code Issues Packages Projects Releases Wiki Activity

Removed unused parameter from the get_readers() and removed unused import LocalFileSystem

Browse Source
This commit is contained in:
Faeez Shaikh 2023-04-01 16:14:59 -05:00
parent 78c3235eee
commit cb1d1190f1
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
reader/dataset.py
View File

@ -9,7 +9,7 @@ import functools
import random
from typing import Optional
from fsspec.implementations.local import LocalFileSystem
from fsspec.implementations.local
import pyarrow.dataset as pads
import pyarrow as pa
import pyarrow.parquet
@ -105,7 +105,7 @@ class Dataset(torch.utils.data.IterableDataset):
def dataloader(self, remote: bool = False):
if not remote:
return map(self.pa_to_batch, self.to_batches())
readers = get_readers(2)
readers = get_readers()
return map(self.pa_to_batch, reader_utils.roundrobin(*readers))
@ -116,7 +116,7 @@ GRPC_OPTIONS = [
]
def get_readers(num_readers_per_worker: int):
def get_readers():
addresses = env.get_flight_server_addresses()
readers = []