Example 1¶
Fetch data synchronously (slow!)
In [ ]:
Copied!
from fsspec import filesystem
fs = filesystem("http")
url1, url2, url3 = "http://google.com", "http://bing.com", "http://eugo.io"
out = fs.cat([url1, url2, url3]) # fetches data concurrently
from fsspec import filesystem
fs = filesystem("http")
url1, url2, url3 = "http://google.com", "http://bing.com", "http://eugo.io"
out = fs.cat([url1, url2, url3]) # fetches data concurrently
Example 2¶
Fetch data asynchronously (faster!)
In [ ]:
Copied!
from fsspec import filesystem
import asyncio
fs = filesystem("http")
url1, url2, url3 = "http://google.com", "http://bing.com", "https://eugo.io"
async def work_coroutine():
fs = filesystem("http", asynchronous=True)
session = await fs.set_session() # creates client
out = await fs._cat([url1, url2, url3]) # fetches data concurrently
await session.close() # explicit destructor
asyncio.run(work_coroutine())
from fsspec import filesystem
import asyncio
fs = filesystem("http")
url1, url2, url3 = "http://google.com", "http://bing.com", "https://eugo.io"
async def work_coroutine():
fs = filesystem("http", asynchronous=True)
session = await fs.set_session() # creates client
out = await fs._cat([url1, url2, url3]) # fetches data concurrently
await session.close() # explicit destructor
asyncio.run(work_coroutine())
Example 3¶
Download remote data asynchronously
In [ ]:
Copied!
from fsspec import filesystem
from pathlib import Path
import asyncio
tmp = "/tmp/ganymede_demo"
Path(tmp).mkdir(parents=True, exist_ok=True)
url = "https://raw.githubusercontent.com/pandas-dev/pandas/main/doc/data/titanic.csv"
async def download_csv():
fs = filesystem("http", asynchronous=True)
session = await fs.set_session()
out = await fs._get(url, f"{tmp}/titanic.csv")
await session.close()
asyncio.run(download_csv())
from fsspec import filesystem
from pathlib import Path
import asyncio
tmp = "/tmp/ganymede_demo"
Path(tmp).mkdir(parents=True, exist_ok=True)
url = "https://raw.githubusercontent.com/pandas-dev/pandas/main/doc/data/titanic.csv"
async def download_csv():
fs = filesystem("http", asynchronous=True)
session = await fs.set_session()
out = await fs._get(url, f"{tmp}/titanic.csv")
await session.close()
asyncio.run(download_csv())