Environment variables¶
This notebook reads these environment variables. Set them before you start Jupyter:
AZURE_STORAGE_ACCOUNT_NAME: your Azure Storage account name.AZURE_STORAGE_ACCOUNT_KEY: an access key for that storage account.AZURE_TENANT_ID: your Microsoft Entra (Azure AD) tenant id.AZURE_CLIENT_ID: the application (client) id of your app registration or service principal.AZURE_CLIENT_SECRET: a client secret for that service principal.
Example 1: Public data sets (No auth necessary)¶
Access the GOES dataset and list contents in a flder
In [ ]:
Copied!
from fsspec import filesystem
storage_options = {'account_name': 'goes'}
az = filesystem("az", **storage_options) # Note: This is the same as abfs = filesystem("abfs") -> (see https://github.com/fsspec/adlfs/pull/111)
az.ls('noaa-goes16')
from fsspec import filesystem
storage_options = {'account_name': 'goes'}
az = filesystem("az", **storage_options) # Note: This is the same as abfs = filesystem("abfs") -> (see https://github.com/fsspec/adlfs/pull/111)
az.ls('noaa-goes16')
In [ ]:
Copied!
# Alternatively
from adlfs import AzureBlobFileSystem
storage_options = {'account_name': 'goes'}
abfs = AzureBlobFileSystem(**storage_options)
abfs.ls("noaa-goes16")
# Alternatively
from adlfs import AzureBlobFileSystem
storage_options = {'account_name': 'goes'}
abfs = AzureBlobFileSystem(**storage_options)
abfs.ls("noaa-goes16")
Example 2: Public data sets (No auth necessary)¶
Access the AzureOpenDataStorage datasets and list contents in a folder
In [ ]:
Copied!
from adlfs import AzureBlobFileSystem
storage_options = {'account_name': 'azureopendatastorage'}
abfs = AzureBlobFileSystem(**storage_options)
abfs.ls("nyctlc/green/puYear=2019")
from adlfs import AzureBlobFileSystem
storage_options = {'account_name': 'azureopendatastorage'}
abfs = AzureBlobFileSystem(**storage_options)
abfs.ls("nyctlc/green/puYear=2019")
In [ ]:
Copied!
# Alternatively
import dask.dataframe as dd
storage_options = {'account_name': 'azureopendatastorage'}
ddf = dd.read_parquet('az://nyctlc/green/puYear=2019/puMonth=*/*.parquet', storage_options=storage_options) # (same as abfs://...)
ddf.head()
# Alternatively
import dask.dataframe as dd
storage_options = {'account_name': 'azureopendatastorage'}
ddf = dd.read_parquet('az://nyctlc/green/puYear=2019/puMonth=*/*.parquet', storage_options=storage_options) # (same as abfs://...)
ddf.head()
Example 3: Private data sets (Authentication with an account_key)¶
Access a private account and list the contents
In [ ]:
Copied!
import os
from adlfs import AzureBlobFileSystem
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
account_key = os.environ["AZURE_STORAGE_ACCOUNT_KEY"]
abfs = AzureBlobFileSystem(account_name=account_name, account_key=account_key)
abfs.ls('')
import os
from adlfs import AzureBlobFileSystem
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
account_key = os.environ["AZURE_STORAGE_ACCOUNT_KEY"]
abfs = AzureBlobFileSystem(account_name=account_name, account_key=account_key)
abfs.ls('')
In [ ]:
Copied!
# Alternatively
import os
from fsspec import filesystem
storage_options = {'account_name': os.environ["AZURE_STORAGE_ACCOUNT_NAME"], 'account_key': os.environ["AZURE_STORAGE_ACCOUNT_KEY"]}
az = filesystem("az", **storage_options)
az.ls('')
# Alternatively
import os
from fsspec import filesystem
storage_options = {'account_name': os.environ["AZURE_STORAGE_ACCOUNT_NAME"], 'account_key': os.environ["AZURE_STORAGE_ACCOUNT_KEY"]}
az = filesystem("az", **storage_options)
az.ls('')
Example 4: Private data sets (Authentication with an Azure ServicePrincipal)¶
Access a private account and list the contents
In [ ]:
Copied!
import os
from adlfs import AzureBlobFileSystem
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
tenant_id = os.environ["AZURE_TENANT_ID"]
client_id = os.environ["AZURE_CLIENT_ID"]
client_secret = os.environ["AZURE_CLIENT_SECRET"]
abfs = AzureBlobFileSystem(account_name=account_name, tenant_id=tenant_id, client_id=client_id, client_secret=client_secret)
abfs.ls('')
import os
from adlfs import AzureBlobFileSystem
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
tenant_id = os.environ["AZURE_TENANT_ID"]
client_id = os.environ["AZURE_CLIENT_ID"]
client_secret = os.environ["AZURE_CLIENT_SECRET"]
abfs = AzureBlobFileSystem(account_name=account_name, tenant_id=tenant_id, client_id=client_id, client_secret=client_secret)
abfs.ls('')
Example 5: Private data sets (Authentication with DefaultAzureCredential)¶
Access a private account and list the contents
In [ ]:
Copied!
import os
from azure.identity.aio import DefaultAzureCredential
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
abfs = AzureBlobFileSystem(account_name=account_name, anon=False)
abfs.ls('')
import os
from azure.identity.aio import DefaultAzureCredential
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
abfs = AzureBlobFileSystem(account_name=account_name, anon=False)
abfs.ls('')
In [ ]:
Copied!
# Alternatively, using dask
import os
import dask.dataframe as dd
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
tenant_id = os.environ["AZURE_TENANT_ID"]
client_id = os.environ["AZURE_CLIENT_ID"]
client_secret = os.environ["AZURE_CLIENT_SECRET"]
# Read files as a dask dataframe
ddf = dd.read_csv('abfs://container_name/folder/*.csv', storage_options={'account_name': account_name, 'tenant_id': tenant_id, 'client_id': client_id, 'client_secret': client_secret})
# Alternatively, using dask
import os
import dask.dataframe as dd
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
tenant_id = os.environ["AZURE_TENANT_ID"]
client_id = os.environ["AZURE_CLIENT_ID"]
client_secret = os.environ["AZURE_CLIENT_SECRET"]
# Read files as a dask dataframe
ddf = dd.read_csv('abfs://container_name/folder/*.csv', storage_options={'account_name': account_name, 'tenant_id': tenant_id, 'client_id': client_id, 'client_secret': client_secret})
In [ ]:
Copied!
import os
# Alternateively, Sharded csv files can be read as:
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
account_key = os.environ["AZURE_STORAGE_ACCOUNT_KEY"]
ddf = dd.read_csv('abfs://container_name/folder/*.csv', storage_options={'account_name': account_name, 'account_key': account_key})
import os
# Alternateively, Sharded csv files can be read as:
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
account_key = os.environ["AZURE_STORAGE_ACCOUNT_KEY"]
ddf = dd.read_csv('abfs://container_name/folder/*.csv', storage_options={'account_name': account_name, 'account_key': account_key})
In [ ]:
Copied!
import os
# Alternateively, Sharded Parquet files can be read as:
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
account_key = os.environ["AZURE_STORAGE_ACCOUNT_KEY"]
ddf = dd.read_parquet('abfs://container_name/folder.parquet', storage_options={'account_name': account_name, 'account_key': account_key})
import os
# Alternateively, Sharded Parquet files can be read as:
account_name = os.environ["AZURE_STORAGE_ACCOUNT_NAME"]
account_key = os.environ["AZURE_STORAGE_ACCOUNT_KEY"]
ddf = dd.read_parquet('abfs://container_name/folder.parquet', storage_options={'account_name': account_name, 'account_key': account_key})