Here is some script for "openml" collection of datasets.
Hopefully one can provide something similar for other databases.
#see docs: https://docs.openml.org/Python-guide/
!pip install openml
import openml
import numpy as np
import pandas as pd
import time
# Get information on all collection of openml datasets:
datalist = openml.datasets.list_datasets(output_format="dataframe")
# select datasets by some conditions (just pandas) - we will get just 4 such datasets
datasets_selected = datalist[ (datalist.NumberOfInstances < 2550) & (datalist.NumberOfInstances > 300)& (datalist.NumberOfFeatures > 10000) & (datalist.NumberOfFeatures < 40000) & \
( datalist.NumberOfFeatures != 10937) ].sort_values(["NumberOfInstances"], ascending=False)#.head(n=20)
print(datasets_selected.shape)
# load all selected datasets and print short info:
for i in range(len(datasets_selected)):
nm = datasets_selected['name'].iloc[i]
print(nm, i )
did = int( datasets_selected['did'].iloc[i] ) # did - dataset_id
t0 = time.time()
data = openml.datasets.get_dataset(did)
X, y, categorical_indicator, attribute_names = data.get_data(
dataset_format="array", target=data.default_target_attribute )
print(X.shape, y.shape, time.time()-t0,'secs passed' )
Here is even more simple example for sklearn built-in datasets:
import numpy as np
from sklearn import datasets
import time
list_id = ['load_boston', 'load_iris', 'load_diabetes', 'load_digits', 'load_linnerud', 'load_wine' , 'load_breast_cancer'] + \
['fetch_california_housing', 'fetch_covtype', 'fetch_lfw_people', 'fetch_20newsgroups_vectorized','fetch_olivetti_faces' ]
# 'fetch_rcv1', - too long
# 'fetch_lfw_pairs' - TypeError fetch_lfw_pairs() got an unexpected keyword argument 'return_X_y
# 'fetch_kddcup99' - sometimes problem happens
for id in list_id:
print(id)
t0 = time.time()
func_load = getattr(datasets, id )
X,y = func_load(return_X_y = True)
print(id, X.shape, time.time()-t0, 'secs passed')