beamds.beam.data package

Submodules

beamds.beam.data.beam_data module

class beamds.beam.data.beam_data.BeamData(*args, data=None, path=None, name=None, all_paths=None, index=None, label=None, columns=None, lazy=True, device=None, target_device=None, schema=None, override=False, compress=None, split_by='keys', chunksize=1000000000, chunklen=None, n_chunks=None, key_map=None, partition=None, archive_size=1000000, preferred_orientation='index', read_kwargs=None, write_kwargs=None, quick_getitem=False, orientation=None, glob_filter=None, info=None, synced=False, write_metadata=True, read_metadata=True, metadata_path_prefix=None, key_fold_map=None, chunksize_policy='round', **kwargs)

Bases: BeamName

abs_all_paths(all_paths=None, root_path=None)

property all_paths

apply(func, *args, preferred_orientation='columns', **kwargs)

as_numpy()

Convert the data to numpy in place @return:

as_tensor(device=None, dtype=None, return_vector=False)

Convert the data to tensor in place @param device: @param dtype: @param return_vector: @return:

cache(path=None, all_paths=None, schema=None, update=False, in_place=True, **kwargs)

cached(*args, **kwargs)

clear_index()

clear_label()

static clear_metadata(path)

clone(*args, data=None, path=None, all_paths=None, key_map=None, index=None, label=None, columns=None, schema=None, orientation=None, info=None, constructor=None, key_fold_map=None, **kwargs)

static collate(*args, batch=None, split_by=None, **kwargs)

columns_chunk_file_extension = '.columns_chunk'

property columns_map

columns_partition_directory_name = '.columns_part'

static concat(bds, dim=0)

concatenate_values(data=None, orientation=None, objects_type=None)

property conf

static containerize_keys_and_values(keys, values)

static data_batch(data, index=None, label=None, orientation=None, info=None, flatten_index=False, flatten_label=False)

property data_slicer

property data_type

property data_types

default_data_file_name = 'data_container'

property device

divide_chunks(**kwargs)

property dtypes

static exists(paths)

property flatten_data

property flatten_items

classmethod from_indexed_pandas(data, *args, **kwargs)

classmethod from_path(path, *args, **kwargs)

get_default_params(*args, **kwargs)

Get default parameters from the class

@param args: @param kwargs: @return:

get_index_by_key_fold_map(keys, keys_type=None)

get_info_groups()

static get_n_chunks(data, n_chunks=None, chunklen=None, chunksize=None, size=None, chunksize_policy='round')

static get_schema_from_subset(schema, key, schema_type=None)

static get_schema_from_tupled_key(schema, key, schema_type=None)

property has_index

property has_label

property hash

head(n=20)

hierarchical_keys(recursive=False)

property index

index_chunk_file_extension = '.index_chunk'

index_partition_directory_name = '.index_part'

property index_slicer

property index_type

property info

inverse_columns_map(columns)

inverse_map(ind)

items(level=1)

property key_map

keys(level=1)

property label

property label_slicer

property label_type

classmethod load_state_dict(state_dict)

metadata_files = {'all_paths': '.all_paths.pkl', 'aux': '.aux.pkl', 'conf': '.conf.pkl', 'index': '.index', 'info': '.info.fea', 'label': '.label', 'schema': '.schema.pkl'}

metadata_path_exists(key)

property metadata_paths

static normalize_key(key)

property objects_type

property orientation

property parent

property path

static read(paths, schema=None, strict=False, _check_existence=False, **kwargs)

recursive_filter(x, info)

static recursive_map_path(root_path, relative_path=None, glob_filter=None)

static recursive_root_finder(all_paths, head=None)

reset_index()

reset_metadata(*args, avoid_reset=None)

property root_path

sample(n, replace=True)

property schema

property schema_type

static set_first_key(key, value)

set_property(p)

property shape

classmethod simple(*args, preferred_orientation='index', **kwargs)

property simplified

single_file_case(root_path, all_paths, metadata_paths)

property size

slice_columns(columns)

slice_data(index)

slice_index(index, index_type=None)

slice_keys(keys)

static slice_scalar_or_list(data, keys, data_type=None, keys_type=None, replace_missing=False)

property stack

property stacked_index

property stacked_labels

property stacked_values

state_dict()

store(path=None, data=None, compress=None, chunksize=None, chunklen=None, n_chunks=None, partition=None, split_by=None, archive_size=None, override=None, split=True, chunksize_policy=None, **kwargs)

to(device)

to_path(path)

property total_size

update_all_paths_file()

static update_hierarchy(root_path, all_paths)

property values

static write_file(data, path, override=True, schema=None, **kwargs)

static write_object(data, path, override=True, size=None, archive=False, compress=None, chunksize=1000000000, chunklen=None, n_chunks=None, partition=None, file_type=None, schema=None, textual_serialization=False, split_by=None, split=True, priority=None, blacklist_priority=None, chunksize_policy='round', **kwargs)

static write_tree(data, path, sizes=None, split_by='keys', archive_size=1000000, chunksize=1000000000, override=True, chunklen=None, n_chunks=None, partition=None, file_type=None, root=False, schema=None, split=False, textual_serialization=False, blacklist_priority=None, chunksize_policy='round', **kwargs)

beamds.beam.data.beam_schema module

class beamds.beam.data.beam_schema.BeamSchema(read_schema=None, write_schema=None, **kwargs)

Bases: object

property read_schema

property write_schema

beamds.beam.data.elements module

Module contents