#include <string>
#include <cstring>
#include <typeinfo>
#include "GHdf5.hpp"
#include "GException.hpp"
#include "GTools.hpp"

Macros
#define	G_LOAD "GHdf5::load(GFilename&)"

#define	G_XML_HDF5_ENTRY "GHdf5::xml_hdf5_entry(std::string&)"

#define	G_READ_SUPERBLOCK "GHdf5::read_superblock(FILE*)"

#define	G_READ_SYMBOL_TABLE_NODE

#define	G_READ_SYMBOL_TABLE_ENTRY

#define	G_READ_GROUP

#define	G_READ_BTREE_CHUNKED

#define	G_READ_OBJECT_HEADER

#define	G_READ_OBJECT_HEADER_V1 "GHdf5::read_object_header_v1(FILE*, int&)"

#define	G_READ_OBJECT_HEADER_V2 "GHdf5::read_object_header_v2(FILE*, int&)"

#define	G_READ_OBJECT_HEADER_MESSAGE

#define	G_READ_MESSAGE_DATASPACE

#define	G_READ_MESSAGE_DATATYPE

#define	G_READ_MESSAGE_LAYOUT

#define	G_READ_MESSAGE_FILTER

#define	G_READ_MESSAGE_ATTRIBUTE

#define	G_GLOBAL_HEAP_STRING

#define	G_FREAD_DATA1 "GHdf5::fread_data(FILE*, int&)"

#define	G_FREAD_DATA2

#define	G_FREAD_DATA_CHUNK

#define	G_FREAD_INT "gammalib::hdf5::fread_int(FILE*, int&)"

#define	G_FREAD_UINT32 "gammalib::hdf5::fread_uint32(FILE*, int&)"

#define	G_FREAD_UINT64 "gammalib::hdf5::fread_uint64(FILE*, int&)"

#define	G_FREAD_STRING "gammalib::hdf5::fread_string(FILE*, int&)"

#define	G_FREAD_DATA_AS_STRING

#define	G_FREAD_ZERO "gammalib::hdf5::fread_zero(FILE*, int&)"

#define	G_DATA_TO_INT "gammalib::hdf5::data_to_int(char, GXmlElement)"

#define	G_DATA_TO_DOUBLE

#define	G_DATA_TO_STRING

#define	G_DATA_FILTER

#define	G_DATA_FILTER_BITSHUFFLE "gammalib::hdf5::data_filter_bitshuffle(std::string&, GXmlElement*)"

#define	G_XML_MSG_TYPE

#define	G_XML_MSG_ATTRIBUTE "gammalib::hdf5::xml_msg_attribute(GXmlElement*, std::string&)"

#define	G_DECOMPRESS_LZ4 "decompress_lz4(char, char, size_t&, size_t&)"

#define	G_READ_VARIABLE_LENGTH

#define	G_MEMCPY_USING_OFFSET

Functions
void	bitshuffle_decompress (char input, char output, const size_t &elements, const size_t &element_size, size_t &block_size)
	Decompress and bitshuffle data.

uint64_t	bitshuffle_read_uint64 (const char *buffer)
	Read a 64 bit unsigned integer from a buffer big endian order.

uint32_t	bitshuffle_read_uint32 (const char *buffer)
	Read a 32 bit unsigned integer from a buffer big endian order.

void	bitshuffle_elements (const char input, char output, const size_t &elements, const size_t &element_size)
	Untranspose bits within elements.

int	decompress_lz4 (const char src, char dst, const size_t &srcSize, const size_t &outputSize)
	Decompress data using the LZ4 algorithm.

size_t	read_variable_length (const uint8_t *ip, const uint8_t ilimit, const bool &initial_check)
	Read the variable-length literal or match length.

uint16_t	read_uint16 (const void *memPtr)
	Read unsigned 16 Bit integer stored as little endian.

void	memcpy_using_offset (const uint8_t srcPtr, uint8_t dstPtr, uint8_t *dstEnd, const size_t &offset)
	Memory copy.

void	memcpy_beyond8 (const uint8_t srcPtr, uint8_t dstPtr, uint8_t *dstEnd)
	Memory copy allowing to overwrite up to 8 Bytes beyond the end.

void	memory_beyond32 (const uint8_t srcPtr, uint8_t dstPtr, uint8_t *dstEnd)
	Memory copy allowing to overwrite up to 32 Bytes beyond the end.

Macro Definition Documentation

◆ G_DATA_FILTER

#define G_DATA_FILTER

Value:

"gammalib::hdf5::data_filter(std::string&, "\

"GXmlElement*)"

Definition at line 85 of file GHdf5.cpp.

Referenced by gammalib::hdf5::data_filter().

◆ G_DATA_FILTER_BITSHUFFLE

#define G_DATA_FILTER_BITSHUFFLE "gammalib::hdf5::data_filter_bitshuffle(std::string&, GXmlElement*)"

Definition at line 87 of file GHdf5.cpp.

Referenced by gammalib::hdf5::data_filter_bitshuffle().

◆ G_DATA_TO_DOUBLE

#define G_DATA_TO_DOUBLE

Value:

"gammalib::hdf5::data_to_double(char*, "\

"GXmlElement*)"

Definition at line 81 of file GHdf5.cpp.

Referenced by gammalib::hdf5::data_to_double().

◆ G_DATA_TO_INT

#define G_DATA_TO_INT "gammalib::hdf5::data_to_int(char*, GXmlElement*)"

Definition at line 80 of file GHdf5.cpp.

Referenced by gammalib::hdf5::data_to_int().

◆ G_DATA_TO_STRING

#define G_DATA_TO_STRING

Value:

"gammalib::hdf5::data_to_string(char*, "\

"GXmlElement*)"

Definition at line 83 of file GHdf5.cpp.

Referenced by gammalib::hdf5::data_to_string().

◆ G_DECOMPRESS_LZ4

#define G_DECOMPRESS_LZ4 "decompress_lz4(char*, char*, size_t&, size_t&)"

Definition at line 93 of file GHdf5.cpp.

Referenced by decompress_lz4().

◆ G_FREAD_DATA1

#define G_FREAD_DATA1 "GHdf5::fread_data(FILE*, int&)"

Definition at line 68 of file GHdf5.cpp.

Referenced by bitshuffle_decompress(), and gammalib::hdf5::fread_data().

◆ G_FREAD_DATA2

#define G_FREAD_DATA2

Value:

"gammalib::hdf5::fread_data(FILE*, GXmlElement*, "\

"GXmlElement*, GXmlElement*, GXmlElement*)"

Definition at line 69 of file GHdf5.cpp.

Referenced by gammalib::hdf5::fread_data().

◆ G_FREAD_DATA_AS_STRING

#define G_FREAD_DATA_AS_STRING

Value:

"gammalib::hdf5::fread_data_as_string(FILE*, "\

"GXmlElement*)"

Definition at line 77 of file GHdf5.cpp.

Referenced by gammalib::hdf5::fread_data_as_string().

◆ G_FREAD_DATA_CHUNK

#define G_FREAD_DATA_CHUNK

Value:

"gammalib::hdf5::fread_data_chunk(FILE*, "\

"GXmlElement*, GXmlElement*, GXmlElement*, GXmlElement*, GXmlElement*)"

Definition at line 71 of file GHdf5.cpp.

Referenced by gammalib::hdf5::fread_data_chunk().

◆ G_FREAD_INT

#define G_FREAD_INT "gammalib::hdf5::fread_int(FILE*, int&)"

Definition at line 73 of file GHdf5.cpp.

Referenced by gammalib::hdf5::fread_int().

◆ G_FREAD_STRING

#define G_FREAD_STRING "gammalib::hdf5::fread_string(FILE*, int&)"

Definition at line 76 of file GHdf5.cpp.

Referenced by gammalib::hdf5::fread_string().

◆ G_FREAD_UINT32

#define G_FREAD_UINT32 "gammalib::hdf5::fread_uint32(FILE*, int&)"

Definition at line 74 of file GHdf5.cpp.

Referenced by gammalib::hdf5::fread_uint32().

◆ G_FREAD_UINT64

#define G_FREAD_UINT64 "gammalib::hdf5::fread_uint64(FILE*, int&)"

Definition at line 75 of file GHdf5.cpp.

Referenced by gammalib::hdf5::fread_uint64().

◆ G_FREAD_ZERO

#define G_FREAD_ZERO "gammalib::hdf5::fread_zero(FILE*, int&)"

Definition at line 79 of file GHdf5.cpp.

Referenced by gammalib::hdf5::fread_zero().

◆ G_GLOBAL_HEAP_STRING

#define G_GLOBAL_HEAP_STRING

Value:

"GHdf5::global_heap_string(FILE*, uint64_t&, "\

"int&, uint32_t&)"

Definition at line 66 of file GHdf5.cpp.

Referenced by GHdf5::global_heap_string().

◆ G_LOAD

#define G_LOAD "GHdf5::load(GFilename&)"

Definition at line 39 of file GHdf5.cpp.

◆ G_MEMCPY_USING_OFFSET

#define G_MEMCPY_USING_OFFSET

Value:

"memcpy_using_offset(uint8_t*, uint8_t*, "\

"uint8_t*, size_t&)"

Definition at line 96 of file GHdf5.cpp.

Referenced by memcpy_using_offset().

◆ G_READ_BTREE_CHUNKED

#define G_READ_BTREE_CHUNKED

Value:

"GHdf5::read_btree(FILE*, "\

"uint64_t&, GXmlElement*, int& indent)"

Definition at line 48 of file GHdf5.cpp.

Referenced by GHdf5::read_btree_chunked().

◆ G_READ_GROUP

#define G_READ_GROUP

Value:

"GHdf5::read_group(FILE*, uint64_t&, uint64_t&, "\

"GXmlElement*, int&)"

Definition at line 46 of file GHdf5.cpp.

Referenced by GHdf5::read_group().

◆ G_READ_MESSAGE_ATTRIBUTE

#define G_READ_MESSAGE_ATTRIBUTE

Value:

"GHdf5::read_message_attribute(FILE*, "\

"GXmlElement*, std::string&)"

Definition at line 64 of file GHdf5.cpp.

Referenced by GHdf5::read_message_attribute().

◆ G_READ_MESSAGE_DATASPACE

#define G_READ_MESSAGE_DATASPACE

Value:

"GHdf5::read_message_dataspace(FILE*, "\

"GXmlElement*, std::string&)"

Definition at line 56 of file GHdf5.cpp.

◆ G_READ_MESSAGE_DATATYPE

#define G_READ_MESSAGE_DATATYPE

Value:

"GHdf5::read_message_datatype(FILE*, "\

"GXmlElement*, std::string&)"

Definition at line 58 of file GHdf5.cpp.

Referenced by GHdf5::read_message_datatype().

◆ G_READ_MESSAGE_FILTER

#define G_READ_MESSAGE_FILTER

Value:

"GHdf5::read_message_filter(FILE*, "\

"GXmlElement*, std::string&)"

Definition at line 62 of file GHdf5.cpp.

Referenced by GHdf5::read_message_filter().

◆ G_READ_MESSAGE_LAYOUT

#define G_READ_MESSAGE_LAYOUT

Value:

"GHdf5::read_message_layout(FILE*, "\

"GXmlElement*, std::string&)"

Definition at line 60 of file GHdf5.cpp.

Referenced by GHdf5::read_message_layout().

◆ G_READ_OBJECT_HEADER

#define G_READ_OBJECT_HEADER

Value:

"GHdf5::read_object_header(FILE*, "\

"uint64_t&, GXmlElement*, int&)"

Definition at line 50 of file GHdf5.cpp.

Referenced by GHdf5::read_object_header().

◆ G_READ_OBJECT_HEADER_MESSAGE

#define G_READ_OBJECT_HEADER_MESSAGE

Value:

"GHdf5::read_object_header_message("\

"FILE*, int&, int&)"

Definition at line 54 of file GHdf5.cpp.

◆ G_READ_OBJECT_HEADER_V1

#define G_READ_OBJECT_HEADER_V1 "GHdf5::read_object_header_v1(FILE*, int&)"

Definition at line 52 of file GHdf5.cpp.

Referenced by GHdf5::read_object_header_v1().

◆ G_READ_OBJECT_HEADER_V2

#define G_READ_OBJECT_HEADER_V2 "GHdf5::read_object_header_v2(FILE*, int&)"

Definition at line 53 of file GHdf5.cpp.

Referenced by GHdf5::read_object_header_v2().

◆ G_READ_SUPERBLOCK

#define G_READ_SUPERBLOCK "GHdf5::read_superblock(FILE*)"

Definition at line 41 of file GHdf5.cpp.

Referenced by GHdf5::read_superblock().

◆ G_READ_SYMBOL_TABLE_ENTRY

#define G_READ_SYMBOL_TABLE_ENTRY

Value:

"GHdf5::read_symbol_table_entry(FILE*, "\

"uint64_t&, GXmlElement*, int&)"

Definition at line 44 of file GHdf5.cpp.

Referenced by GHdf5::read_symbol_table_entry().

◆ G_READ_SYMBOL_TABLE_NODE

#define G_READ_SYMBOL_TABLE_NODE

Value:

"GHdf5::read_symbol_table_node(FILE*, "\

"uint64_t&, uint64_t&, std::string&, GXmlElement*, int&)"

Definition at line 42 of file GHdf5.cpp.

Referenced by GHdf5::read_symbol_table_node().

◆ G_READ_VARIABLE_LENGTH

#define G_READ_VARIABLE_LENGTH

Value:

"read_variable_length(uint8_t**, uint8_t*, "\

"bool&)"

Definition at line 94 of file GHdf5.cpp.

Referenced by read_variable_length().

◆ G_XML_HDF5_ENTRY

#define G_XML_HDF5_ENTRY "GHdf5::xml_hdf5_entry(std::string&)"

Definition at line 40 of file GHdf5.cpp.

Referenced by GHdf5::xml_hdf5_entry().

◆ G_XML_MSG_ATTRIBUTE

#define G_XML_MSG_ATTRIBUTE "gammalib::hdf5::xml_msg_attribute(GXmlElement*, std::string&)"

Definition at line 91 of file GHdf5.cpp.

Referenced by gammalib::hdf5::xml_msg_attribute().

◆ G_XML_MSG_TYPE

#define G_XML_MSG_TYPE

Value:

"gammalib::hdf5::xml_msg_type(GXmlElement*, "\

"std::string& type, int&)"

Definition at line 89 of file GHdf5.cpp.

Referenced by gammalib::hdf5::xml_msg_type().

Function Documentation

◆ bitshuffle_decompress()

void bitshuffle_decompress	(	char *	input,
		char *	output,
		const size_t &	elements,
		const size_t &	element_size,
		size_t &	block_size )

Decompress and bitshuffle data.

Parameters

[in]	input	Pointer to input buffer.
[out]	out	Pointer to output buffer (`elements` * `element_size` bytes).
[in]	elements	Number of elements in input.
[in]	element_size	Element size of typed data.
[in]	block_size	Process in blocks of this many elements.

The method was inspired by https://github.com/kiyo-masui/bitshuffle and specifically the bshuf_decompress_lz4() and bshuf_decompress_lz4_block() functions in the bitshuffle.c file and the bshuf_blocked_wrap_fun() function in the bitshuffle_core.c file.

Definition at line 4438 of file GHdf5.cpp.

References bitshuffle_elements(), bitshuffle_read_uint32(), decompress_lz4(), G_FREAD_DATA1, and gammalib::str().

Referenced by gammalib::hdf5::data_filter_bitshuffle().

◆ bitshuffle_elements()

void bitshuffle_elements	(	const char *	input,
		char *	output,
		const size_t &	elements,
		const size_t &	element_size )

Untranspose bits within elements.

Parameters

[in]	input	Pointer to input buffer.
[out]	output	Pointer to output buffer (`elements` * `element_size` bytes).
[in]	elements	Number of elements in input.
[in]	element_size	Element size of typed data.

The method was inspired by https://github.com/kiyo-masui/bitshuffle and specifically the bshuf_untrans_bit_elem_scal() function in the file bitshuffle_core.c.

Definition at line 4633 of file GHdf5.cpp.

References gammalib::little_endian().

Referenced by bitshuffle_decompress().

◆ bitshuffle_read_uint32()

uint32_t bitshuffle_read_uint32 ( const char * buffer )

Read a 32 bit unsigned integer from a buffer big endian order.

Parameters

[in] buffer Buffer.

Returns: 32 bit unsigned integer.

Reads a 32 bit unsigned integer stored in big endian order from a Byte buffer.

The method was inspired by https://github.com/kiyo-masui/bitshuffle and specifically the bshuf_read_uint32_BE() function in the bitshuffle.c file.

Definition at line 4598 of file GHdf5.cpp.

Referenced by bitshuffle_decompress(), and gammalib::hdf5::data_filter_bitshuffle().

◆ bitshuffle_read_uint64()

uint64_t bitshuffle_read_uint64 ( const char * buffer )

Read a 64 bit unsigned integer from a buffer big endian order.

Parameters

[in] buffer Buffer.

Returns: 64 bit unsigned integer.

Reads a 64 bit unsigned integer stored in big endian order from a Byte buffer.

The method was inspired by https://github.com/kiyo-masui/bitshuffle and specifically the bshuf_read_uint64_BE() function in the bitshuffle.c file.

Definition at line 4563 of file GHdf5.cpp.

Referenced by gammalib::hdf5::data_filter_bitshuffle().

◆ decompress_lz4()

int decompress_lz4	(	const char *	src,
		char *	dst,
		const size_t &	srcSize,
		const size_t &	outputSize )

Decompress data using the LZ4 algorithm.

Parameters

[in]	src	Pointer to compressed data.
[out]	dest	Pointer to decompressed data (must be already allocated).
[in]	srcSize	Size of compressed data.
[in]	outputSize	Maximum size of allocated decompressed data.

Returns: Number of decompressed Bytes.

Decompress data using the LZ4 algorithm.

The method was inspired by https://github.com/kiyo-masui/bitshuffle and specifically the LZ4_decompress_safe() function in the file lz4.c.

Definition at line 4720 of file GHdf5.cpp.

References G_DECOMPRESS_LZ4, memcpy_beyond8(), memcpy_using_offset(), memory_beyond32(), read_uint16(), and read_variable_length().

Referenced by bitshuffle_decompress().

◆ memcpy_beyond8()

void memcpy_beyond8	(	const uint8_t *	srcPtr,
		uint8_t *	dstPtr,
		uint8_t *	dstEnd )

Memory copy allowing to overwrite up to 8 Bytes beyond the end.

Parameters

[in]	srcPtr	Pointer to start of source.
[in]	dstPtr	Pointer to start of destination.
[in]	dstEnd	Pointer to end of destination.

This method is a customised variant of memcpy, which can overwrite up to 8 Bytes beyond dstEnd. This method copies 8 Bytes a time using the std::memcpy() function.

The method was inspired by https://github.com/kiyo-masui/bitshuffle and specifically the LZ4_wildCopy8() function in the file lz4.c.

Definition at line 5286 of file GHdf5.cpp.

Referenced by decompress_lz4(), and memcpy_using_offset().

◆ memcpy_using_offset()

void memcpy_using_offset	(	const uint8_t *	srcPtr,
		uint8_t *	dstPtr,
		uint8_t *	dstEnd,
		const size_t &	offset )

Memory copy.

Parameters

[in]	srcPtr	Input pointer.
[in]	dstPtr	Pointer to start of output.
[in]	dstEnd	Pointer to end of output
[in]	offset	.

Exceptions

The method was inspired by https://github.com/kiyo-masui/bitshuffle and specifically the LZ4_memcpy_using_offset() function in the file lz4.c.

Definition at line 5203 of file GHdf5.cpp.

References G_MEMCPY_USING_OFFSET, memcpy_beyond8(), and gammalib::str().

Referenced by decompress_lz4().

◆ memory_beyond32()

void memory_beyond32	(	const uint8_t *	srcPtr,
		uint8_t *	dstPtr,
		uint8_t *	dstEnd )

Memory copy allowing to overwrite up to 32 Bytes beyond the end.

Parameters

[in]	srcPtr	Pointer to start of source.
[in]	dstPtr	Pointer to start of destination.
[in]	dstEnd	Pointer to end of destination.

This method is a customised variant of memcpy, which can overwrite up to 32 Bytes beyond dstEnd. This method copies 16 Bytes a time using the std::memcpy() function.

The method was inspired by https://github.com/kiyo-masui/bitshuffle and specifically the LZ4_wildCopy8() function in the file lz4.c.

Definition at line 5316 of file GHdf5.cpp.

Referenced by decompress_lz4().

◆ read_uint16()

uint16_t read_uint16 ( const void * memPtr )

Read unsigned 16 Bit integer stored as little endian.

Parameters

[in] memPtr Input pointer.

Returns: Unsigned 16 Bit integer.

Reads an unsigend 16 Bit integer from the current memory location that is stored in little endian order. The code handles both little and big endian architectures.

Definition at line 5168 of file GHdf5.cpp.

References gammalib::little_endian().

Referenced by decompress_lz4().

◆ read_variable_length()

size_t read_variable_length	(	const uint8_t **	ip,
		const uint8_t *	ilimit,
		const bool &	initial_check )

Read the variable-length literal or match length.

Parameters

[in]	ip	Input pointer.
[in]	ilimit	Limiting position.
[in]	initial_check	Performs initial check.

Returns: Length literal or match length.

Exceptions

GException::invalid_argument	Initial check revealed that read limit was reached
GException::invalid_value	Read limit was reached or accumulator overflow was detected

ilimit specifies the position after which, if length is not decoded, the input is necessarily corrupted. In this case an exception is thrown.

If initial_check is true then a check ip < ilimit is performed, and if the check is false, an exception is thrown.

The method was inspired by https://github.com/kiyo-masui/bitshuffle and specifically the read_variable_length() function in the file lz4.c.

Definition at line 5110 of file GHdf5.cpp.

References G_READ_VARIABLE_LENGTH.

Referenced by decompress_lz4().

Macros

Functions

Macro Definition Documentation

◆ G_DATA_FILTER

◆ G_DATA_FILTER_BITSHUFFLE

◆ G_DATA_TO_DOUBLE

◆ G_DATA_TO_INT

◆ G_DATA_TO_STRING

◆ G_DECOMPRESS_LZ4

◆ G_FREAD_DATA1

◆ G_FREAD_DATA2

◆ G_FREAD_DATA_AS_STRING

◆ G_FREAD_DATA_CHUNK

◆ G_FREAD_INT

◆ G_FREAD_STRING

◆ G_FREAD_UINT32

◆ G_FREAD_UINT64

◆ G_FREAD_ZERO

◆ G_GLOBAL_HEAP_STRING

◆ G_LOAD

◆ G_MEMCPY_USING_OFFSET

◆ G_READ_BTREE_CHUNKED

◆ G_READ_GROUP

◆ G_READ_MESSAGE_ATTRIBUTE

◆ G_READ_MESSAGE_DATASPACE

◆ G_READ_MESSAGE_DATATYPE

◆ G_READ_MESSAGE_FILTER

◆ G_READ_MESSAGE_LAYOUT

◆ G_READ_OBJECT_HEADER

◆ G_READ_OBJECT_HEADER_MESSAGE

◆ G_READ_OBJECT_HEADER_V1

◆ G_READ_OBJECT_HEADER_V2

◆ G_READ_SUPERBLOCK

◆ G_READ_SYMBOL_TABLE_ENTRY

◆ G_READ_SYMBOL_TABLE_NODE

◆ G_READ_VARIABLE_LENGTH

◆ G_XML_HDF5_ENTRY

◆ G_XML_MSG_ATTRIBUTE

◆ G_XML_MSG_TYPE

Function Documentation

◆ bitshuffle_decompress()

◆ bitshuffle_elements()

◆ bitshuffle_read_uint32()

◆ bitshuffle_read_uint64()

◆ decompress_lz4()

◆ memcpy_beyond8()

◆ memcpy_using_offset()

◆ memory_beyond32()

◆ read_uint16()

◆ read_variable_length()