Small example Eiger 2X 16M data set from Diamond Light Source I04 revised for HDRMX Gold Standard Discussion

Revised useful small (488 frame) Eiger data set recorded during routine testing, useful for software testing as it is small. Data recorded is from a thaumatin crystal by Graeme Winter,  The original dataset is  https://zenodo.org/record/3385862  which contains two Eiger MX datasets, Therm_6_1 and Them_6_2, each with a data file and two versions each of the metadata -- a "..._master.h5" file and a "....nxs" file.  The former are the usual Eiger metadata files using exposed external links to connect the metadata to the date, and the latter are HDF5-1.10 VDS files.  This revision has the same data as the original Therm_6_2 data, but now includes with the "master.h5" file a "..._master_rev.h5" file and with the ".nxs" file a "..._rev.nxs" VDS file.

The purpose to the changes in the "..._rev" files is to provide a supporting example for the HDRMX discusssion of a new proposed Eiger "gold standard" to improve the ability to process Eiger MX data collected at one facility at other facilties, by ensuring that sufficient metadata is stored with all datasets.

The changes were made by the following script

cp Therm_6_2.nxs Therm_6_2_rev.nxs
cp Therm_6_2_master.h5 Therm_6_2_master_rev.h5
export LD_LIBRARY_PATH=$HOME/lib
export HDF5_PLUGIN_PATH=$HOME/lib
export PATH=$HOME/bin:$PATH
h5copy -i Therm_6_2_rev.nxs -o Therm_6_2_master_rev.h5 -s /entry/instrument/name -d /entry/instrument/name -f ref
h5copy -i Therm_6_2_rev.nxs -o Therm_6_2_master_rev.h5 -s /entry/instrument/source -d /entry/source -f ref
h5copy -i Therm_6_2_rev.nxs -o Therm_6_2_rev.nxs -s /entry/instrument/source -d /entry/source -f ref
export end_time=`h5dump -d "/entry/end_time" Therm_6_2_master.h5 | grep ":" | sed 's/^.........//'|sed 's/.\$//'`
echo "end_time: $end_time"
python << 'EOL'
import h5py
import numpy as np
import os
end_time=os.environ['end_time']
fvds = h5py.File('Therm_6_2_rev.nxs','r+')
fmaster = h5py.File('Therm_6_2_master_rev.h5','r+')
fvds_keys=fvds.keys()
fmaster_keys=fmaster.keys()
fvds_entry=fvds['entry']
fmaster_entry=fmaster['entry']
fvds_entry_keys=fvds_entry.keys()
fmaster_entry_keys=fmaster_entry.keys()
fvds_entry_instrument=fvds['entry']['instrument']
fmaster_entry_instrument=fmaster['entry']['instrument']
fvds_entry_instrument_keys=fvds_entry_instrument.keys()
fmaster_entry_instrument_keys=fmaster_entry_instrument.keys()
fvds_entry_instrument_name=(fvds['entry']['instrument']['name'])
fmaster_entry_instrument_name=(fmaster['entry']['instrument']['name'])
fvds_entry_instrument_short_name=fvds_entry_instrument.attrs['short_name']
fmaster_entry_instrument_short_name=fmaster_entry_instrument.attrs['short_name']
zero_offset=fmaster_entry_instrument['detector']['module']['fast_pixel_direction'].attrs['offset']
fmaster_det_z=fmaster_entry_instrument['transformations']['det_z']
fvds_det_z=fvds_entry_instrument['transformations']['det_z']
print('fvds_keys: ',fvds_keys)
print('fmaster_keys: ',fmaster_keys)
print('fvds_entry_keys: ',fvds_entry_keys)
print('fmaster_entry_keys: ',fmaster_entry_keys)
print('fvds_entry_instrument_keys: ',fvds_entry_instrument_keys)
print('fmaster_entry_instrument_keys: ',fmaster_entry_instrument_keys)
print('fvds_entry_instrument_name: ',fvds_entry_instrument_name)
print('fmaster_entry_instrument_name: ',fmaster_entry_instrument_name)
print('fvds_entry_instrument_short_name: ',fvds_entry_instrument_short_name)
print('fmaster_entry_instrument_short_name: ',fmaster_entry_instrument_short_name)
print('fmaster_entry_instrument_detector_module_fast_pixel_direction_offset: ',zero_offset)
print('fmaster_entry_instrument_detector_detector_z_det_z: ',fmaster_det_z)
print('fmaster_entry_end_time: ',end_time)
fmaster.attrs.modify('file_time',np.string_(end_time))
fmaster.attrs.modify('file_name',np.string_('Therm_6_2_master_rev.h5'))
fmaster.attrs.modify('HDF5_Version',np.string_('hdf5-1.8.18'))
fvds.attrs.modify('file_time',np.string_(end_time))
fvds.attrs.modify('file_name',np.string_('Therm_6_2_master_rev.h5'))
fvds.attrs.modify('HDF5_Version',np.string_('hdf5-1.10.5'))
fvds_entry_instrument_name.attrs.modify('short_name',np.string_(fvds_entry_instrument.attrs['short_name']))
fmaster_entry_instrument_name.attrs.modify('short_name',np.string_(fmaster_entry_instrument.attrs['short_name']))
fmaster_entry_instrument['attenuator']['attenuator_transmission'].attrs.modify('units',np.string_(""))
fmaster_entry_instrument['detector']['count_time'].attrs.modify('units',np.string_("s"))
fvds_entry_instrument_name.attrs.modify('short_name',np.string_(fvds_entry_instrument.attrs['short_name']))
fvds_entry_instrument['attenuator']['attenuator_transmission'].attrs.modify('units',np.string_(""))
fvds_entry_instrument['detector']['count_time'].attrs.modify('units',np.string_("s"))
fmaster_det_z.attrs.modify('offset',zero_offset)
fvds_det_z.attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['phi'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['chi'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['sam_x'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['sam_y'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['sam_z'].attrs.modify('offset',zero_offset)
fmaster_entry['sample']['transformations']['omega'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['phi'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['chi'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['sam_x'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['sam_y'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['sam_z'].attrs.modify('offset',zero_offset)
fvds_entry['sample']['transformations']['omega'].attrs.modify('offset',zero_offset)
print(fmaster['entry']['instrument']['name'].attrs['short_name'])
print(fmaster['entry']['instrument']['name'].attrs['short_name'].shape)
print(fmaster['entry']['instrument']['name'].attrs['short_name'].dtype)
del fvds_entry_instrument.attrs['short_name']
del fmaster_entry_instrument.attrs['short_name']
del fmaster_entry_instrument['source']
fvds.close()
fmaster.close()
quit()
EOL
$HOME/bin/nxvalidate -a NXmx -l /home/yaya/hdrmx_rev_29Sep19/hdrmx/definitions  Therm_6_2_master_rev.h5
$HOME/bin/nxvalidate -a NXmx -l /home/yaya/hdrmx_rev_29Sep19/hdrmx/definitions  Therm_6_2_rev.nxs

The revised cnxvalidate and definitions are available on github

https://github.com/HDRMX/cnxvalidate.git

https://github.com/HDRMX/definitions.git