diff --git a/preProcessing/intan/read_Intan_RHD2000_file.m b/preProcessing/intan/read_Intan_RHD2000_file.m index d193219..12758fa 100644 --- a/preProcessing/intan/read_Intan_RHD2000_file.m +++ b/preProcessing/intan/read_Intan_RHD2000_file.m @@ -1,4 +1,4 @@ -function read_Intan_RHD2000_file +function read_Intan_RHD2000_file(varargin) % read_Intan_RHD2000_file % @@ -16,6 +16,21 @@ % >> whos % >> amplifier_channels(1) % >> plot(t_amplifier, amplifier_data(1,:)) +% +% For large recordings that do not fit in memory, stream the data directly +% to the traditional Intan .dat files instead: +% >> read_Intan_RHD2000_file('streamToDat', true) +% Output files are overwritten in the selected RHD file's directory. Analog +% values remain raw ADC counts; auxiliary samples are repeated four times. +% Digital inputs remain packed uint16 words and time.dat stores sample ticks. +% Amplifiers retain the legacy reader's centering and software notch filter. +% Streaming does not populate the base workspace or export supply, temperature, +% digital output, or header files. + +p = inputParser; +p.addParameter('streamToDat', false, @(x) islogical(x) && isscalar(x)); +p.parse(varargin{:}); +stream_to_dat = p.Results.streamToDat; [file, path, filterindex] = ... uigetfile('*.rhd', 'Select an RHD2000 Data File', 'MultiSelect', 'off'); @@ -27,6 +42,10 @@ tic; filename = [path,file]; fid = fopen(filename, 'r'); +if (fid == -1) + error('Intan:InputOpenFailed', 'Could not open %s.', filename); +end +cleanup_input = onCleanup(@() fclose(fid)); s = dir(filename); filesize = s.bytes; @@ -290,6 +309,23 @@ record_time = num_amplifier_samples / sample_rate; +if (stream_to_dat && data_present) + if (mod(bytes_remaining, bytes_per_block) ~= 0) + error('Intan:IncompleteDataBlock', 'RHD file contains an incomplete data block.'); + end + stream_data_to_dat_files(fid, path, num_data_blocks, ... + num_samples_per_data_block, num_amplifier_channels, ... + num_aux_input_channels, num_supply_voltage_channels, ... + num_board_adc_channels, num_board_dig_in_channels, ... + num_board_dig_out_channels, num_temp_sensor_channels, ... + data_file_main_version_number, data_file_secondary_version_number, ... + sample_rate, notch_filter_frequency); + clear cleanup_input; + fprintf(1, 'Done! Elapsed time: %0.1f seconds\n', toc); + fprintf(1, 'Streamed data to .dat files in %s\n\n', path); + return; +end + if (data_present) fprintf(1, 'File contains %0.3f seconds of data. Amplifiers were sampled at %0.2f kS/s.\n', ... record_time, sample_rate / 1000); @@ -384,7 +420,7 @@ end % Close data file. -fclose(fid); +clear cleanup_input; if (data_present) @@ -528,6 +564,127 @@ return +function stream_data_to_dat_files(fid, output_path, num_data_blocks, ... + samples_per_block, num_amplifier_channels, num_aux_input_channels, ... + num_supply_voltage_channels, num_board_adc_channels, ... + num_board_dig_in_channels, num_board_dig_out_channels, ... + num_temp_sensor_channels, main_version, secondary_version, ... + sample_rate, notch_filter_frequency) + +% Stream RHD blocks directly to the traditional Intan .dat files. This +% reproduces the layout produced by fwrite(amplifier_data, 'int16') without +% retaining the complete recording in memory. + +fid_amplifier = fopen(fullfile(output_path, 'amplifier.dat'), 'w', 'ieee-le'); +fid_auxiliary = fopen(fullfile(output_path, 'auxiliary.dat'), 'w', 'ieee-le'); +fid_analogin = fopen(fullfile(output_path, 'analogin.dat'), 'w', 'ieee-le'); +fid_digitalin = fopen(fullfile(output_path, 'digitalin.dat'), 'w', 'ieee-le'); +fid_time = fopen(fullfile(output_path, 'time.dat'), 'w', 'ieee-le'); + +file_ids = [fid_amplifier, fid_auxiliary, fid_analogin, fid_digitalin, fid_time]; +if any(file_ids == -1) + cellfun(@fclose, num2cell(file_ids(file_ids ~= -1))); + error('Intan:OutputOpenFailed', 'Could not open one or more output .dat files in %s.', output_path); +end +cleanup_files = onCleanup(@() cellfun(@fclose, num2cell(file_ids))); + +fprintf(1, 'Streaming data to .dat files...\n'); +print_increment = 10; +percent_done = print_increment; +filter_states = cell(1, num_amplifier_channels); +num_gaps = 0; +previous_timestamp = []; + +for block_index = 1:num_data_blocks + if ((main_version == 1 && secondary_version >= 2) || main_version > 1) + timestamps = read_stream_samples(fid, samples_per_block, 'int32=>int32'); + else + timestamps = read_stream_samples(fid, samples_per_block, 'uint32=>uint32'); + end + num_gaps = num_gaps + sum(diff([previous_timestamp; double(timestamps)]) ~= 1); + previous_timestamp = double(timestamps(end)); + % Preserve timestamp bits even for unsigned v1.0/v1.1 values above intmax. + write_stream_samples(fid_time, typecast(timestamps, 'int32'), 'int32'); + + if (num_amplifier_channels > 0) + amplifier_block = read_stream_samples(fid, [samples_per_block, num_amplifier_channels], 'uint16=>uint16')'; + % The legacy workflow first subtracts 32768, then writes int16. + amplifier_block = double(amplifier_block) - 32768; + if (notch_filter_frequency > 0 && main_version < 3) + for channel_index = 1:num_amplifier_channels + [amplifier_block(channel_index, :), filter_states{channel_index}] = ... + notch_filter(amplifier_block(channel_index, :), sample_rate, ... + notch_filter_frequency, 10, filter_states{channel_index}); + end + end + write_stream_samples(fid_amplifier, amplifier_block, 'int16'); + end + + if (num_aux_input_channels > 0) + auxiliary_block = read_stream_samples(fid, [samples_per_block / 4, num_aux_input_channels], 'uint16=>uint16')'; + write_stream_samples(fid_auxiliary, repelem(auxiliary_block, 1, 4), 'uint16'); + end + + % These signals are not written by the legacy export workflow, but must + % be consumed to remain aligned with the next data block. + if (num_supply_voltage_channels > 0) + read_stream_samples(fid, [1, num_supply_voltage_channels], 'uint16=>uint16'); + end + + if (num_temp_sensor_channels > 0) + read_stream_samples(fid, [1, num_temp_sensor_channels], 'int16=>int16'); + end + + if (num_board_adc_channels > 0) + analogin_block = read_stream_samples(fid, [samples_per_block, num_board_adc_channels], 'uint16=>uint16')'; + write_stream_samples(fid_analogin, analogin_block, 'uint16'); + end + + if (num_board_dig_in_channels > 0) + digital_input_raw = read_stream_samples(fid, samples_per_block, 'uint16=>uint16'); + % Intan .dat readers decode native digital channels from packed bits. + write_stream_samples(fid_digitalin, digital_input_raw, 'uint16'); + end + + if (num_board_dig_out_channels > 0) + read_stream_samples(fid, samples_per_block, 'uint16=>uint16'); + end + + fraction_done = 100 * block_index / num_data_blocks; + if (fraction_done >= percent_done) + fprintf(1, '%d%% done...\n', percent_done); + percent_done = percent_done + print_increment; + end +end + +clear cleanup_files; +if (num_gaps > 0) + fprintf(1, 'Warning: %d gaps in timestamp data found. Time scale will not be uniform!\n', num_gaps); +end +fprintf(1, 'Finished streaming data.\n'); + +return + + +function data = read_stream_samples(fid, dimensions, precision) + +[data, count] = fread(fid, dimensions, precision); +if (count ~= prod(dimensions)) + error('Intan:ShortRead', 'Could not read a complete RHD data block.'); +end + +return + + +function write_stream_samples(fid, data, precision) + +if (fwrite(fid, data, precision) ~= numel(data)) + error('Intan:ShortWrite', 'Could not write all samples to an output .dat file.'); +end + +return + + function a = fread_QString(fid) % a = read_QString(fid) @@ -567,7 +724,7 @@ return -function out = notch_filter(in, fSample, fNotch, Bandwidth) +function [out, state] = notch_filter(in, fSample, fNotch, Bandwidth, state) % out = notch_filter(in, fSample, fNotch, Bandwidth) % @@ -587,6 +744,10 @@ tstep = 1/fSample; Fc = fNotch*tstep; +continuing = (nargin >= 5 && ~isempty(state)); +if continuing + in = [state.input, in]; +end L = length(in); % Calculate IIR filter parameters @@ -601,8 +762,12 @@ b2 = 1; out = zeros(size(in)); -out(1) = in(1); -out(2) = in(2); +if continuing + out(1:2) = state.output; +else + out(1) = in(1); + out(2) = in(2); +end % (If filtering a continuous data stream, change out(1) and out(2) to the % previous final two values of out.) @@ -611,6 +776,12 @@ out(i) = (a*b2*in(i-2) + a*b1*in(i-1) + a*b0*in(i) - a2*out(i-2) - a1*out(i-1))/a0; end +state.input = in(end-1:end); +state.output = out(end-1:end); +if continuing + out = out(3:end); +end + return diff --git a/test/test_read_Intan_RHD2000_file.m b/test/test_read_Intan_RHD2000_file.m new file mode 100644 index 0000000..ad93a2e --- /dev/null +++ b/test/test_read_Intan_RHD2000_file.m @@ -0,0 +1,238 @@ +function tests = test_read_Intan_RHD2000_file +tests = functiontests(localfunctions); +end + +function setupOnce(testCase) +root = fileparts(fileparts(mfilename('fullpath'))); +testCase.TestData.originalPath = path; +testCase.TestData.folder = tempname; +mkdir(testCase.TestData.folder); +% Exercise the public reader without displaying its file picker. +mock = fullfile(testCase.TestData.folder, 'uigetfile.m'); +fid = fopen(mock, 'w'); +fprintf(fid, ['function [file, folder, index] = uigetfile(varargin)\n' ... + 'global NEUROCODE_INTAN_TEST_FILE\n' ... + 'if isempty(NEUROCODE_INTAN_TEST_FILE)\n' ... + 'file = 0; folder = 0; index = 0; return;\nend\n' ... + '[folder, name, extension] = fileparts(NEUROCODE_INTAN_TEST_FILE);\n' ... + 'file = [name extension]; folder = [folder filesep]; index = 1;\nend\n']); +fclose(fid); +addpath(fullfile(root, 'preProcessing', 'intan')); +addpath(testCase.TestData.folder, '-begin'); +end + +function teardownOnce(testCase) +path(testCase.TestData.originalPath); +clear uigetfile; +clear global NEUROCODE_INTAN_TEST_FILE; +rmdir(testCase.TestData.folder, 's'); +end + +function setup(testCase) +global NEUROCODE_INTAN_TEST_FILE; +testCase.TestData.output = tempname(testCase.TestData.folder); +mkdir(testCase.TestData.output); +NEUROCODE_INTAN_TEST_FILE = fullfile(testCase.TestData.output, 'test.rhd'); +end + +function testRawExportsAcrossVersions(testCase) +for version = {[1 0], [1 1], [1 2], [1 3], [2 0], [3 0]} + data = writeFixture(testCase, version{1}, 0, false); + read_Intan_RHD2000_file('streamToDat', true); + verifyEqual(testCase, readDat(testCase, 'amplifier', 'int16'), ... + reshape(int16(double(data.amplifier) - 32768), [], 1)); + verifyEqual(testCase, readDat(testCase, 'auxiliary', 'uint16'), ... + reshape(repelem(data.auxiliary, 1, 4), [], 1)); + verifyEqual(testCase, readDat(testCase, 'analogin', 'uint16'), data.analog(:)); + verifyEqual(testCase, readDat(testCase, 'digitalin', 'uint16'), data.digital(:)); + verifyEqual(testCase, readDat(testCase, 'time', 'int32'), data.time(:)); +end +end + +function testNotchMatchesInMemoryAcrossBlocks(testCase) +for version = {[1 3], [2 0], [3 0]} + for mode = [1 2] + writeFixture(testCase, version{1}, mode, false); + read_Intan_RHD2000_file; + expected = evalin('base', 'int16(amplifier_data(:))'); + read_Intan_RHD2000_file('streamToDat', true); + verifyEqual(testCase, readDat(testCase, 'amplifier', 'int16'), expected); + end +end +end + +function testNoSignalChannels(testCase) +data = writeFixture(testCase, [2 0], 0, true); +read_Intan_RHD2000_file('streamToDat', true); +for name = {'amplifier', 'auxiliary', 'analogin', 'digitalin'} + verifyEmpty(testCase, readDat(testCase, name{1}, 'uint16')); +end +verifyEqual(testCase, readDat(testCase, 'time', 'int32'), data.time(:)); +end + +function testIncompleteBlockRejectedBeforeExport(testCase) +writeFixture(testCase, [2 0], 0, false); +global NEUROCODE_INTAN_TEST_FILE; +fid = fopen(NEUROCODE_INTAN_TEST_FILE, 'a'); +fwrite(fid, 1, 'uint8'); +fclose(fid); +opened = openFileIds; +verifyError(testCase, @() read_Intan_RHD2000_file('streamToDat', true), ... + 'Intan:IncompleteDataBlock'); +verifyEqual(testCase, openFileIds, opened); +verifyEmpty(testCase, dir(fullfile(testCase.TestData.output, '*.dat'))); +end + +function testOutputOpenFailureClosesFiles(testCase) +writeFixture(testCase, [2 0], 0, false); +mkdir(fullfile(testCase.TestData.output, 'analogin.dat')); +opened = openFileIds; +verifyError(testCase, @() read_Intan_RHD2000_file('streamToDat', true), ... + 'Intan:OutputOpenFailed'); +verifyEqual(testCase, openFileIds, opened); +end + +function testTimestampGapAcrossBlocks(testCase) +data = writeFixture(testCase, [2 0], 0, false); +global NEUROCODE_INTAN_TEST_FILE; +fid = fopen(NEUROCODE_INTAN_TEST_FILE, 'r+', 'ieee-le'); +fseek(fid, data.headerBytes + data.blockBytes, 'bof'); +fwrite(fid, 1000, 'int32'); +fclose(fid); +output = evalc('read_Intan_RHD2000_file(''streamToDat'', true)'); +verifySubstring(testCase, output, '2 gaps in timestamp data found'); +end + +function testUnsignedTimestampBits(testCase) +data = writeFixture(testCase, [1 0], 0, false); +global NEUROCODE_INTAN_TEST_FILE; +fid = fopen(NEUROCODE_INTAN_TEST_FILE, 'r+', 'ieee-le'); +fseek(fid, data.headerBytes, 'bof'); +fwrite(fid, uint32(2^31 + (0:59)), 'uint32'); +fclose(fid); +read_Intan_RHD2000_file('streamToDat', true); +ticks = readDat(testCase, 'time', 'uint32'); +verifyEqual(testCase, ticks(1:60), uint32(2^31 + (0:59)')); +end + +function testDefaultStillPopulatesWorkspace(testCase) +data = writeFixture(testCase, [2 0], 0, false); +read_Intan_RHD2000_file; +verifyEqual(testCase, evalin('base', 'amplifier_data'), double(data.amplifier) - 32768); +verifyEqual(testCase, evalin('base', 'aux_input_data'), 37.4e-6 * double(data.auxiliary)); +verifyEqual(testCase, evalin('base', 't_amplifier'), double(data.time) / 1000); +verifyEmpty(testCase, dir(fullfile(testCase.TestData.output, '*.dat'))); +end + +function testHeaderOnlyDoesNotExport(testCase) +data = writeFixture(testCase, [2 0], 0, false); +global NEUROCODE_INTAN_TEST_FILE; +fid = fopen(NEUROCODE_INTAN_TEST_FILE, 'r'); +header = fread(fid, data.headerBytes, 'uint8=>uint8'); +fclose(fid); +fid = fopen(NEUROCODE_INTAN_TEST_FILE, 'w'); +fwrite(fid, header, 'uint8'); +fclose(fid); +read_Intan_RHD2000_file('streamToDat', true); +verifyEmpty(testCase, dir(fullfile(testCase.TestData.output, '*.dat'))); +end + +function testCancelledPicker(testCase) +global NEUROCODE_INTAN_TEST_FILE; +NEUROCODE_INTAN_TEST_FILE = ''; +read_Intan_RHD2000_file('streamToDat', true); +verifyEmpty(testCase, dir(fullfile(testCase.TestData.output, '*.dat'))); +end + +function data = writeFixture(testCase, version, notchMode, noChannels) +global NEUROCODE_INTAN_TEST_FILE; +fid = fopen(NEUROCODE_INTAN_TEST_FILE, 'w', 'ieee-le'); +cleanup = onCleanup(@() fclose(fid)); +fwrite(fid, hex2dec('c6912702'), 'uint32'); +fwrite(fid, version, 'int16'); +fwrite(fid, 1000, 'single'); +fwrite(fid, 1, 'int16'); +fwrite(fid, [1 1 400 1 1 400], 'single'); +fwrite(fid, notchMode, 'int16'); +fwrite(fid, [1000 1000], 'single'); +for i = 1:3 + writeString(fid, ''); +end +hasTemp = version(1) > 1 || version(2) >= 1; +if hasTemp + fwrite(fid, ~noChannels, 'int16'); +end +if version(1) > 1 || version(2) >= 3 + fwrite(fid, 13, 'int16'); +end +if version(1) > 1 + writeString(fid, ''); +end +types = [0 0 1 1 2 3 3 4 4 5]; +orders = [0 1 0 1 0 0 1 1 15 0]; +if noChannels + fwrite(fid, 0, 'int16'); +else + fwrite(fid, 1, 'int16'); + writeString(fid, 'Port A'); + writeString(fid, 'A'); + fwrite(fid, [1 numel(types) 2], 'int16'); + for i = 1:numel(types) + writeString(fid, sprintf('A-%03d', i)); + writeString(fid, sprintf('A-%03d', i)); + fwrite(fid, [orders(i) i-1 types(i) 1 0 0 0 0 0 0], 'int16'); + fwrite(fid, [1000 0], 'single'); + end +end +data.headerBytes = ftell(fid); +samples = 128; +if version(1) == 1 + samples = 60; +end +data.amplifier = uint16(reshape(mod(0:6*samples-1, 65536), 2, [])); +data.amplifier(:, 1:2) = uint16([0 32768; 65535 32767]); +data.auxiliary = uint16(reshape(1000:1000+6*samples/4-1, 2, [])); +data.analog = uint16(reshape(30000:30000+6*samples-1, 2, [])); +data.digital = uint16(mod(0:3*samples-1, 2)*2 + mod(floor((0:3*samples-1)/3), 2)*32768); +data.time = int32(-10:3*samples-11); +if version(1) == 1 && version(2) < 2 + data.time = int32(0:3*samples-1); +end +for block = 1:3 + start = ftell(fid); + indices = (block-1)*samples + (1:samples); + fwrite(fid, data.time(indices), 'int32'); + if ~noChannels + fwrite(fid, data.amplifier(:, indices)', 'uint16'); + auxIndices = (block-1)*samples/4 + (1:samples/4); + fwrite(fid, data.auxiliary(:, auxIndices)', 'uint16'); + fwrite(fid, 50000, 'uint16'); % supply voltage + if hasTemp + fwrite(fid, -100, 'int16'); + end + fwrite(fid, data.analog(:, indices)', 'uint16'); + fwrite(fid, data.digital(indices), 'uint16'); + fwrite(fid, 1234*ones(1, samples), 'uint16'); % digital output + end + data.blockBytes = ftell(fid) - start; +end +end + +function writeString(fid, value) +fwrite(fid, 2*numel(value), 'uint32'); +fwrite(fid, uint16(value), 'uint16'); +end + +function ids = openFileIds +if exist('openedFiles', 'builtin') || exist('openedFiles', 'file') + ids = openedFiles; +else + ids = fopen('all'); +end +end + +function data = readDat(testCase, name, precision) +fid = fopen(fullfile(testCase.TestData.output, [name '.dat']), 'r', 'ieee-le'); +cleanup = onCleanup(@() fclose(fid)); +data = fread(fid, Inf, [precision '=>' precision]); +end