function [uniqueData, indexUniqueRow] = RemoveDuplicates(rawData)
%UNTITLED2 Summary of this function goes here
%   Detailed explanation goes here

% Check if it is a struct because we are using this in
% CalculateParameterValuesForModel where the input is most likely a struct
% and also in the RunInference where the data is most likely already a
% table
transformBackToStruct = false;
if isstruct(rawData)
    transformBackToStruct = true;
    rawData = struct2table(rawData);
end

colsToRemove = {'transformedCoord', 'CoeffPCA', 'ImageOrientationPatient', 'PtNumber', 'PatientWeight'};
intersectColsToRemove = intersect(colsToRemove, rawData.Properties.VariableNames);
if ~isempty(intersectColsToRemove)
    % Define a table without transformedCoord as the unique() function can
    % not handle group rows. However we still want to return the full
    % rawData to the user later. So identifierData is only used to identify
    % unique Rows but not used for anything else.
    identifierData = removevars(rawData, intersectColsToRemove);
end

% Compare for any cases where we have identical values for all parameters.
% unique() has some issues handling NaN values as distinct instead of
% unique. So rows with just one NaN value will always be considered a
% distinct row.
% TODO: how to generalize this?
[~, indexUniqueRow, ~] = unique(identifierData(:,1:15), "rows");
duplicateIndices = setdiff(1:height(identifierData), indexUniqueRow);

if ~isempty(duplicateIndices)
    warning('Duplicates found where all the volumetric features that we calculated are the same! Removing everything but the first occurance.')
end


uniqueData = rawData(indexUniqueRow, :);

% Compare for any cases where we have the identical study ID.
% If this is the case, we will create a new StudyNumber (StdNumber) for it.
[~, indexUniqueStudy, ~] = unique(uniqueData(:, {'StudyInstanceUID', 'ROIName'}));
duplicateStudies = setdiff(1:height(uniqueData), indexUniqueStudy);


if ~isempty(duplicateStudies)
    warning('We found duplicate ROI Names for the same StudyInstanceUID found! We are not removing that but you should have a look..')
end

% uniqueData = uniqueData(indexUniqueStudy, :); % uncomment this if you
% actually want to remove duplica ROI Names from the same StudyInstanceUID.
% Not recommended though.

if transformBackToStruct
    uniqueData = table2struct(uniqueData);
    uniqueData = uniqueData';
end
end