% UDAS2nc - Convert all of the .txt data files in directory to NetCDF
%
% This program will loop through all original text data files output from
% Underway Data Acquisition System in the current directory, build global
% metadata attributes from the header in each .txt file, and write the data
% out to a Climate Forecast convention 'trajectory' netCDF data file using
% the same base name as the original data file.
%
% Other m-files required: gen_AUV_ncfile and SNCTOOLS toolbox 
%
% Author: Mike McCann
% Work address: MBARI, 7700 Sandholdt Rd, Moss Landing, CA 95039  USA
% Author e-mail: mccann@mbaril.org
% Website: http://www.mbaril.org
% Creation: 1-March-2012

%======Sample .txt data file ======
% %R/V John H. Martin Underway Data Acquisition System Data File
% %Column 1:  YYYYMMDD
% %Column 2:  HHMMSS
% %Column 3:  Absolute Decimal Julian Day
% %Column 4:  Decimal Julian Day
% %Column 5:  Decimal Hour
% %Column 6:  Latitude
% %Column 7:  Longitude
% %Column 8:  Salinity PSS78 from SBE45: S/N 4529602-0057
% %Column 9:  Temperature in degrees C fromSBE45
% %Column 10: Conductivity in S/m from SBE45
% %Column 11: Calibrated Fluorescence from Scufa: S/N 0239
% %Column 12: Turbidity from Scufa
% %Column 13: Temperature in degrees C from Scufa
% %Times converted from raw file PDST into GMT
% 
% 20101028,155530,734439.6635417,301.6635417,15.92500,36.806600,121.790433,31.778,17.284,4.1442,4.45,3.73,18.4
% 20101028,155542,734439.6636806,301.6636806,15.92833,36.806300,121.791000,32.802,15.753,4.1199,3.23,2.22,17.2
%======Sample .txt data file ======

files = dir('*.txt');
pointCount = 0;
for i=1:length(files),
    
    [path, base, ext] = fileparts(files(i).name);
    dataFile = files(i).name;
    
    % Make sure we start with empty metadata for each data file
    clear metadata;
    metadata.vehicle_name = 'JohnMartin';
    metadata.institution = 'MBARI';
    metadata.ship_name = 'JohnMartin';
    
    % Open file
    disp(['Reading file for metadata: ' dataFile]); 
    try
        fid = fopen(dataFile);
        tline = fgetl(fid);
    catch me
        disp(['Cannot read file ' dataFile])
        continue
    end
    
    % Save first line and column 8 following into comment field of
    % metadata for storing in netCDF global attributes
    finishedHeader = 0;
    fieldname = 'comment';
    i = 1;
    metadata.title = tline(2:end);
    while ischar(tline) & ~finishedHeader    
        if i == 1,
            value = tline(2:end);
        end
        if i > 8,
            value = [value '\n' tline(13:end)];
        end
        if i == 16,
            finishedHeader = 1;
        end
        tline = fgetl(fid);
        i = i + 1;
    end
    metadata = setfield(metadata, fieldname, value);
    
    
    % %Column 8:  Salinity PSS78 from SBE45: S/N 4529602-0057
    % %Column 9:  Temperature in degrees C fromSBE45
    % %Column 10: Conductivity in S/m from SBE45
    % %Column 11: Calibrated Fluorescence from Scufa: S/N 0239
    % %Column 12: Turbidity from Scufa
    % %Column 13: Temperature in degrees C from Scufa
    
    header = ''; 
    header = [header 'time,'];       % Parsed from columns 1 & 2
    header = [header 'depth,'];      % Nominal depth of intake: 1m (TimP, Personal Communication) 
    header = [header 'latitude,'];
    header = [header 'longitude,'];  % Must negate from original
    header = [header 'salinity,'];
    header = [header 'temperature,'];
    header = [header 'conductivity,'];
    header = [header 'fluorescence,'];
    header = [header 'turbidity'];
    
    % Write the data to a separate file with simplified header so that we can read it back into a structure with mfcsvread
    newFile = [base '_simple.stxt'];
    newHeader = '';
    fout = fopen(newFile, 'w');
    fwrite(fout, [header char(10)]);
    nomDepth = 1.0;
    dataRecordCount = 0;
    while ischar(tline)
        cols = regexp(tline, ',', 'split'); 
        dtstr = [char(cols(1)) char(cols(2))];
        s = regexp(dtstr, '(\d\d\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)', 'match');
        d = datenum(s, 'yyyymmddHHMMss');
        dateString = datestr(d, 'yyyy-mm-ddTHH:MM:ss'); 
        esecs = (d - datenum(1970,1,1,0,0,0)) * 86400;

        %fwrite(fout, [str(esecs) ',1.0,' cols(8) ',' cols(9) ',' cols(10) ]);
        fprintf(fout, '%.0f,%.1f,%s,-%s,%s,%s,%s,%s,%s\n', esecs, nomDepth, char(cols(6)), char(cols(7)), char(cols(8)), char(cols(9)), char(cols(10)), char(cols(11)), char(cols(12)));
        tline = fgetl(fid);
        dataRecordCount = dataRecordCount + 1;
    end            
    fclose(fout);
    fclose(fid);
    
    if dataRecordCount == 0,
        delete(newFile)         % remove and skip over files with no records
        continue
    end
    
    % Read data file into M structure
    disp(['Reading file for its data: ' newFile])
    M = mfcsvread(newFile);
    
    % Write netCDF file to netcdf subdirectory
    ncFile = char([base '.nc']);
    gen_UDAS_ncfile(ncFile, metadata, M.time, M.latitude, ...
       M.longitude, M.depth, M.conductivity, M.temperature, M.salinity, M.fluorescence, M.turbidity)

    disp(['Wrote file: ' ncFile ])
   
    delete(newFile)     % Clean up
    
end
