Load Double-Quoted Text Data File

#pragma rtGlobals=3     // Use modern global access method and strict wave access.

// The LoadDoubleQuotedTextDataFile procedure loads data files in which text is double-quoted, like this:
//  "<ColumnName>"<comma>"<ColumnName>"
//  "<string possibly containing comma>"<comma>"<string possibly containing comma>"...
//
// The file may also contain numeric columns, like this:
//  "<string possibly containing comma>"<comma><number>...
//
// LoadWave by itself does not handle this because the comma inside the double-quoted string is taken as a column delimiter.
// To load this, LoadDoubleQuotedTextDataFile does the following:
//  1.  Create a temporary file in which the true comma delimiters (those outside the quotes) are replaced by tabs
//      and the quotes are removed.
//  2.  Load the temporary file.
//  3.  Delete the temporary file.
//
// This also works on tab-delimited test files containing double-quotes that you don't want to load, e.g.:
//  "<ColumnName>"<comma>"<ColumnName>"
//  "<string>"<tab>"<string>"...

// These set the initial values in the dialog displayed by LoadDoubleQuotedTextDataFile.
// You can customize them for your file if you like.
static Constant kDefaultNameLine = 0
static Constant kDefaultFirstDataLine = 0
static Constant kDefaultNumDataLines = 0
static Constant kDefaultFirstColumnToLoad = 0
static Constant kDefaultNumColumnsToLoad = 0

Menu "Load Waves"
    "Load Double-Quoted Text Data File . . .", LoadDoubleQuotedTextDataFile("", "")
End

static Function/S CleanupText(origText, cleanedText)
    String origText                         // Input
    String& cleanedText                     // Output

    Variable numOrigBytes = strlen(origText)
   
    // This speeds things up by avoiding resizing the output string over and over again
    cleanedText = PadString("", numOrigBytes, 32)
   
    Variable numOutputBytes = 0
    String doubleQuoteStr = "\""
    String commaStr = ",", tabStr = "\t"
    Variable inDoubleQuote = 0
   
    Variable i
    for(i=0; i<numOrigBytes; i+=1)
        Variable skip = 0
        String byte = origText[i]
        if (CmpStr(byte,doubleQuoteStr) == 0)
            skip = 1
            inDoubleQuote = !inDoubleQuote  // We are either entering or leaving a double-quoted string
        else
            if (CmpStr(byte,commaStr) == 0)
                if (!inDoubleQuote)
                    // This is a delimiter comma - replace it with tab
                    byte = tabStr
                endif
            endif
        endif
       
        if (!skip)
            cleanedText[numOutputBytes] = byte
            numOutputBytes += 1
        endif  
    endfor
   
    cleanedText = cleanedText[0,numOutputBytes-1]
   
    return cleanedText
End

static Function TestCleanupText()
    String origText = "\"Quick brown, fox\",\"0\""
    String cleanedText
    CleanupText(origText, cleanedText)
    Print cleanedText
End

static Function CreateCleanedUpTempFile(pathName, origFileName, tempFileName)
    String pathName                     // Symbolic path name
    String origFileName
    String tempFileName                     // Name to use for temporary file
   
    Variable origRefNum
    Open /R /P=$pathName /Z origRefNum as origFileName
    if (V_flag != 0)
        Print "Error opening " + origFileName
        return -1                           // Error of some kind
    endif
   
    FStatus origRefNum
    Variable numBytesInOrigFile = V_logEOF
    String origText = PadString("", numBytesInOrigFile, 32)
    FBinRead origRefNum, origText
    Close origRefNum

    String cleanedText = CleanupText(origText, cleanedText)

    Variable tempRefNum
    Open /P=$pathName /Z tempRefNum as tempFileName
    if (V_flag != 0)
        Print "Error opening " + tempFileName
        return -1                       // Error of some kind
    endif
    FBinWrite tempRefNum, cleanedText
    Close tempRefNum
   
    return 0   
End

static StrConstant kExtensionStr = "????"       // Shows files with any extension. Change to, e.g., ".dat", to show just .dat files.

// LoadDoubleQuotedTextDataFile(pathName, fileName, [nameLine, firstDataLine, numDataLines, firstColumn, numColumns])
// nameLine, firstDataLine, numDataLines, firstColumn and numColumns are optional parameters.
// If you omit them, LoadDoubleQuotedTextDataFile displays a dialog in which the user can enter them
Function LoadDoubleQuotedTextDataFile(pathName, fileName, [nameLine, firstDataLine, numDataLines, firstColumn, numColumns])
    String pathName     // Name of an Igor symbolic path or ""
    String fileName         // Name of file or full path to file
   
    // Optional parameters
    Variable nameLine
    Variable firstDataLine
    Variable numDataLines
    Variable firstColumn
    Variable numColumns
   
    if (ParamIsDefault(nameLine) || ParamIsDefault(firstDataLine) || ParamIsDefault(numDataLines) || ParamIsDefault(firstColumn) || ParamIsDefault(numColumns))
        // The caller did not supply optional parameters so display dialog
       
        // These set the initial values displayed in the dialog
        nameLine = kDefaultNameLine
        firstDataLine = kDefaultFirstDataLine
        numDataLines = kDefaultNumDataLines
        firstColumn = kDefaultFirstColumnToLoad
        numColumns = kDefaultNumColumnsToLoad
       
        // Display dialog
        Prompt nameLine, "Line containing column names (zero-based - 0 if no column names): "
        Prompt firstDataLine, "First line containing data (zero-based): "
        Prompt numDataLines, "Number of data lines to load (zero for all): "
        Prompt firstColumn, "First column to load (zero-based): "
        Prompt numColumns, "Number of columns to load (zero for all): "
        DoPrompt "Enter LoadWave Parameters", nameLine, firstDataLine, numDataLines, firstColumn, numColumns
        if (V_Flag)
            return -1                               // User canceled
        endif
    endif

    Variable refNum
   
    String message

    // First get a valid reference to a file.
    if ((strlen(pathName)==0) || (strlen(fileName)==0))
        // Display dialog looking for file.
        message = "Select double-quoted text data file"
        Open /D /R /P=$pathName /T=kExtensionStr /M=message refNum as fileName
        fileName = S_fileName               // S_fileName is set by Open/D
        if (strlen(fileName) == 0)          // User cancelled?
            return -1
        endif
    endif
   
    String tempFileName = fileName + "_tmp"
    if (CreateCleanedUpTempFile(pathName, fileName, tempFileName) != 0)
        return -1                           // Error
    endif

    // Remove /D if you want to load as single-precision floating point
    String delimiters = "\t"
    LoadWave /O /J /D /K=0 /V={delimiters, "", 0, 0} /L={nameLine, firstDataLine, numDataLines, firstColumn, numColumns} /E=1 /Q /P=$pathName tempFileName
   
    if (V_flag > 0)
        Printf "Loaded double-quoted data from \"%s\"\r", fileName
    endif
   
    DeleteFile /P=$pathName tempFileName
   
    return 0
End