Load EC-Lab Data File

#pragma TextEncoding = "UTF-8"
#pragma rtGlobals=3		// Use modern global access method and strict wave access.

// About "Load EC-Lab Data File" procedures

// To use this procedure, execute a command such as:
// 		LoadECLabDataFile(pathName, filePath)
// where
// 		String pathName					// Name of an Igor symbolic path or ""
// 		String filePath					// Name of file or full path to file

// If you are not familiar with Igor symbolic paths, execute:
// 		DisplayHelpTopic "Symbolic Paths"

// LoadECLabDataFile automatically determines the name line, first data line, and delimiter character.
// If it gets those parameters wrong, you can use LoadECLabDataFile2 instead. With LoadECLabDataFile2,
// you specify those parameters in addition to the pathName and filePath.

// These procedures solve issues raised by the column names in EC-Lab data files.

// There are two categories of wave names in Igor: standard and liberal. Standard names start
// with a letter and can include letters, numbers, and the underscore character. Liberal names
// can include any character except for single-quote, double-quote, colon, and semicolon.

// For further discussion of standard and liberal names, execute:
// 		DisplayHelpTopic "Object Names"

// Programming with liberal names is tricky and most Igor procedures do not work with
// liberal names. Consequently, LoadWaves, by default, "cleans up" column names to produce
// standard wave names. Using LoadWave/B, you can load waves with liberal names but this
// is usually a bad idea because liberal names cause issues down the road unless you are
// very careful.

// The EC-Lab data file from Bio-Logic uses column names that contain characters that
// are not legal in Igor standard names. Examples are "<Ewe>/V" and "|Ewe|/V". Furthermore,
// when LoadWaves cleans these up, they both evaluate to X_Ewe__V, causing a name conflict.
// And, there are some EC-Lab data files that contain columns with the exact same name
// more than once - see https://www.wavemetrics.com/node/21003.

// The LoadECLabDataFile procedure handles these issues by removing characters that
// are not legal in standard Igor names and then by making sure that each name is unique.
// As a result, you wind up with waves named, for example, Ewe_V and Ewe_V1.
//
// Here is a more extensive, though not complete, list of wave names produced by LoadECLabDataFile:
// 		freq_Hz, ReZ_Ohm, ImZ_Ohm, Z_Ohm, PhaseZ_deg, time_s, Ewe_V, I_mA
//		Cs_uF, Cp_uF, cyclenumber, IRange, Ewe_V1, I_A, Q_Qo_mA_h, ReY_Ohm_1
// 		ImY_Ohm_1, Y_Ohm_1, PhaseY_deg, I_mA1, dq_mA_h, x1

static Function/S MakeNameUnique(name, nameList)
	String name				// Name that we are about to add to /B flag
	String nameList			// List of names already added to /B flag
	
	String originalName = name
	Variable index = 0
	do
		if (WhichListItem(name, nameList) < 0)
			break				// name is not in the list
		endif
		index += 1
		sprintf name, "%s%d", originalName, index		// Generate new name by appending a number	
	while(1)
	
	return name
End

// RemoveIllegalCharacters(name)
// Removes characters that we know are not legal in Igor standard names.
// Here are typical resulting wave names:
// 		freq_Hz, ReZ_Ohm, ImZ_Ohm, Z_Ohm, PhaseZ_deg, time_s, Ewe_V, I_mA
//		Cs_uF, Cp_uF, cyclenumber, IRange, Ewe_V1, I_A, Q_Qo_mA_h, ReY_Ohm_1
// 		ImY_Ohm_1, Y_Ohm_1, PhaseY_deg, I_mA1, dq_mA_h, x1
static Function/S RemoveIllegalCharacters(name)
	String name
	
	name = ReplaceString("|", name, "")
	name = ReplaceString("<", name, "")
	name = ReplaceString(">", name, "")
	name = ReplaceString("(", name, "")
	name = ReplaceString(")", name, "")
	name = ReplaceString("/", name, "_")
	name = ReplaceString(" ", name, "")
	
	return name
End

Function/S GenerateColumnInfoStr(pathName, filePath, nameLine, delimiter)
	String pathName		// Name of an Igor symbolic path or ""
	String filePath		// Name of file or full path to file
	Variable nameLine	// 0-based number of line containing column names
	String delimiter	// Typically "," or "\t" (tab)
	
	Variable refNum
	Open/R/P=$pathName refNum as filePath
	
	Variable lineNumber = 0
	do
		String text
		FReadLine refNum, text
		if (lineNumber == nameLine)
			break
		endif
		lineNumber += 1
	while(1)
	
	text = ReplaceString("\r", text, "")		// Remove any CR terminator character
	text = ReplaceString("\n", text, "")		// Remove any LF terminator character
	
	Close refNum
	
	String nameList = ""	// Accumulates names
	
	String columnInfoStr = ""
	Variable numNames = ItemsInList(text, delimiter)
	Variable index
	for(index=0; index<numNames; index+=1)
		String name = StringFromList(index, text, delimiter)
		if (strlen(name) == 0)
			name = "_skip_"		
		endif
		
		// Remove characters that we know are not legal in Igor standard names
		name = RemoveIllegalCharacters(name)
		
		// Make sure this is a legal standard wave name
		name = CleanupName(name, 0)
		
		// Make the name unique if it conflicts with a name already added to nameList
		name = MakeNameUnique(name, nameList)
		
		columnInfoStr += "N='" + name + "';"
		nameList += name + ";"
	endfor
	
	return columnInfoStr
End

// GetFileToLoad(pathName, filePath)
// If necessary, displays an Open File dialog to allow user to choose the file.
// Returns -1 if cancelled, 0 if OK.
// If the function result is 0 then, on return, pathName and filePath are suitable
// for use by LoadWave.
static Function GetFileToLoad(pathName, filePath)
	String& pathName				// Input and output: Name of an Igor symbolic path or ""
	String& filePath				// Input and output: Name of file or full path to file

	if ((strlen(pathName)==0) || (strlen(filePath)==0))
		// Display dialog looking for file.
		String fileFilters = "Text Files (*.txt):.txt;"
		fileFilters += "CSV Files (*.csv):.csv;"
		fileFilters += "DAT Files (*.dat):.dat;"
		fileFilters += "All Files:.*;"
		Variable refNum
		Open/D/R/F=fileFilters/P=$pathName refNum as filePath
		filePath = S_fileName				// S_fileName is full path set by Open/D
		if (strlen(filePath) == 0)		// User cancelled?
			return -1
		endif
	endif
	
	return 0
End

Function LoadECLabDataFile2(pathName, filePath, nameLine, firstDataLine, delimiter)
	String pathName					// Name of an Igor symbolic path or ""
	String filePath					// Name of file or full path to file
	Variable nameLine				// Zero-based line number of names line
	Variable firstDataLine		// Zero-based line number of first data line
	String delimiter				// Delimiter between names and numbers, typically "," or "\t" (tab)

	// First get a valid reference to a file
	Variable result = GetFileToLoad(pathName, filePath)
	if (result != 0)
		return result				// User cancelled Open File dialog
	endif

	String columnInfoStr = GenerateColumnInfoStr(pathName, filePath, nameLine, delimiter)
	// Print columnInfoStr
	
	LoadWave/J/Q/L={nameLine,firstDataLine,0,0,0}/A/B=columnInfoStr/E=1/P=$pathName filePath
	
	return 0
End

static Function FindFirstDataLineAndDelimiter(pathName, filePath, firstDataLine, delimiter)
	String pathName					// Name of an Igor symbolic path or ""
	String filePath					// Name of file or full path to file
	Variable& firstDataLine		// Output
	String& delimiter				// Output
	
	Variable refNum = 0
	Open/R/P=$pathName refNum as filePath
	if (refNum == 0)
		return -1						// Unexpected error
	endif
	
	Variable lineNum = 0
	do
		String text
		FReadLine refNum, text
		if (strlen(text) == 0)
			Close refNum
			return -1					// Did not find first data line - Should not happen if this is a valid EC-Lab file
		endif
		
		Variable num
		sscanf text, "%g", num
		if (V_Flag == 1)
			// The line starts with a number
			firstDataLine = lineNum
			String junk
			sscanf text, "%[^\t ,]", junk		// Deposit all characters other than tab, space, or comma into junk
			int len = strlen(junk)
			delimiter = text[len]
			break
		endif		
		
		lineNum += 1
	while(1)

	Close refNum
	return 0
End

// LoadECLabDataFile(pathName, filePath)
// Automatically determines the name line, first data line, and delimiter,
// and then calls LoadECLabDataFile2.
// It is assumed that the first line that starts with a valid number is the first
// data line and that the line before the first data line is the name line.
Function LoadECLabDataFile(pathName, filePath)
	String pathName					// Name of an Igor symbolic path or ""
	String filePath					// Name of file or full path to file

	// First get a valid reference to a file
	Variable result = GetFileToLoad(pathName, filePath)
	if (result != 0)
		return result				// User cancelled Open File dialog
	endif
	
	Variable nameLine = 0			// Zero-based line number of names line
	Variable firstDataLine = 0	// Zero-based line number of first data line
	String delimiter = ""			// Delimiter between names and numbers, typically "," or "\t" (tab)
	result = FindFirstDataLineAndDelimiter(pathName, filePath, firstDataLine, delimiter)
	if (result != 0)
		return result				// Should not happen if this is a valid EC-Lab file
	endif
	nameLine = firstDataLine - 1
	
	// For debugging only
	// Printf "Name line = %d, first data line = %d, delimiter = %d\r", nameLine, firstDataLine, char2num(delimiter)
	
	String columnInfoStr = GenerateColumnInfoStr(pathName, filePath, nameLine, delimiter)
	// Print columnInfoStr
	
	LoadWave/J/Q/L={nameLine,firstDataLine,0,0,0}/A/B=columnInfoStr/E=1/P=$pathName filePath
	
	return 0
End