Load JPK Data File

#pragma rtGlobals=3		// Use modern global access method and strict wave access.

// Load JPK Data File
// This file loader load a JPK Instruments segmented data file like the one posted at http://www.igorexchange.com/node/7321
// These files contain one or more "segments" of data.
// To use the file loader, choose Data->Load JPK Data File.
// The file loader creates a data folder based on the file name.
// It creates a data folder for each segment in the file. These are named Segment0, Segment1, ...
// The data from each segment is loaded into the corresponding data folder
// in the form of waves. The waves are named according to the "# columns" tag in the segment.

Menu "Load Waves"
	"Load JPK Data File...", LoadJPKDataFile("", "")
End

Structure JPKDataFileHeaderInfo
	Variable numSegments				// Specified by "# force-settings.segments.size:" tag
	Variable startSegmentsFPos			// File position of first segment, specified by "# segmentIndex: 0" tag
	Variable startSegmentsLineNumber	// Zero-based line number of start of first segment
EndStructure

static Function TextStartsWith(text, tagText, tagValueText)	// Returns the truth that text starts with tagText
	String text
	String tagText
	String& tagValueText				// The text after the tagText and followign space
	
	tagValueText = ""
	
	Variable len = strlen(tagText)
	String subText = text[0,len-1]
	if (CmpStr(subText,tagText) == 0)
		tagValueText = text[len+1,10000]		// +1 to skip space after tag text
		return 1
	endif
	
	return 0
End

// GetJPKDataFileHeaderInfo(refNum, fileInfo)
// Returns information about a JPK file from the file header.
static Function GetJPKDataFileHeaderInfo(refNum, fileInfo)
	Variable refNum			// File reference number from Open command
	STRUCT JPKDataFileHeaderInfo& fileInfo
	
	FSetPos refNum, 0

	Variable lineNumber = 0
	do
		Variable fPos
		FStatus refNum
		fPos = V_FilePos			// Current file position for the file in bytes from the start
		
		String text
		FReadLine refNum, text
		if (strlen(text) == 0)
			// Reached end-of-file
			Printf "GetJPKDataFileHeaderInfo hit end of file without reading all required fields"
			return -1
		endif
		
		String tagText, tagValueText
		
		tagText = "# force-settings.segments.size:"
		if (TextStartsWith(text,tagText, tagValueText))
			fileInfo.numSegments = str2num(tagValueText)
		endif
		
		tagText = "# segmentIndex: 0"
		if (TextStartsWith(text,tagText, tagValueText))
			// Hit start of segments
			fileInfo.startSegmentsFPos = fPos
			fileInfo.startSegmentsLineNumber = lineNumber
			break
		endif
		
		lineNumber += 1
	while(1)
	
	return 0
End

// MakeJPKCatalog(refNum)
// Returns a 2D wave describing the segments of a file.
//	Column 0: File position of start of segment ("# segmentIndex:" tag)
//	Column 1: Line number of start of segment ("# segmentIndex:" tag)
//	Column 2: Line number of column names line ("# columns:" tag)
//	Column 3: Line number of start of segment data
//	Column 4: Line number of last line of segment data
// All line numbers are zero-based.
static Function/WAVE MakeJPKCatalog(refNum)
	Variable refNum			// File reference number from Open command
	
	FSetPos refNum, 0

	// The information about the file is returned in this wave
	Make /O /N=(1000,6) JPKCatalog = NaN			// Assume never more than 1000 segments
	SetDimLabel 1, 0, SegmentStartFPos, JPKCatalog	// Column 0 contains the segment file position
	SetDimLabel 1, 1, SegmentStartLine, JPKCatalog		// Column 1 contains the segment start line number
	SetDimLabel 1, 2, ColumnNamesFPos, JPKCatalog	// Column 2 contains the column names file position
	SetDimLabel 1, 3, ColumnNamesLine, JPKCatalog	// Column 3 contains the column names line number
	SetDimLabel 1, 4, DataStartLine, JPKCatalog			// Column 5 contains the data start line number
	SetDimLabel 1, 5, DataEndLine, JPKCatalog			// Column 6 contains the data end line number

	Variable numSegments = 0
	Variable segmentNumber = -1	// No segments yet
	Variable inData = 0

	Variable lineNumber = 0
	do
		Variable fPos
		FStatus refNum
		fPos = V_FilePos			// Current file position for the file in bytes from the start
		
		String text
		FReadLine refNum, text
		if (strlen(text) == 0)
			// Reached end-of-file
			if (segmentNumber >= 0)						// Hit start of segments already?
				JPKCatalog[segmentNumber][%DataEndLine] = lineNumber - 1
			endif
			break
		endif
		
		String tagText, tagValueText
		
		tagText = "# segmentIndex:"
		if (TextStartsWith(text,tagText, tagValueText))
			segmentNumber += 1							// Start of new segment
			numSegments += 1
			JPKCatalog[segmentNumber][] = NaN
			JPKCatalog[segmentNumber][%SegmentStartFPos] = fPos
			JPKCatalog[segmentNumber][%SegmentStartLine] = lineNumber
		endif
		
		tagText = "# columns:"
		if (TextStartsWith(text,tagText, tagValueText))
			JPKCatalog[segmentNumber][%ColumnNamesFPos] = fPos
			JPKCatalog[segmentNumber][%ColumnNamesLine] = lineNumber
		endif

		if (segmentNumber >= 0)				// Hit start of segments already?
			if (!inData)
				if (CmpStr(text[0],"#") != 0)			// A line that does not start with # is a data line
					JPKCatalog[segmentNumber][%DataStartLine] = lineNumber
					inData = 1
				endif
			endif
		endif

		if (segmentNumber >= 0)				// Hit start of segments already?
			if (inData)
				if (CmpStr(text,"\r") == 0)			// Blank line signifies end of segment data
					JPKCatalog[segmentNumber][%DataEndLine] = lineNumber - 1
					inData = 0
				endif
			endif
		endif
		
		lineNumber += 1
	while(1)
	
	Redimension /N=(numSegments,6) JPKCatalog
	
	return JPKCatalog	
End

static Function SkipSpaces(text, len, pos)
	String text
	Variable len
	Variable pos
	
	Variable i
	for(i=0; i<len; i+=1)
		if (CmpStr(text[pos]," ") != 0)
			break
		else
			pos += 1
		endif
	endfor
	return pos
End

// GetJPKSegmentColumnInfoStr(refNum, columnNamesFPos)
// Returns a string suitable for the LoadWave /B flag. The string specifies the column
// name to use for each column in the segment. The name line in the segment looks
// something like this:
// # columns: tipSampleSeparation vDeflection height error smoothedCapacitiveSensorHeight capacitiveSensorHeight hDeflection seriesTime time xTipPosition yTipPosition
// There may be multiple spaces between names.
static Function/S GetJPKSegmentColumnInfoStr(refNum, columnNamesFPos)
	Variable refNum					// File reference number from Open command
	Variable columnNamesFPos
	
	FSetPos refNum, columnNamesFPos
	
	String text
	FReadLine refNum, text
	Variable len = strlen(text)
	
	String tagText = "# columns: "
	text = ReplaceString(tagText, text, "")			// Remove tag
	
	String str = ""
	
	Variable pos = 0
	Variable numNames = 0
	
	do
		Variable origPos = pos
		pos = SkipSpaces(text, len, pos)
		String name
		sscanf text[pos,len-1], "%s", name
		
		name = CleanupName(name, 0)
		
		// This converts, e.g., Time into Time0. Time is a built-in function and is not allowed for wave names.
		Variable tmp = Exists(name)
		if (tmp!=0 && tmp!=1)
			name = UniqueName(name, 1, 0)
		endif

		String temp = "N='" + name + "';"		// e.g., "N=vDeflection;"
		str += temp
		numNames += 1
		
		pos += strlen(name)
	while(pos < len)

	return str
End

static Function LoadJPKSegment(pathName, filePath, refNum, segmentNumber, segmentStartFPos, segmentStartLine, columnNamesFPos, dataStartLine, dataNumLines) 
	String pathName				// Name of an Igor symbolic path or ""
	String filePath					// Name of file or partial path relative to symbolic path or full path to file
	Variable refNum					// File reference number from Open command
	Variable segmentNumber
	Variable segmentStartFPos
	Variable segmentStartLine
	Variable columnNamesFPos
	Variable dataStartLine
	Variable dataNumLines
	
	// Each segment is loaded into a separate data folder
	String dfName = "Segment" + num2istr(segmentNumber)
	NewDataFolder /O /S $dfName
	
	String columnInfoStr = GetJPKSegmentColumnInfoStr(refNum, columnNamesFPos)
	
	LoadWave /G /O /A=Column /L={0, dataStartLine, dataNumLines, 0, 0} /B=columnInfoStr /P=$pathName /Q filePath
	
	SetDataFolder ::					// Reset current data folder to original
	
	return 0
End

static Function/S FileNameToDFName(filePath)
	String filePath

	String fileName = ParseFilePath(3, filePath, ":", 0, 0)
	String dfName = CleanupName(fileName, 0)		// Convert to standard Igor name for easier programming
	return dfName	
End

// LoadJPKDataFile(pathName, filePath)
// Loads Ωdata and header information from JPK Instruments segmented .txt files.
// If pathName or filePath is "", an Open File dialog is displayed.
// Creates a main data folder based on the file name.
// Each segment of the file is loaded into a separate data folder in the main data folder.
// For example if your file is named "MyFile" and contains 3 segments and you call this while
// the current data folder is root:, you get the following hierarchy:
//	root:
//		MyFile
//			Segment0
//			Segment1
//			Segment2
// The data and header information for a given segment is loaded into the corresponding
// segment data folder.
// If a data folder already exists, conflicting waves and variables are overwritten.
Function LoadJPKDataFile(pathName, filePath)
	String pathName				// Name of an Igor symbolic path or ""
	String filePath					// Name of file or partial path relative to symbolic path or full path to file

	Variable refNum = 0

	// First get a valid reference to a file.
	if ((strlen(pathName)==0) || (strlen(filePath)==0))
		// Display dialog looking for file.
		String fileFilters = "JPK Files (*.txt):.txt;"
		fileFilters += "All Files:.*;"
		Open/D/R/F=fileFilters/P=$pathName refNum as filePath
		filePath = S_fileName					// S_fileName is set by Open/D
		if (strlen(filePath) == 0)					// User cancelled?
			return -1
		endif
	endif

	// Open the file for reading
	Open /R /P=$pathName refNum as filePath
	if (refNum == 0)
		return -1								// Error opening file
	endif

	STRUCT JPKDataFileHeaderInfo fileInfo
	
	Variable result = GetJPKDataFileHeaderInfo(refNum, fileInfo)
	if (result != 0)
		Close refNum
		return result
	endif
	
	// Print fileInfo			// For debugging only
	
	Wave/Z catalog = MakeJPKCatalog(refNum)
	if (!WaveExists(catalog))
		Print "Error in MakeJPKCatalog"
		Close refNum
		return -1
	endif
	
	// Make a data folder for the file
	String dfName = FileNameToDFName(filePath)
	NewDataFolder /O /S $dfName
	
	Variable numSegments = DimSize(catalog,0)
	Variable segmentNumber
	Variable segmentsLoaded = 0
	for(segmentNumber = 0; segmentNumber<numSegments; segmentNumber+=1)
		Variable segmentStartFPos = catalog[segmentNumber][%SegmentStartFPos]
		Variable segmentStartLine = catalog[segmentNumber][%SegmentStartLine]
		Variable columnNamesFPos = catalog[segmentNumber][%ColumnNamesFPos]
		Variable columnNamesLine = catalog[segmentNumber][%ColumnNamesLine]
		Variable dataStartLine = catalog[segmentNumber][%DataStartLine]
		Variable dataEndLine = catalog[segmentNumber][%DataEndLine]
		Variable dataNumLines = dataEndLine - dataStartLine + 1
		
		result = LoadJPKSegment(pathName, filePath, refNum, segmentNumber, segmentStartFPos, segmentStartLine, columnNamesFPos, dataStartLine, dataNumLines) 
		if (result != 0)
			Printf "Error loading segment %d\r", segmentNumber
			break
		endif
		
		segmentsLoaded += 1
	endfor
	
	SetDataFolder ::				// Reset current data folder to original

	Close refNum
	
	Printf "Created data folder %s containing %d segments from \"%s\"\r", dfName, segmentsLoaded, filePath
	
	return 0
End