Extracting compressed archives from within Igor Pro using the CallFunction XOP and libarchive

The following Igor Pro code uses the CallFunction XOP to uncompress zip and tar files.
 

#pragma TextEncoding = "UTF-8"
#pragma rtGlobals=3		// Use modern global access method and strict wave access.

#include "FCALL_functions"

/// This example shows how the CallFunction-XOP is used to extract an archive.
///
/// The libarchive library is used for extraction (https://github.com/libarchive/libarchive)
///
/// How to run this example:
/// - have some tar or zip archive file ready you want to decompress.
///   (The archive.dll for this example is compiled with zip and tar support only.)
///
/// - Run UncompressFiles(archiveName, targetPath) -- e.g. UncompressFiles("C:\\archive\\data.zip", "C:\\analysis")

static StrConstant UCF_LIBARCHIVE_DLL = "archive.dll"

static StrConstant UCF_ARCHIVE_ENTRYPTR = "root:arcEntry"
static StrConstant UCF_ARCHIVE_BUFFERPTR = "root:arcBuffer"
static StrConstant UCF_ARCHIVE_SIZETR = "root:arcSize"
static StrConstant UCF_ARCHIVE_OFFSETPTR = "root:arcOffset"
static Constant UCF_ARCHIVE_BLOCKSIZE = 10240

static Constant ARCHIVE_EXTRACT_TIME = 0x0004
static Constant ARCHIVE_EXTRACT_PERM = 0x0002
static Constant ARCHIVE_EXTRACT_ACL = 0x0020
static Constant ARCHIVE_EXTRACT_FFLAGS = 0x0040
static Constant ARCHIVE_OK = 0
static Constant ARCHIVE_EOF = 1
static Constant ARCHIVE_WARN = -20

/// @brief Extracts an archive to a target path
/// @param[in] arcFileName full file path to the archive file
/// @param[in] targetPath file path to the location where the archive is decompressed, use "" for current path of Igor process
/// @returns NaN on error, 0 otherwise
Function UncompressFiles(arcFileName, targetPath)
	string arcFileName, targetPath

	string libHandle
	string dllPath, functionName, entryPath, entryTarget
	variable numItems, result
	UInt64 archiveRead, archiveWrite
	Int64 entrySize

	string reflectedProcpath = FunctionPath("UncompressFiles")
	numItems = ItemsInList(reflectedProcpath, ":")
	if(!numItems)
		return NaN
	endif
	reflectedProcpath = RemoveListItem(numItems - 1, reflectedProcpath, ":")
	dllPath = reflectedProcpath + UCF_LIBARCHIVE_DLL

	if(!IsEmpty(targetPath))
		targetPath = ParseFilepath(5, targetPath, "\\", 0, 0)
		targetPath = ParseFilepath(2, targetPath, "\\", 0, 0)
	endif

	UCF_CreateFunctionTemplates()
	Make/O/L/U/N=1 $UCF_ARCHIVE_ENTRYPTR, $UCF_ARCHIVE_BUFFERPTR, $UCF_ARCHIVE_SIZETR
	Make/O/L/N=1 $UCF_ARCHIVE_OFFSETPTR

	libHandle = FCALL_Load(dllPath)

	[archiveRead] = UCF_archive_read_new(libHandle)
	UCF_archive_read_support_format_all(libHandle, archiveRead)
	UCF_archive_read_support_compression_all(libHandle, archiveRead)

	[archiveWrite] = UCF_archive_write_disk_new(libHandle)
	UCF_archive_write_disk_set_options(libHandle, archiveWrite, ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_PERM | ARCHIVE_EXTRACT_ACL | ARCHIVE_EXTRACT_FFLAGS)
	UCF_archive_write_disk_set_standard_lookup(libHandle, archiveWrite)

	result = UCF_archive_read_open_filename(libHandle, archiveRead, arcFileName, UCF_ARCHIVE_BLOCKSIZE)
	if(result != ARCHIVE_OK)
		printf "Error %d on open file %s\r", result, arcFileName
		FCALL_Free(libHandle)
		return NaN
	endif
	for(;;)
		result = UCF_archive_read_next_header(libHandle, archiveRead, UCF_ARCHIVE_ENTRYPTR)
		if(result == ARCHIVE_EOF)
			break
		endif
		if(result < ARCHIVE_OK)
			print "error: " + UCF_archive_error_string(libHandle, archiveRead)
		endif
		if(result < ARCHIVE_WARN)
			FCALL_Free(libHandle)
			return NaN
		endif
		WAVE entry = $UCF_ARCHIVE_ENTRYPTR
		entryPath = UCF_archive_entry_pathname(libHandle, entry[0])
		entryTarget = UCF_FormatFilePath(targetPath + entryPath)
		printf "%s ", ReplaceString("\\\\", entryTarget, "\\")
		UCF_archive_entry_set_pathname(libHandle, entry[0], entryTarget)
		result = UCF_archive_write_header(libHandle, archiveWrite, entry[0])
		if(result < ARCHIVE_OK)
			print "error: " + UCF_archive_error_string(libHandle, archiveWrite)
		else
			[entrySize] = UCF_archive_entry_size(libHandle, entry[0])
			result = copy_data(libHandle, archiveRead, archiveWrite, entrySize)
			if(result < ARCHIVE_WARN)
				FCALL_Free(libHandle)
				return NaN
			endif
		endif
		result = UCF_archive_write_finish_entry(libHandle, archiveWrite)
		if(result < ARCHIVE_OK)
			print "error: " + UCF_archive_error_string(libHandle, archiveRead)
		endif
		if(result < ARCHIVE_WARN)
			FCALL_Free(libHandle)
			return NaN
		endif
		printf "\r"
	endfor
	UCF_archive_read_close(libHandle, archiveRead)
	UCF_archive_read_free(libHandle, archiveRead)
	UCF_archive_write_close(libHandle, archiveWrite)
	UCF_archive_write_free(libHandle, archiveWrite)

	FCALL_Free(libHandle)

	return 0
End

/// @brief Copy data for one archived file from the archive to the unpacked file
///        Show some basic progress.
static Function copy_data(String libHandle, UInt64 arcread, UInt64 arcWrite, Int64 entrySize)

	variable result, perc, oldperc
	string dots
	WAVE buffer = $UCF_ARCHIVE_BUFFERPTR
	WAVE size = $UCF_ARCHIVE_SIZETR
	WAVE offset = $UCF_ARCHIVE_OFFSETPTR

	printf "["
	for(;;)
		result = UCF_archive_read_data_block(libHandle, arcread, UCF_ARCHIVE_BUFFERPTR, UCF_ARCHIVE_SIZETR, UCF_ARCHIVE_OFFSETPTR)
		if(result == ARCHIVE_EOF)
			dots = PadString("", 100 - oldperc, char2num("."))
			printf dots + "]"
			return ARCHIVE_OK
		endif
		if(result < ARCHIVE_OK)
			print "error: " + UCF_archive_error_string(libHandle, arcread)
			return result
		endif
		result = UCF_archive_write_data_block(libHandle, arcWrite, buffer[0], size[0], offset[0])
		if(result < ARCHIVE_OK)
			print "error: " + UCF_archive_error_string(libHandle, arcWrite)
			return result
		endif
		if(entrySize)
			perc = trunc(offset[0] * 100 / entrySize)
			if(perc != oldperc)
				dots = PadString("", perc - oldperc, char2num("."))
				printf dots
				oldperc = perc
				DoUpdate
			endif
		endif
	endfor
End

/// Function wrappers for the library calls

static Function [UInt64 arcHandle] UCF_archive_read_new(String libHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_read_new"

	jsonID = jsonIDList[%$(funcName)]

	[arcHandle] = UCF_ParseResultToUInt64(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))

	return [arcHandle]
End

static Function UCF_archive_read_support_format_all(String libHandle, UInt64 arcHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_read_support_format_all"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

	JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_support_compression_all(String libHandle, UInt64 arcHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_read_support_compression_all"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

	JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function [UInt64 arcHandle] UCF_archive_write_disk_new(String libHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_write_disk_new"

	jsonID = jsonIDList[%$(funcName)]

	[arcHandle] = UCF_ParseResultToUInt64(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))

	return [arcHandle]
End

static Function UCF_archive_write_disk_set_options(String libHandle, UInt64 arcHandle, Variable flags)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_write_disk_set_options"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
	JSON_SetVariable(jsonID, "/Parameter/1/value", flags)

	JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_write_disk_set_standard_lookup(String libHandle, UInt64 arcHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_write_disk_set_standard_lookup"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

	JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_open_filename(String libHandle, UInt64 arcHandle, String fileName, Variable blockSize)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_read_open_filename"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
	JSON_SetString(jsonID, "/Parameter/1/value", fileName)
	JSON_SetVariable(jsonID, "/Parameter/2/value", blockSize)

	return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_next_header(String libHandle, UInt64 arcHandle, String entryPtr)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_read_next_header"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
	JSON_SetString(jsonID, "/Parameter/1/value", entryPtr)

	return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function/S UCF_archive_error_string(String libHandle, UInt64 arcHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_error_string"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

	return UCF_ParseResultToString(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_write_header(String libHandle, UInt64 arcHandle, UInt64 entry)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_write_header"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
	JSON_SetUInt64(jsonID, "/Parameter/1/value", entry)

	return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function [Int64 entrySize] UCF_archive_entry_size(String libHandle, UInt64 arcEntry)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_entry_size"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcEntry)

	[entrySize] = UCF_ParseResultToInt64(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))

	return [entrySize]
End

static Function UCF_archive_read_data_block(String libHandle, UInt64 arcHandle, String bufferPtr, String sizePtr, String offsetPtr)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_read_data_block"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
	JSON_SetString(jsonID, "/Parameter/1/value", bufferPtr)
	JSON_SetString(jsonID, "/Parameter/2/value", sizePtr)
	JSON_SetString(jsonID, "/Parameter/3/value", offsetPtr)

	return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_write_data_block(String libHandle, UInt64 arcHandle, UInt64 buffer, UInt64 size, Int64 offset)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_write_data_block"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
	JSON_SetUInt64(jsonID, "/Parameter/1/value", buffer)
	JSON_SetUInt64(jsonID, "/Parameter/2/value", size)
	JSON_SetInt64(jsonID, "/Parameter/3/value", offset)

	return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_write_finish_entry(String libHandle, UInt64 arcHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_write_finish_entry"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

	return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_close(String libHandle, UInt64 arcHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_read_close"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

	JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_free(String libHandle, UInt64 arcHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_read_free"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

	JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function/S UCF_archive_write_close(String libHandle, UInt64 arcHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_write_close"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

	JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function/S UCF_archive_write_free(String libHandle, UInt64 arcHandle)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_write_free"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

	JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function/S UCF_archive_entry_pathname(String libHandle, UInt64 arcEntry)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_entry_pathname"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcEntry)

	return UCF_ParseResultToString(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_entry_set_pathname(String libHandle, UInt64 arcEntry, String pathName)

	variable jsonID
	WAVE jsonIDList
	WAVE/T functionNameList
	string funcName = "archive_entry_set_pathname"

	jsonID = jsonIDList[%$(funcName)]
	JSON_SetUInt64(jsonID, "/Parameter/0/value", arcEntry)
	JSON_SetString(jsonID, "/Parameter/1/value", pathName)

	JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

/// Various support functions

static Function/S UCF_FormatFilePath(fPath)
	string fPath

	string winFilePath = ParseFilepath(5, fPath, "\\", 0, 0)
	return ReplaceString("\\", winFilePath, "\\\\")
End

static Function UCF_ParseResultToNumber(Variable jsonID)

	variable result = JSON_GetVariable(jsonID, "/result/value")
	JSON_Release(jsonID)

	return result
End

static Function/S UCF_ParseResultToString(Variable jsonID)

	string result = JSON_GetString(jsonID, "/result/value")
	JSON_Release(jsonID)

	return result
End


static Function [Int64 result] UCF_ParseResultToInt64(Variable jsonID)

	[result] = JSON_GetInt64(jsonID, "/result/value")
	JSON_Release(jsonID)

	return [result]
End

static Function [UInt64 result] UCF_ParseResultToUInt64(Variable jsonID)

	[result] = JSON_GetUInt64(jsonID, "/result/value")
	JSON_Release(jsonID)

	return [result]
End

threadsafe static Function IsEmpty(str)
	string& str

	variable len = strlen(str)
	return numtype(len) == 2 || len <= 0
End

/// @brief Creates a json template for a specific library function
///        The templates are later used in the wrapper function that execute the call to the library
static Function UCF_AddFunctionTemplate(functionName, paramList, [callFunctionName])
	string functionName, paramList, callFunctionName

	WAVE jsonIDList
	WAVE/T functionNameList
	variable size

	size = DimSize(jsonIDList, 0)
	Redimension/N=(size + 1) jsonIDList, functionNameList

	functionNameList[size] = SelectString(ParamIsDefault(callFunctionName), callFunctionName, functionName)
	jsonIDList[size] = FCALL_SetupParameterIn(paramList)
	SetDimLabel 0, size, $functionName, jsonIDList, functionNameList
End

/// @brief Creates json templates for libarchive functions
static Function UCF_CreateFunctionTemplates()

	Make/O/D/N=(0) jsonIDList
	Make/O/T/N=(0) functionNameList

	// archive_read_new
	UCF_AddFunctionTemplate("archive_read_new", "UINT64")
	// archive_read_support_format_all
	UCF_AddFunctionTemplate("archive_read_support_format_all", "INT32;UINT64")
	// archive_read_support_compression_all
	UCF_AddFunctionTemplate("archive_read_support_compression_all", "INT32;UINT64")
	// archive_write_disk_new
	UCF_AddFunctionTemplate("archive_write_disk_new", "UINT64")
	// archive_write_disk_set_options
	UCF_AddFunctionTemplate("archive_write_disk_set_options", "INT32;UINT64;INT32")
	// archive_write_disk_set_standard_lookup
	UCF_AddFunctionTemplate("archive_write_disk_set_standard_lookup", "INT32;UINT64")
	// archive_read_open_filename
	UCF_AddFunctionTemplate("archive_read_open_filename", "INT32;UINT64;STRING;UINT64")
	// archive_read_next_header
	UCF_AddFunctionTemplate("archive_read_next_header", "INT32;UINT64;WAVEREF")
	// archive_error_string
	UCF_AddFunctionTemplate("archive_error_string", "STRING;UINT64")
	// archive_write_header
	UCF_AddFunctionTemplate("archive_write_header", "INT32;UINT64;UINT64")
	// 	archive_entry_size
	UCF_AddFunctionTemplate("archive_entry_size", "INT32;UINT64")
	// archive_read_data_block
	UCF_AddFunctionTemplate("archive_read_data_block", "INT32;UINT64;WAVEREF;WAVEREF;WAVEREF")
	// archive_write_data_block
	UCF_AddFunctionTemplate("archive_write_data_block", "INT32;UINT64;UINT64;UINT64;INT64")
	// archive_write_finish_entry
	UCF_AddFunctionTemplate("archive_write_finish_entry", "INT32;UINT64")
	// archive_read_close
	UCF_AddFunctionTemplate("archive_read_close", "INT32;UINT64")
	// archive_read_free
	UCF_AddFunctionTemplate("archive_read_free", "INT32;UINT64")
	// archive_write_close
	UCF_AddFunctionTemplate("archive_write_close", "INT32;UINT64")
	// archive_write_free
	UCF_AddFunctionTemplate("archive_write_free", "INT32;UINT64")
	// archive_entry_pathname
	UCF_AddFunctionTemplate("archive_entry_pathname", "STRING;UINT64")
	// archive_entry_set_pathname
	UCF_AddFunctionTemplate("archive_entry_set_pathname", "INT32;UINT64;STRING")
End