Extracting compressed archives from within Igor Pro using the CallFunction XOP and libarchive

#pragma TextEncoding = "UTF-8"
#pragma rtGlobals=3     // Use modern global access method and strict wave access.

#include "FCALL_functions"

/// This example shows how the CallFunction-XOP is used to extract an archive.
///
/// The libarchive library is used for extraction (https://github.com/libarchive/libarchive)
///
/// How to run this example:
/// - have some tar or zip archive file ready you want to decompress.
///   (The archive.dll for this example is compiled with zip and tar support only.)
///
/// - Run UncompressFiles(archiveName, targetPath) -- e.g. UncompressFiles("C:\\archive\\data.zip", "C:\\analysis")

static StrConstant UCF_LIBARCHIVE_DLL = "archive.dll"

static StrConstant UCF_ARCHIVE_ENTRYPTR = "root:arcEntry"
static StrConstant UCF_ARCHIVE_BUFFERPTR = "root:arcBuffer"
static StrConstant UCF_ARCHIVE_SIZETR = "root:arcSize"
static StrConstant UCF_ARCHIVE_OFFSETPTR = "root:arcOffset"
static Constant UCF_ARCHIVE_BLOCKSIZE = 10240

static Constant ARCHIVE_EXTRACT_TIME = 0x0004
static Constant ARCHIVE_EXTRACT_PERM = 0x0002
static Constant ARCHIVE_EXTRACT_ACL = 0x0020
static Constant ARCHIVE_EXTRACT_FFLAGS = 0x0040
static Constant ARCHIVE_OK = 0
static Constant ARCHIVE_EOF = 1
static Constant ARCHIVE_WARN = -20

/// @brief Extracts an archive to a target path
/// @param[in] arcFileName full file path to the archive file
/// @param[in] targetPath file path to the location where the archive is decompressed, use "" for current path of Igor process
/// @returns NaN on error, 0 otherwise
Function UncompressFiles(arcFileName, targetPath)
    string arcFileName, targetPath

    string libHandle
    string dllPath, functionName, entryPath, entryTarget
    variable numItems, result
    UInt64 archiveRead, archiveWrite
    Int64 entrySize

    string reflectedProcpath = FunctionPath("UncompressFiles")
    numItems = ItemsInList(reflectedProcpath, ":")
    if(!numItems)
        return NaN
    endif
    reflectedProcpath = RemoveListItem(numItems - 1, reflectedProcpath, ":")
    dllPath = reflectedProcpath + UCF_LIBARCHIVE_DLL

    if(!IsEmpty(targetPath))
        targetPath = ParseFilepath(5, targetPath, "\\", 0, 0)
        targetPath = ParseFilepath(2, targetPath, "\\", 0, 0)
    endif

    UCF_CreateFunctionTemplates()
    Make/O/L/U/N=1 $UCF_ARCHIVE_ENTRYPTR, $UCF_ARCHIVE_BUFFERPTR, $UCF_ARCHIVE_SIZETR
    Make/O/L/N=1 $UCF_ARCHIVE_OFFSETPTR

    libHandle = FCALL_Load(dllPath)

    [archiveRead] = UCF_archive_read_new(libHandle)
    UCF_archive_read_support_format_all(libHandle, archiveRead)
    UCF_archive_read_support_compression_all(libHandle, archiveRead)

    [archiveWrite] = UCF_archive_write_disk_new(libHandle)
    UCF_archive_write_disk_set_options(libHandle, archiveWrite, ARCHIVE_EXTRACT_TIME | ARCHIVE_EXTRACT_PERM | ARCHIVE_EXTRACT_ACL | ARCHIVE_EXTRACT_FFLAGS)
    UCF_archive_write_disk_set_standard_lookup(libHandle, archiveWrite)

    result = UCF_archive_read_open_filename(libHandle, archiveRead, arcFileName, UCF_ARCHIVE_BLOCKSIZE)
    if(result != ARCHIVE_OK)
        printf "Error %d on open file %s\r", result, arcFileName
        FCALL_Free(libHandle)
        return NaN
    endif
    for(;;)
        result = UCF_archive_read_next_header(libHandle, archiveRead, UCF_ARCHIVE_ENTRYPTR)
        if(result == ARCHIVE_EOF)
            break
        endif
        if(result < ARCHIVE_OK)
            print "error: " + UCF_archive_error_string(libHandle, archiveRead)
        endif
        if(result < ARCHIVE_WARN)
            FCALL_Free(libHandle)
            return NaN
        endif
        WAVE entry = $UCF_ARCHIVE_ENTRYPTR
        entryPath = UCF_archive_entry_pathname(libHandle, entry[0])
        entryTarget = UCF_FormatFilePath(targetPath + entryPath)
        printf "%s ", ReplaceString("\\\\", entryTarget, "\\")
        UCF_archive_entry_set_pathname(libHandle, entry[0], entryTarget)
        result = UCF_archive_write_header(libHandle, archiveWrite, entry[0])
        if(result < ARCHIVE_OK)
            print "error: " + UCF_archive_error_string(libHandle, archiveWrite)
        else
            [entrySize] = UCF_archive_entry_size(libHandle, entry[0])
            result = copy_data(libHandle, archiveRead, archiveWrite, entrySize)
            if(result < ARCHIVE_WARN)
                FCALL_Free(libHandle)
                return NaN
            endif
        endif
        result = UCF_archive_write_finish_entry(libHandle, archiveWrite)
        if(result < ARCHIVE_OK)
            print "error: " + UCF_archive_error_string(libHandle, archiveRead)
        endif
        if(result < ARCHIVE_WARN)
            FCALL_Free(libHandle)
            return NaN
        endif
        printf "\r"
    endfor
    UCF_archive_read_close(libHandle, archiveRead)
    UCF_archive_read_free(libHandle, archiveRead)
    UCF_archive_write_close(libHandle, archiveWrite)
    UCF_archive_write_free(libHandle, archiveWrite)

    FCALL_Free(libHandle)

    return 0
End

/// @brief Copy data for one archived file from the archive to the unpacked file
///        Show some basic progress.
static Function copy_data(String libHandle, UInt64 arcread, UInt64 arcWrite, Int64 entrySize)

    variable result, perc, oldperc
    string dots
    WAVE buffer = $UCF_ARCHIVE_BUFFERPTR
    WAVE size = $UCF_ARCHIVE_SIZETR
    WAVE offset = $UCF_ARCHIVE_OFFSETPTR

    printf "["
    for(;;)
        result = UCF_archive_read_data_block(libHandle, arcread, UCF_ARCHIVE_BUFFERPTR, UCF_ARCHIVE_SIZETR, UCF_ARCHIVE_OFFSETPTR)
        if(result == ARCHIVE_EOF)
            dots = PadString("", 100 - oldperc, char2num("."))
            printf dots + "]"
            return ARCHIVE_OK
        endif
        if(result < ARCHIVE_OK)
            print "error: " + UCF_archive_error_string(libHandle, arcread)
            return result
        endif
        result = UCF_archive_write_data_block(libHandle, arcWrite, buffer[0], size[0], offset[0])
        if(result < ARCHIVE_OK)
            print "error: " + UCF_archive_error_string(libHandle, arcWrite)
            return result
        endif
        if(entrySize)
            perc = trunc(offset[0] * 100 / entrySize)
            if(perc != oldperc)
                dots = PadString("", perc - oldperc, char2num("."))
                printf dots
                oldperc = perc
                DoUpdate
            endif
        endif
    endfor
End

/// Function wrappers for the library calls

static Function [UInt64 arcHandle] UCF_archive_read_new(String libHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_read_new"

    jsonID = jsonIDList[%$(funcName)]

    [arcHandle] = UCF_ParseResultToUInt64(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))

    return [arcHandle]
End

static Function UCF_archive_read_support_format_all(String libHandle, UInt64 arcHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_read_support_format_all"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

    JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_support_compression_all(String libHandle, UInt64 arcHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_read_support_compression_all"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

    JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function [UInt64 arcHandle] UCF_archive_write_disk_new(String libHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_write_disk_new"

    jsonID = jsonIDList[%$(funcName)]

    [arcHandle] = UCF_ParseResultToUInt64(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))

    return [arcHandle]
End

static Function UCF_archive_write_disk_set_options(String libHandle, UInt64 arcHandle, Variable flags)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_write_disk_set_options"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
    JSON_SetVariable(jsonID, "/Parameter/1/value", flags)

    JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_write_disk_set_standard_lookup(String libHandle, UInt64 arcHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_write_disk_set_standard_lookup"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

    JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_open_filename(String libHandle, UInt64 arcHandle, String fileName, Variable blockSize)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_read_open_filename"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
    JSON_SetString(jsonID, "/Parameter/1/value", fileName)
    JSON_SetVariable(jsonID, "/Parameter/2/value", blockSize)

    return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_next_header(String libHandle, UInt64 arcHandle, String entryPtr)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_read_next_header"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
    JSON_SetString(jsonID, "/Parameter/1/value", entryPtr)

    return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function/S UCF_archive_error_string(String libHandle, UInt64 arcHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_error_string"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

    return UCF_ParseResultToString(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_write_header(String libHandle, UInt64 arcHandle, UInt64 entry)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_write_header"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
    JSON_SetUInt64(jsonID, "/Parameter/1/value", entry)

    return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function [Int64 entrySize] UCF_archive_entry_size(String libHandle, UInt64 arcEntry)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_entry_size"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcEntry)

    [entrySize] = UCF_ParseResultToInt64(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))

    return [entrySize]
End

static Function UCF_archive_read_data_block(String libHandle, UInt64 arcHandle, String bufferPtr, String sizePtr, String offsetPtr)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_read_data_block"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
    JSON_SetString(jsonID, "/Parameter/1/value", bufferPtr)
    JSON_SetString(jsonID, "/Parameter/2/value", sizePtr)
    JSON_SetString(jsonID, "/Parameter/3/value", offsetPtr)

    return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_write_data_block(String libHandle, UInt64 arcHandle, UInt64 buffer, UInt64 size, Int64 offset)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_write_data_block"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)
    JSON_SetUInt64(jsonID, "/Parameter/1/value", buffer)
    JSON_SetUInt64(jsonID, "/Parameter/2/value", size)
    JSON_SetInt64(jsonID, "/Parameter/3/value", offset)

    return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_write_finish_entry(String libHandle, UInt64 arcHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_write_finish_entry"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

    return UCF_ParseResultToNumber(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_close(String libHandle, UInt64 arcHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_read_close"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

    JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_read_free(String libHandle, UInt64 arcHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_read_free"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

    JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function/S UCF_archive_write_close(String libHandle, UInt64 arcHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_write_close"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

    JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function/S UCF_archive_write_free(String libHandle, UInt64 arcHandle)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_write_free"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcHandle)

    JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function/S UCF_archive_entry_pathname(String libHandle, UInt64 arcEntry)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_entry_pathname"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcEntry)

    return UCF_ParseResultToString(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

static Function UCF_archive_entry_set_pathname(String libHandle, UInt64 arcEntry, String pathName)

    variable jsonID
    WAVE jsonIDList
    WAVE/T functionNameList
    string funcName = "archive_entry_set_pathname"

    jsonID = jsonIDList[%$(funcName)]
    JSON_SetUInt64(jsonID, "/Parameter/0/value", arcEntry)
    JSON_SetString(jsonID, "/Parameter/1/value", pathName)

    JSON_Release(FCALL_Call(libHandle, functionNameList[%$(funcName)], jsonID))
End

/// Various support functions

static Function/S UCF_FormatFilePath(fPath)
    string fPath

    string winFilePath = ParseFilepath(5, fPath, "\\", 0, 0)
    return ReplaceString("\\", winFilePath, "\\\\")
End

static Function UCF_ParseResultToNumber(Variable jsonID)

    variable result = JSON_GetVariable(jsonID, "/result/value")
    JSON_Release(jsonID)

    return result
End

static Function/S UCF_ParseResultToString(Variable jsonID)

    string result = JSON_GetString(jsonID, "/result/value")
    JSON_Release(jsonID)

    return result
End


static Function [Int64 result] UCF_ParseResultToInt64(Variable jsonID)

    [result] = JSON_GetInt64(jsonID, "/result/value")
    JSON_Release(jsonID)

    return [result]
End

static Function [UInt64 result] UCF_ParseResultToUInt64(Variable jsonID)

    [result] = JSON_GetUInt64(jsonID, "/result/value")
    JSON_Release(jsonID)

    return [result]
End

threadsafe static Function IsEmpty(str)
    string& str

    variable len = strlen(str)
    return numtype(len) == 2 || len <= 0
End

/// @brief Creates a json template for a specific library function
///        The templates are later used in the wrapper function that execute the call to the library
static Function UCF_AddFunctionTemplate(functionName, paramList, [callFunctionName])
    string functionName, paramList, callFunctionName

    WAVE jsonIDList
    WAVE/T functionNameList
    variable size

    size = DimSize(jsonIDList, 0)
    Redimension/N=(size + 1) jsonIDList, functionNameList

    functionNameList[size] = SelectString(ParamIsDefault(callFunctionName), callFunctionName, functionName)
    jsonIDList[size] = FCALL_SetupParameterIn(paramList)
    SetDimLabel 0, size, $functionName, jsonIDList, functionNameList
End

/// @brief Creates json templates for libarchive functions
static Function UCF_CreateFunctionTemplates()

    Make/O/D/N=(0) jsonIDList
    Make/O/T/N=(0) functionNameList

    // archive_read_new
    UCF_AddFunctionTemplate("archive_read_new", "UINT64")
    // archive_read_support_format_all
    UCF_AddFunctionTemplate("archive_read_support_format_all", "INT32;UINT64")
    // archive_read_support_compression_all
    UCF_AddFunctionTemplate("archive_read_support_compression_all", "INT32;UINT64")
    // archive_write_disk_new
    UCF_AddFunctionTemplate("archive_write_disk_new", "UINT64")
    // archive_write_disk_set_options
    UCF_AddFunctionTemplate("archive_write_disk_set_options", "INT32;UINT64;INT32")
    // archive_write_disk_set_standard_lookup
    UCF_AddFunctionTemplate("archive_write_disk_set_standard_lookup", "INT32;UINT64")
    // archive_read_open_filename
    UCF_AddFunctionTemplate("archive_read_open_filename", "INT32;UINT64;STRING;UINT64")
    // archive_read_next_header
    UCF_AddFunctionTemplate("archive_read_next_header", "INT32;UINT64;WAVEREF")
    // archive_error_string
    UCF_AddFunctionTemplate("archive_error_string", "STRING;UINT64")
    // archive_write_header
    UCF_AddFunctionTemplate("archive_write_header", "INT32;UINT64;UINT64")
    //  archive_entry_size
    UCF_AddFunctionTemplate("archive_entry_size", "INT32;UINT64")
    // archive_read_data_block
    UCF_AddFunctionTemplate("archive_read_data_block", "INT32;UINT64;WAVEREF;WAVEREF;WAVEREF")
    // archive_write_data_block
    UCF_AddFunctionTemplate("archive_write_data_block", "INT32;UINT64;UINT64;UINT64;INT64")
    // archive_write_finish_entry
    UCF_AddFunctionTemplate("archive_write_finish_entry", "INT32;UINT64")
    // archive_read_close
    UCF_AddFunctionTemplate("archive_read_close", "INT32;UINT64")
    // archive_read_free
    UCF_AddFunctionTemplate("archive_read_free", "INT32;UINT64")
    // archive_write_close
    UCF_AddFunctionTemplate("archive_write_close", "INT32;UINT64")
    // archive_write_free
    UCF_AddFunctionTemplate("archive_write_free", "INT32;UINT64")
    // archive_entry_pathname
    UCF_AddFunctionTemplate("archive_entry_pathname", "STRING;UINT64")
    // archive_entry_set_pathname
    UCF_AddFunctionTemplate("archive_entry_set_pathname", "INT32;UINT64;STRING")
End