diff --git a/Makefile b/Makefile index b01624ac81..8d3c5ad9a2 100644 --- a/Makefile +++ b/Makefile @@ -319,7 +319,7 @@ clean-assets: rm -f $(DATA_ASM_SUBDIR)/layouts/layouts.inc $(DATA_ASM_SUBDIR)/layouts/layouts_table.inc rm -f $(DATA_ASM_SUBDIR)/maps/connections.inc $(DATA_ASM_SUBDIR)/maps/events.inc $(DATA_ASM_SUBDIR)/maps/groups.inc $(DATA_ASM_SUBDIR)/maps/headers.inc $(DATA_SRC_SUBDIR)/map_group_count.h find sound -iname '*.bin' -exec rm {} + - find . \( -iname '*.1bpp' -o -iname '*.4bpp' -o -iname '*.8bpp' -o -iname '*.gbapal' -o -iname '*.lz' -o -iname '*.smol' -o -iname '*.fastSmol' -o -iname '*.rl' -o -iname '*.latfont' -o -iname '*.hwjpnfont' -o -iname '*.fwjpnfont' \) -exec rm {} + + find . \( -iname '*.1bpp' -o -iname '*.4bpp' -o -iname '*.8bpp' -o -iname '*.gbapal' -o -iname '*.lz' -o -iname '*.smol' -o -iname '*.fastSmol' -o -iname '*.smolTM' -o -iname '*.rl' -o -iname '*.latfont' -o -iname '*.hwjpnfont' -o -iname '*.fwjpnfont' \) -exec rm {} + find $(DATA_ASM_SUBDIR)/maps \( -iname 'connections.inc' -o -iname 'events.inc' -o -iname 'header.inc' \) -exec rm {} + tidy: tidymodern tidycheck tidydebug diff --git a/tools/compresSmol/Makefile b/tools/compresSmol/Makefile index 768de9d4a7..e4799483f7 100644 --- a/tools/compresSmol/Makefile +++ b/tools/compresSmol/Makefile @@ -1,14 +1,14 @@ CXX ?= g++ -CXXFLAGS := -Werror -std=c++17 -pthread -O2 -Wunused +CXXFLAGS := -Werror -std=c++17 -O2 INCLUDES := -I . -SRCS := compresSmol.cpp compressAlgo.cpp tANS.cpp fileDispatcher.cpp -TILEMAP_SRCS := mainTiles.cpp compressAlgo.cpp compressSmolTiles.cpp tANS.cpp fileDispatcher.cpp +SRCS := compresSmol.cpp compressAlgo.cpp tANS.cpp +TILEMAP_SRCS := mainTiles.cpp compressSmolTiles.cpp tANS.cpp compressAlgo.cpp -HEADERS := compressAlgo.h tANS.h fileDispatcher.h -TILEMAP_HEADERS := compressAlgo.h compressSmolTiles.h tANS.h fileDispatcher.h +HEADERS := compressAlgo.h tANS.h +TILEMAP_HEADERS := compressSmolTiles.h tANS.h compressAlgo.h ifeq ($(OS),Windows_NT) EXE := .exe diff --git a/tools/compresSmol/compresSmol.cpp b/tools/compresSmol/compresSmol.cpp index 80d7ae9f88..1f7d2b1cb3 100644 --- a/tools/compresSmol/compresSmol.cpp +++ b/tools/compresSmol/compresSmol.cpp @@ -18,7 +18,6 @@ bool isNumber(std::string input) } enum Option { - ANALYZE, WRITE, FRAME_WRITE, DECODE, @@ -31,15 +30,12 @@ int main(int argc, char *argv[]) bool printUsage = false; std::string input; std::string output; - int numThreads = 1; InputSettings settings(true, true, true); if (argc > 1) { std::string argument = argv[1]; - if (argument.compare("-a") == 0) - option = ANALYZE; - else if (argument.compare("-w") == 0) + if (argument.compare("-w") == 0) option = WRITE; else if (argument.compare("-fw") == 0) option = FRAME_WRITE; @@ -48,46 +44,12 @@ int main(int argc, char *argv[]) } switch (option) { - case ANALYZE: - if (argc > 2) - input = argv[2]; - else - printUsage = true; - if (argc > 4) - { - std::string arg2 = argv[3]; - std::string arg2arg = argv[4]; - if (arg2.compare("-t") == 0 && isNumber(arg2arg)) - numThreads = std::stoi(arg2arg.c_str()); - } - if (argc > 7) - { - std::string setting1 = argv[5]; - std::string setting2 = argv[6]; - std::string setting3 = argv[7]; - if (setting1.compare("true") == 0) - settings.canEncodeLO = true; - else if (setting1.compare("false") == 0) - settings.canEncodeLO = false; - else - fprintf(stderr, "Unrecognized setting1 \"%s\", defaulting to \"true\"\n", setting1.c_str()); - if (setting2.compare("true") == 0) - settings.canEncodeSyms = true; - else if (setting2.compare("false") == 0) - settings.canEncodeSyms = false; - else - fprintf(stderr, "Unrecognized setting2 \"%s\", defaulting to \"true\"\n", setting2.c_str()); - if (setting3.compare("true") == 0) - settings.canDeltaSyms = true; - else if (setting3.compare("false") == 0) - settings.canDeltaSyms = false; - else - fprintf(stderr, "Unrecognized setting3 \"%s\", defaulting to \"true\"\n", setting3.c_str()); - } - break; case FRAME_WRITE: + // Not implemented yet + fprintf(stderr, "Frame writing isn't implemented yet\n"); settings.useFrames = true; option = WRITE; + return 1; case WRITE: if (argc > 3) { @@ -141,10 +103,6 @@ int main(int argc, char *argv[]) if (printUsage) { printf("Usage:\n\ - %s -a \"path/to/some/directory\"\n\ - Analyses all images currently in .4bpp.lz format and compares with this compression.\n\ - -t can be appended to this mode to specify how many threads to use.\n\ - \n\ %s -w \"path/to/some/file.4bpp\" \"path/to/some/file.4bpp.smol\"\ Compresses the first argument and writes the result to the second argument.\n\ These modes can also be appended with 4 true/false statements that control the following settings of the compression:\n\ @@ -153,64 +111,11 @@ int main(int argc, char *argv[]) - If the compression instructions can be delta encoded.\n\ - If the raw symbols in the compression ca be delta encoded.\n\ %s -d \"path/to/some/file.4bpp.smol\" \"path/to/some/file.4bpp\"\n\ - Decompresses the first argument and writes it to the second argument.", argv[0], argv[0], argv[0]); + Decompresses the first argument and writes it to the second argument.", argv[0], argv[0]); return 0; } - if (option == ANALYZE) - { - std::filesystem::path dirPath = input; - FileDispatcher dispatcher(dirPath); - if (!dispatcher.initFileList()) - { - fprintf(stderr, "Failed to init file list\n"); - return 1; - } - std::mutex dispatchMutex; - std::vector allImages; - std::mutex imageMutex; - - settings.shouldCompare = true; - - std::vector threads; - for (int i = 0; i < numThreads; i++) - { - threads.emplace_back(analyzeImages, &allImages, &imageMutex, - &dispatcher, &dispatchMutex, - settings); - } - - for (int i = 0; i < numThreads; i++) - threads[i].join(); - - size_t lzSizes = 0; - size_t newSizes = 0; - size_t rawSizes = 0; - size_t totalImages = 0; - size_t invalidImages = 0; - for (CompressedImage currImage : allImages) - { - totalImages++; - if (currImage.isValid) - { - lzSizes += currImage.lzSize; - newSizes += currImage.compressedSize; - rawSizes += currImage.rawNumBytes; - } - else - { - fprintf(stderr, "Failed to solve %s\n", currImage.fileName.c_str()); - invalidImages++; - } - } - - fprintf(stderr, "RawSize: %zu\n", rawSizes); - fprintf(stderr, "LZsize: %zu\n", lzSizes); - fprintf(stderr, "SmolSize: %zu\n", newSizes); - fprintf(stderr, "Total Images: %zu\n", totalImages); - fprintf(stderr, "Invalid Images: %zu\n", invalidImages); - } if (option == WRITE) { if (std::filesystem::exists(input)) @@ -240,8 +145,13 @@ int main(int argc, char *argv[]) { if (std::filesystem::exists(input)) { - std::vector inData = readFileAsUInt(input); - std::vector image4bpp = readRawDataVecs(&inData); + std::vector inData; + if (!readFileAsUInt(input, &inData)) + { + return 0; + } + std::vector image4bpp; + readRawDataVecs(&inData, &image4bpp); std::vector charVec(image4bpp.size()*2); for (size_t i = 0; i < image4bpp.size(); i++) { diff --git a/tools/compresSmol/compressAlgo.cpp b/tools/compresSmol/compressAlgo.cpp index 07ac59da81..261abcf24a 100644 --- a/tools/compresSmol/compressAlgo.cpp +++ b/tools/compresSmol/compressAlgo.cpp @@ -1,18 +1,30 @@ #include "compressAlgo.h" -std::vector getShortCopies(std::vector input, size_t minLength) +ShortCopy::ShortCopy(size_t index, size_t length, size_t offset, unsigned short firstSymbol) { - std::vector copies; - for (size_t startIndex = 1; startIndex < input.size(); startIndex++) + this->index = index; + this->length = length; + this->offset = offset; + this->firstSymbol = firstSymbol; +} + +bool getShortCopies(std::vector *pInput, size_t minLength, std::vector *pShortCopies) +{ + size_t iteration = 0; + std::vector checkVec(pInput->size()); + for (size_t i = 0; i < checkVec.size(); i++) + checkVec[i] = ' '; + for (size_t startIndex = 1; startIndex < pInput->size(); startIndex++) { size_t longestLength = 0; size_t longestOffset; for (size_t searchOffset = 1; searchOffset <= startIndex && searchOffset < 32767; searchOffset++) { size_t currLength = 0; - while (input[startIndex + currLength] == input[startIndex + currLength - searchOffset] - && startIndex + currLength < input.size()) + while (startIndex + currLength < pInput->size() + && (*pInput)[startIndex + currLength] == (*pInput)[startIndex + currLength - searchOffset]) currLength++; + if (currLength > longestLength) { longestLength = currLength; @@ -24,118 +36,118 @@ std::vector getShortCopies(std::vector input, size_t longestLength = 32767; if (longestLength >= minLength) { - std::vector::const_iterator start = input.begin() + startIndex; - std::vector::const_iterator end = input.begin() + startIndex + longestLength; - copies.push_back(ShortCopy(startIndex, longestLength, longestOffset, std::vector(start, end))); - copies[copies.size() - 1].firstSymbol = input[startIndex - 1]; + // Handle non-copies + if (startIndex > 0) + checkVec[startIndex - 1] = 'X'; + + for (size_t i = 0; i < longestLength; i++) + { + checkVec[startIndex + i] = 'X'; + } + for (size_t i = 0; i < startIndex - 1; i++) + { + if (checkVec[i] == ' ') + { + size_t extraIndex = i; + checkVec[extraIndex] = 'O'; + size_t extraLength = 1; + while (checkVec[extraIndex + extraLength] == ' ') + { + checkVec[extraIndex + extraLength] = 'O'; + extraLength++; + } + pShortCopies->push_back(ShortCopy(extraIndex, extraLength, 0, 0)); + break; + } + } + + pShortCopies->push_back(ShortCopy(startIndex, longestLength, longestOffset, (*pInput)[startIndex - 1])); startIndex += longestLength; } } - std::vector checkVec(input.size()); - for (ShortCopy copy : copies) - for (size_t i = 0; i <= copy.length; i++) - checkVec[copy.index + i - 1]++; - size_t currStart = 0; - size_t currLength = 1; - unsigned short prevSym = checkVec[0]; - for (size_t i = 1; i < checkVec.size(); i++) - { - unsigned short currSym = checkVec[i]; - if (currSym == 0 && prevSym == 0) - currLength++; - else if (currSym == 0 && prevSym == 1) - { - currStart = i; - currLength = 1; - } - else if (currSym == 1 && prevSym == 0) - { - std::vector::const_iterator start = input.begin() + currStart; - std::vector::const_iterator end = input.begin() + currStart + currLength; - copies.push_back(ShortCopy(currStart, currLength, 0, std::vector(start, end))); - } - prevSym = currSym; - } - if (prevSym == 0) - { - std::vector::const_iterator start = input.begin() + currStart; - std::vector::const_iterator end = input.begin() + currStart + currLength; - copies.push_back(ShortCopy(currStart, currLength, 0, std::vector(start, end))); - } - return copies; -} - - -bool verifyShortCopies(std::vector *pCopies, std::vector *pImage) -{ - std::vector decodedImage(pImage->size()); - for (ShortCopy copy : (*pCopies)) + // Handle trailing characters + if (checkVec.back() == ' ') { - if (copy.offset == 0) + size_t index = 0; + if (pShortCopies->size() > 0) { - for (size_t i = 0; i < copy.length; i++) - decodedImage[copy.index + i] = copy.usSequence[i]; + index = pShortCopies->back().index + pShortCopies->back().length; + size_t length = checkVec.size() - index; + pShortCopies->push_back(ShortCopy(index, length, 0, 0)); } else { - decodedImage[copy.index - 1] = copy.firstSymbol; - for (size_t i = 0; i < copy.length; i++) - decodedImage[copy.index + i] = (*pImage)[copy.index + i - copy.offset]; + pShortCopies->push_back(ShortCopy(0, checkVec.size(), 0, 0)); } } - for (size_t i = 0; i < decodedImage.size(); i++) - if (decodedImage[i] != (*pImage)[i]) - return false; + + return verifyShortCopies(pShortCopies, pInput); +} + +bool verifyShortCopies(std::vector *pCopies, std::vector *pImage) +{ + size_t totalLength = 0; + for (ShortCopy copy : (*pCopies)) + { + totalLength += copy.length; + if (copy.offset != 0) + { + totalLength++; + } + + if (copy.offset != 0) + { + if (copy.firstSymbol != (*pImage)[copy.index - 1]) + return false; + + for (size_t i = 0; i < copy.length; i++) + { + if ((*pImage)[copy.index + i] != (*pImage)[copy.index + i - copy.offset]) + return false; + } + } + } + if (totalLength != pImage->size()) + return false; + return true; } -SortedShortElement::SortedShortElement() {} - -SortedShortElement::SortedShortElement(size_t index, ShortCopy copy) +bool getShortInstructions(std::vector *pCopies, std::vector *pInstructions, std::vector *pInput) { - this->index = index; - this->copy = copy; - isCopy = true; -} - -std::vector getShortInstructions(std::vector copies, size_t lengthMod) -{ - std::vector instructions; - std::vector unsortedElements; - std::vector sortedElements; - - for (ShortCopy copy : copies) - unsortedElements.push_back(SortedShortElement(copy.index, copy)); - - while (unsortedElements.size() != 0) - { - size_t smallestIndex = 0; - for (size_t i = 0; i < unsortedElements.size(); i++) - { - if (unsortedElements[i].index < unsortedElements[smallestIndex].index) - { - smallestIndex = i; - } - } - sortedElements.push_back(unsortedElements[smallestIndex]); - unsortedElements.erase(unsortedElements.begin() + smallestIndex); - } - for (size_t i = 0; i < sortedElements.size(); i++) + for (ShortCopy copy : (*pCopies)) { ShortCompressionInstruction currInstruction; - currInstruction.index = sortedElements[i].index; - currInstruction.offset = sortedElements[i].copy.offset; - currInstruction.length = sortedElements[i].copy.length; - currInstruction.symbols = sortedElements[i].copy.usSequence; - currInstruction.firstSymbol = sortedElements[i].copy.firstSymbol; - currInstruction.buildBytes(); - instructions.push_back(currInstruction); + currInstruction.index = copy.index; + currInstruction.offset = copy.offset; + currInstruction.length = copy.length; + currInstruction.firstSymbol = copy.firstSymbol; + currInstruction.buildBytes(pInput); + pInstructions->push_back(currInstruction); } - return instructions; + return true; } -void ShortCompressionInstruction::buildBytes() +void getLosFromInstructions(std::vector *pInstructions, std::vector *pOutput) +{ + for (ShortCompressionInstruction instruction : (*pInstructions)) + { + for (unsigned char uc : instruction.loBytes) + pOutput->push_back(uc); + } +} + +void getSymsFromInstructions(std::vector *pInstructions, std::vector *pOutput) +{ + for (ShortCompressionInstruction instruction : (*pInstructions)) + { + for (unsigned short us : instruction.symShorts) + pOutput->push_back(us); + } +} + +void ShortCompressionInstruction::buildBytes(std::vector *pInput) { if (offset != 0) { @@ -173,74 +185,14 @@ void ShortCompressionInstruction::buildBytes() loBytes[loBytes.size() - 1] += LO_CONTINUE_BIT; loBytes.push_back(currLength & BYTE_MASK); } - for (unsigned short currSymbol : symbols) - symShorts.push_back(currSymbol); - } -} - -void CompressionInstruction::buildBytes() -{ - std::vector currBytes; - if (offset != 0) - { - size_t currLength = length; - currBytes.push_back(currLength & 0x7f); - currLength = currLength >> 7; - if (currLength != 0) + for (size_t i = 0; i < length; i++) { - currBytes[currBytes.size() - 1] += 128; - currBytes.push_back(currLength & 0xff); + symShorts.push_back((*pInput)[index + i]); } - size_t currOffset = offset; - currBytes.push_back(currOffset & 0x7f); - currOffset = currOffset >> 7; - if (currOffset != 0) - { - currBytes[currBytes.size() - 1] += 128; - currBytes.push_back(currOffset & 0xff); - } - currBytes.push_back(firstSymbol); } - else - { - currBytes.push_back(0); - size_t currLength = length; - currBytes.push_back(currLength & 0x7f); - currLength = currLength >> 7; - if (currLength != 0) - { - currBytes[currBytes.size() - 1] += 128; - currBytes.push_back(currLength & 0xff); - } - for (unsigned char currSymbol : symbols) - currBytes.push_back(currSymbol); - } - bytes = currBytes; } -std::vector getLosFromInstructions(std::vector instructions) -{ - std::vector loVec; - for (ShortCompressionInstruction instruction : instructions) - { - for (unsigned char uc : instruction.loBytes) - loVec.push_back(uc); - } - return loVec; -} - -std::vector getSymsFromInstructions(std::vector instructions) -{ - std::vector symvec; - for (ShortCompressionInstruction instruction : instructions) - { - for (unsigned short uc : instruction.symShorts) - symvec.push_back(uc); - } - return symvec; -} - -std::vector decodeBytesShort(std::vector *pLoVec, std::vector *pSymVec) +bool verifyBytesShort(std::vector *pLoVec, std::vector *pSymVec, std::vector *pImage) { std::vector decodedImage; size_t loIndex = 0; @@ -279,13 +231,7 @@ std::vector decodeBytesShort(std::vector *pLoVec, } } } - return decodedImage; -} - -bool verifyBytesShort(std::vector *pLoVec, std::vector *pSymVec, std::vector *pImage) -{ - std::vector shorts = decodeBytesShort(pLoVec, pSymVec); - return compareVectorsShort(&shorts, pImage); + return true; } bool compareVectorsShort(std::vector *pVec1, std::vector *pVec2) @@ -298,407 +244,38 @@ bool compareVectorsShort(std::vector *pVec1, std::vector readFileAsUC(std::string filePath) +bool isModeLoEncoded(CompressionMode mode) { - std::ifstream iStream; - iStream.open(filePath.c_str(), std::ios::binary); - if (!iStream.is_open()) - { - fprintf(stderr, "Error: Couldn't open %s for reading bytes\n", filePath.c_str()); - return std::vector(0); - } - iStream.ignore( std::numeric_limits::max() ); - std::streamsize size = iStream.gcount(); - iStream.clear(); - iStream.seekg( 0, std::ios_base::beg ); - std::vector ucVec(size); - iStream.read((char*)(&ucVec[0]), size); - iStream.close(); - return ucVec; + if (mode == ENCODE_LO + || mode == ENCODE_BOTH + || mode == ENCODE_BOTH_DELTA_SYMS) + return true; + return false; } -CompressedImage processImage(std::string fileName, InputSettings settings) +bool isModeSymEncoded(CompressionMode mode) { - CompressedImage image; - std::vector input = readFileAsUC(fileName); - if (settings.useFrames) - { - /* - // Determine number of frames - size_t totalPixels = input.size()*2; - // Split input and append - size_t smallFrames = totalPixels/OVERWORLD_16X16; - size_t largeFrames = totalPixels/OVERWORLD_32X32; - */ - } - else - { - image = processImageData(input, settings, fileName); - } - return image; + if (mode == ENCODE_SYMS + || mode == ENCODE_DELTA_SYMS + || mode == ENCODE_BOTH + || mode == ENCODE_BOTH_DELTA_SYMS) + return true; + return false; } -CompressedImage processImageFrames(std::string fileName, InputSettings settings) +bool isModeSymDelta(CompressionMode mode) { - CompressedImage image; - std::vector input = readFileAsUC(fileName); - std::vector> allInputs(4); - size_t totalSize = input.size(); - size_t partialSize = totalSize/4; - size_t subIndex = 0; - size_t inputIndex = 0; - std::vector frameOffsets; - for (unsigned char currChar : input) - { - frameOffsets.push_back(image.otherBits.size()); - if (subIndex == partialSize) - { - subIndex = 0; - inputIndex++; - } - allInputs[inputIndex].push_back(currChar); - subIndex++; - } - for (size_t i = 0; i < 4; i++) - { - CompressedImage tempImage = processImageData(allInputs[i], settings, fileName); - for (unsigned int currVal : tempImage.writeVec) - image.otherBits.push_back(currVal); - } - unsigned int header = IS_FRAME_CONTAINER; - image.writeVec.push_back(header); - for (size_t i = 0; i < 4; i++) - image.writeVec.push_back((unsigned int)frameOffsets[i]); - for (unsigned int currVal : image.otherBits) - image.writeVec.push_back(currVal); - image.isValid = true; - - return image; + if (mode == ENCODE_DELTA_SYMS + || mode == ENCODE_BOTH_DELTA_SYMS) + return true; + return false; } -CompressedImage processImageData(std::vector input, InputSettings settings, std::string fileName) -{ - CompressedImage bestImage; - CompressionMode someMode; - bool hasImage = false; - std::vector rawBase = input; - std::vector usBase(rawBase.size()/2); - memcpy(usBase.data(), rawBase.data(), rawBase.size()); - size_t baseLZsize = 0; - bool byteFail = false; - bool copyFail = false; - bool compressionFail = false; - bool uIntConversionFail = false; - if (settings.shouldCompare) - { - std::string lzName = fileName + ".lz"; - baseLZsize = getFileSize(lzName); - } - std::vector bestLO; - std::vector bestSym; - std::vector bestInstructions; - for (size_t minCodeLength = 2; minCodeLength <= 15; minCodeLength++) - { - std::vector shortCopies = getShortCopies(usBase, minCodeLength); - if (!verifyShortCopies(&shortCopies, &usBase)) - { - copyFail = true; - continue; - } - std::vector shortInstructions = getShortInstructions(shortCopies, minCodeLength-1); - std::vector loVec = getLosFromInstructions(shortInstructions); - std::vector symVec = getSymsFromInstructions(shortInstructions); - if (!verifyBytesShort(&loVec, &symVec, &usBase)) - { - byteFail = true; - continue; - } - CompressionMode mode = BASE_ONLY; - //std::vector modesToUse = {ENCODE_SYMS}; - std::vector modesToUse = {BASE_ONLY, ENCODE_SYMS, ENCODE_DELTA_SYMS, ENCODE_LO, ENCODE_BOTH, ENCODE_BOTH_DELTA_SYMS}; - if (fileName.find("test/compression/") != std::string::npos) - { - if (fileName.find("mode_0.4bpp") != std::string::npos) - modesToUse = {BASE_ONLY}; - else if (fileName.find("mode_1.4bpp") != std::string::npos) - modesToUse = {ENCODE_SYMS}; - else if (fileName.find("mode_2.4bpp") != std::string::npos) - modesToUse = {ENCODE_DELTA_SYMS}; - else if (fileName.find("mode_3.4bpp") != std::string::npos) - modesToUse = {ENCODE_LO}; - else if (fileName.find("mode_4.4bpp") != std::string::npos) - modesToUse = {ENCODE_BOTH}; - else if (fileName.find("mode_5.4bpp") != std::string::npos) - modesToUse = {ENCODE_BOTH_DELTA_SYMS}; - else if (fileName.find("test/compression/table_") != std::string::npos) - modesToUse = {ENCODE_SYMS}; - - if (modesToUse.size() == 1) - { - settings.canDeltaSyms = true; - settings.canEncodeLO = true; - settings.canEncodeSyms = true; - } - } - - for (CompressionMode currMode : modesToUse) - { - mode = currMode; - if (!settings.canDeltaSyms - && (mode == ENCODE_DELTA_SYMS - || mode == ENCODE_BOTH_DELTA_SYMS)) - continue; - if (!settings.canEncodeLO - && (mode == ENCODE_LO - || mode == ENCODE_BOTH - || mode == ENCODE_BOTH_DELTA_SYMS)) - continue; - if (!settings.canEncodeSyms - && (mode == ENCODE_SYMS - || mode == ENCODE_BOTH - || mode == ENCODE_DELTA_SYMS - || mode == ENCODE_BOTH_DELTA_SYMS)) - continue; - CompressedImage image = fillCompressVecNew(loVec, symVec, mode,rawBase.size(), fileName); - if (!verifyCompressionShort(&image, &usBase)) - { - compressionFail = true; - continue; - } - std::vector uiVec = getUIntVecFromData(&image); - std::vector decodedImage = readRawDataVecs(&uiVec); - if (!compareVectorsShort(&decodedImage, &usBase)) - { - uIntConversionFail = true; - continue; - } - image.compressedSize = uiVec.size() * 4; - if (!hasImage) - { - bestLO = loVec; - bestSym = symVec; - bestInstructions = shortInstructions; - bestImage = image; - hasImage = true; - bestImage.writeVec = uiVec; - someMode = mode; - } - else if (image.compressedSize < bestImage.compressedSize) - { - bestLO = loVec; - bestSym = symVec; - bestInstructions = shortInstructions; - bestImage = image; - hasImage = true; - bestImage.writeVec = uiVec; - someMode = mode; - } - } - } - bestImage.mode = someMode; - bestImage.fileName = fileName; - bestImage.lzSize = baseLZsize; - bestImage.rawNumBytes = rawBase.size(); - if (hasImage) - bestImage.isValid = true; - else - { - fprintf(stderr, "Failed to compress image %s\nErrors: ", fileName.c_str()); - if (copyFail) - fprintf(stderr, "CopyProcessing "); - if (byteFail) - fprintf(stderr, "ByteConversion "); - if (compressionFail) - fprintf(stderr, "Compression "); - if (uIntConversionFail) - fprintf(stderr, "uIntConversion "); - printf("\n"); - } - return bestImage; -} - -CompressedImage getDataFromUIntVec(std::vector *pInput) -{ - CompressedImage image = readNewHeader(pInput); - - size_t readIndex = 2; - - bool loEncoded = isModeLoEncoded(image.mode); - bool symEncoded = isModeSymEncoded(image.mode); - - if (loEncoded) - { - image.loFreqs[0] = (*pInput)[readIndex]; - image.loFreqs[1] = (*pInput)[readIndex + 1]; - image.loFreqs[2] = (*pInput)[readIndex + 2]; - readIndex += 3; - } - if (symEncoded) - { - image.symFreqs[0] = (*pInput)[readIndex]; - image.symFreqs[1] = (*pInput)[readIndex + 1]; - image.symFreqs[2] = (*pInput)[readIndex + 2]; - readIndex += 3; - } - if (loEncoded || symEncoded) - { - for (size_t i = 0; i < image.bitreamSize; i++) - { - image.tANSbits.push_back((*pInput)[readIndex]); - readIndex++; - } - } - std::vector remainders; - while (readIndex < pInput->size()) - { - unsigned int currInt = (*pInput)[readIndex]; - for (size_t i = 0; i < 4; i++) - remainders.push_back((currInt >> (8*i)) & 0xff); - readIndex++; - } - size_t remIndex = 0; - if (!symEncoded) - { - for (size_t i = 0; i < image.symSize; i++) - { - unsigned short currSym = remainders[remIndex]; - remIndex++; - currSym += remainders[remIndex] << 8; - remIndex++; - image.symVec.push_back(currSym); - } - } - if (!loEncoded) - { - for (size_t i = 0; i < image.loSize; i++) - { - unsigned char currChar = remainders[remIndex]; - remIndex++; - image.loVec.push_back(currChar); - } - } - - return image; -} - -bool verifyCompressionShort(CompressedImage *pInput, std::vector *pImage) -{ - std::vector decodedImage = decodeImageShort(pInput); - return compareVectorsShort(&decodedImage, pImage); -} - -std::vector decodeImageShort(CompressedImage *pInput) -{ - DataVecs dataVecs = decodeDataVectorsNew(pInput); - return decodeBytesShort(&dataVecs.loVec, &dataVecs.symVec); -} - -DataVecs decodeDataVectorsNew(CompressedImage *pInput) -{ - CompressedImage headerValues = readNewHeader(&pInput->headers); - size_t loSize = headerValues.loSize; - size_t symSize = headerValues.symSize; - CompressionMode mode = headerValues.mode; - bool loEncoded = isModeLoEncoded(mode); - bool symEncoded = isModeSymEncoded(mode); - bool symDelta = isModeSymDelta(mode); - std::vector loFreqs = unpackFrequencies(&pInput->loFreqs[0]); - std::vector symFreqs = unpackFrequencies(&pInput->symFreqs[0]); - std::vector symbols = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; - std::vector loDecode; - std::vector symDecode; - - std::vector loVec(loSize); - std::vector symVec(symSize); - std::vector loNibbles(loSize*2); - std::vector symNibbles(symSize*4); - - if (loEncoded) - { - loDecode = createDecodingTable(symbols, loFreqs); - } - if (symEncoded) - { - symDecode = createDecodingTable(symbols, symFreqs); - } - - std::vector allBits(pInput->tANSbits.size()*32); - size_t currIndex = 0; - for (unsigned int ui : pInput->tANSbits) - for (size_t i = 0; i < 32; i++) - { - unsigned int currVal = (ui >> i) & 0x1; - allBits[currIndex] = currVal; - currIndex++; - } - - if (!symEncoded) - for (size_t i = 0; i < symSize; i++) - symVec[i] = pInput->symVec[i]; - if (!loEncoded) - for (size_t i = 0; i < loSize; i++) - loVec[i] = pInput->loVec[i]; - - size_t bitIndex = 0; - int currState = pInput->initialState; - if (loEncoded) - { - bitIndex = decodeNibbles(loDecode, &allBits, &currState, &loNibbles, bitIndex, loSize*2); - } - if (loEncoded) - for (size_t i = 0; i < loVec.size(); i++) - loVec[i] = loNibbles[2*i] + (loNibbles[2*i + 1] << 4); - - if (symEncoded) - { - bitIndex = decodeNibbles(symDecode, &allBits, &currState, &symNibbles, bitIndex, symSize*4); - if (symDelta) - deltaDecode(&symNibbles, symNibbles.size()); - } - if (symEncoded) - for (size_t i = 0; i < symVec.size(); i++) - for (size_t j = 0; j < 4; j++) - symVec[i] += (unsigned short)symNibbles[i*4 + j] << (j*4); - - DataVecs returnData; - returnData.loVec = loVec; - returnData.symVec = symVec; - return returnData; -} - -void analyzeImages(std::vector *allImages, std::mutex *imageMutex, FileDispatcher *dispatcher, std::mutex *dispatchMutex, InputSettings settings) -{ - std::string fileName = "Initial Value"; - while (fileName != "") - { - dispatchMutex->lock(); - fileName = dispatcher->requestFileName(); - dispatchMutex->unlock(); - if (fileName == "") - break; - std::string uncompressedFileName = fileName.substr(0, fileName.size()-3); - CompressedImage currImage = processImage(uncompressedFileName, settings); - - imageMutex->lock(); - allImages->push_back(currImage); - imageMutex->unlock(); - } -} - -ShortCopy::ShortCopy() {} -ShortCopy::ShortCopy(size_t index, size_t length, size_t offset, std::vector usSequence) -{ - this->index = index; - this->length = length; - this->offset = offset; - this->usSequence = usSequence; -} - -std::vector getNormalizedCounts(std::vector input) +std::vector getNormalizedCounts(std::vector *input) { std::vector tempVec(16); - for (size_t i = 0; i < input.size(); i++) - tempVec[i] = input[i]; + for (size_t i = 0; i < input->size(); i++) + tempVec[i] = (*input)[i]; tempVec = normalizeCounts(tempVec, TANS_TABLE_SIZE); bool shouldAdjust = false; for (int i = 0; i < 16; i++) @@ -742,499 +319,6 @@ std::vector getFreqWriteInts(std::vector input) return returnVec; } -std::vector unpackFrequencies(unsigned int pInts[3]) -{ - std::vector returnVec; - int freq15 = 0; - for (size_t i = 0; i < 3; i++) - { - for (size_t j = 0; j < 5; j++) - { - int currVal = (pInts[i] >> (6*j)) & 0x3f; - returnVec.push_back(currVal); - } - freq15 += ((pInts[i] >> 30) & 0x3) << (2*i); - } - returnVec.push_back(freq15); - return returnVec; -} - -int findInitialState(EncodeCol encodeCol, unsigned char firstSymbol) -{ - for (size_t i = 0; i < encodeCol.symbols.size(); i++) - { - if (encodeCol.symbols[i].symbol == firstSymbol) - return encodeCol.symbols[i].nextState; - } - return -1; -} -/* - mode 5 - - state 6 11 - bitstream 12 23 8628 max so far, divide by 4 - loLength 12 23 2922 max so far - symLength 14 37 14k max so far - imageSize 10 47 16384 max so far, divide by 16 - - u32:5 mode+lz - u32:10 image size, divided by 32 - u32:14 symlength, divided by 2 - - u32:6 state - u32:12 bitstream, in ints - u32:12 loLength, in bytes -*/ -std::vector getNewHeaders(CompressionMode mode, size_t imageSize, size_t symLength, int initialState, size_t bitstreamSize, size_t loLength) -{ - if (initialState == -1) - initialState = 0; - std::vector returnVec(2); - - returnVec[0] += (unsigned int)mode; // 4 bits - returnVec[0] += (imageSize/IMAGE_SIZE_MODIFIER) << IMAGE_SIZE_OFFSET; // 14 bits - returnVec[0] += (symLength) << SYM_SIZE_OFFSET; // 14 bits - // 32 total - - returnVec[1] += initialState; // 6 bits - returnVec[1] += bitstreamSize << BITSTREAM_SIZE_OFFSET; // 13 bits - returnVec[1] += loLength << LO_SIZE_OFFSET; // 13 bits - - return returnVec; -} - -CompressedImage readNewHeader(std::vector *pInput) -{ - CompressedImage image; - std::vector headers(2); - headers[0] = (*pInput)[0]; - headers[1] = (*pInput)[1]; - image.mode = (CompressionMode)(headers[0] & MODE_MASK); - image.rawNumBytes = ((headers[0] >> IMAGE_SIZE_OFFSET) & IMAGE_SIZE_MASK) * IMAGE_SIZE_MODIFIER; - image.symSize = ((headers[0] >> SYM_SIZE_OFFSET) & SYM_SIZE_MASK); - - image.initialState = headers[1] & INITIAL_STATE_MASK; - image.bitreamSize = (headers[1] >> BITSTREAM_SIZE_OFFSET) & BITSTREAM_SIZE_MASK; - image.loSize = (headers[1] >> LO_SIZE_OFFSET) & LO_SIZE_MASK; - image.headers = headers; - return image; -} - -InputSettings::InputSettings() {} - -InputSettings::InputSettings(bool canEncodeLO, bool canEncodeSyms, bool canDeltaSyms) -{ - this->canEncodeLO = canEncodeLO; - this->canEncodeSyms = canEncodeSyms; - this->canDeltaSyms = canDeltaSyms; -} - -std::vector getUIntVecFromData(CompressedImage *pImage) -{ - CompressedImage otherImage = readNewHeader(&pImage->headers); - std::vector returnVec; - returnVec.push_back(pImage->headers[0]); - returnVec.push_back(pImage->headers[1]); - if (isModeLoEncoded(otherImage.mode)) - for (size_t i = 0; i < 3; i++) - returnVec.push_back(pImage->loFreqs[i]); - if (isModeSymEncoded(otherImage.mode)) - for (size_t i = 0; i < 3; i++) - returnVec.push_back(pImage->symFreqs[i]); - if (isModeLoEncoded(otherImage.mode) || isModeSymEncoded(otherImage.mode)) - for (unsigned int ui : pImage->tANSbits) - returnVec.push_back(ui); - unsigned int currInt = 0; - unsigned int currOffset = 0; - if (!isModeSymEncoded(otherImage.mode)) - { - for (size_t i = 0; i < pImage->symVec.size(); i++) - { - currInt += pImage->symVec[i] << (8*(currOffset % 4)); - currOffset += 2; - if (currOffset % 4 == 0) - { - returnVec.push_back(currInt); - currInt = 0; - } - } - } - if (!isModeLoEncoded(otherImage.mode)) - { - for (size_t i = 0; i < pImage->loVec.size(); i++) - { - currInt += 0; - currInt += pImage->loVec[i] << (8*(currOffset % 4)); - currOffset++; - if (currOffset % 4 == 0) - { - returnVec.push_back(currInt); - currInt = 0; - } - } - } - if (currOffset != 0) - returnVec.push_back(currInt); - return returnVec; -} - -bool verifyUIntVecShort(std::vector *pInput, std::vector *pImage) -{ - CompressedImage image = getDataFromUIntVec(pInput); - return verifyCompressionShort(&image, pImage); -} - -std::vector readFileAsUInt(std::string filePath) -{ - std::vector returnVec; - std::vector ucVec; - std::ifstream iStream; - iStream.open(filePath.c_str(), std::ios::binary); - if (!iStream.is_open()) - { - fprintf(stderr, "Error: Couldn't open file %s for reading\n", filePath.c_str()); - return returnVec; - } - iStream.ignore( std::numeric_limits::max() ); - std::streamsize size = iStream.gcount(); - ucVec.resize(size); - iStream.clear(); - iStream.seekg( 0, std::ios_base::beg ); - iStream.read((char*)(&ucVec[0]), size); - iStream.close(); - unsigned int *pUInt = reinterpret_cast(ucVec.data()); - for (size_t i = 0; i < ucVec.size()/4; i++) - returnVec.push_back(pUInt[i]); - return returnVec; -} - -bool isModeLoEncoded(CompressionMode mode) -{ - if (mode == ENCODE_LO - || mode == ENCODE_BOTH - || mode == ENCODE_BOTH_DELTA_SYMS) - return true; - return false; -} - -bool isModeSymEncoded(CompressionMode mode) -{ - if (mode == ENCODE_SYMS - || mode == ENCODE_DELTA_SYMS - || mode == ENCODE_BOTH - || mode == ENCODE_BOTH_DELTA_SYMS) - return true; - return false; -} - -bool isModeSymDelta(CompressionMode mode) -{ - if (mode == ENCODE_DELTA_SYMS - || mode == ENCODE_BOTH_DELTA_SYMS) - return true; - return false; -} - -CompressedImage fillCompressVecNew(std::vector loVec, std::vector symVec, CompressionMode mode, size_t imageBytes, std::string name) -{ - CompressedImage image; - bool loEncoded = isModeLoEncoded(mode); - bool symEncoded = isModeSymEncoded(mode); - bool symDelta = isModeSymDelta(mode); - - std::vector loNibbles(2*loVec.size()); - std::vector symNibbles(4*symVec.size()); - std::vector loDecode; - std::vector symDecode; - std::vector symbols = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; - if (loEncoded) - { - for (size_t i = 0; i < loVec.size(); i++) - { - loNibbles[2*i] = (loVec)[i] & 0xf; - loNibbles[2*i + 1] = ((loVec)[i] >> 4) & 0xf; - } - std::vector loCounts(16); - for (unsigned char uc : loNibbles) - loCounts[uc]++; - std::vector loFreqs = getNormalizedCounts(loCounts); - std::vector freqs = getFreqWriteInts(loFreqs); - image.loFreqs[0] = freqs[0]; - image.loFreqs[1] = freqs[1]; - image.loFreqs[2] = freqs[2]; - loDecode = createDecodingTable(symbols, loFreqs); - } - if (symEncoded) - { - for (size_t i = 0; i < symVec.size(); i++) - { - for (size_t j = 0; j < 4; j++) - symNibbles[4*i + j] = (symVec[i] >> (4*j)) & 0xf; - } - std::vector symCounts(16); - if (symDelta) - deltaEncode(&symNibbles, symNibbles.size()); - for (unsigned char uc : symNibbles) - symCounts[uc]++; - std::vector symFreqs = getNormalizedCounts(symCounts); - symFreqs = getTestFreqs(symFreqs, name); - std::vector freqs = getFreqWriteInts(symFreqs); - image.symFreqs[0] = freqs[0]; - image.symFreqs[1] = freqs[1]; - image.symFreqs[2] = freqs[2]; - symDecode = createDecodingTable(symbols, symFreqs); - } - int currState = -1; - std::vector bitstream; - if (symEncoded) - { - std::vector reversedVec; - for (size_t i = 0; i < symNibbles.size(); i++) - reversedVec.push_back(symNibbles[symNibbles.size() - i - 1]); - std::vector symEncode = createEncodingTable(symDecode, symbols); - size_t startIndex = 0; - if (currState == -1) - { - currState = findInitialState(symEncode[0], reversedVec[0]) - TANS_TABLE_SIZE; - startIndex = 1; - } - for (size_t i = startIndex; i < reversedVec.size(); i++) - currState = encodeSingleSymbol(symEncode[currState], reversedVec[i], &bitstream) - TANS_TABLE_SIZE; - } - if (loEncoded) - { - std::vector reversedVec; - for (size_t i = 0; i < loNibbles.size(); i++) - reversedVec.push_back(loNibbles[loNibbles.size() - i - 1]); - std::vector loEncode = createEncodingTable(loDecode, symbols); - size_t startIndex = 0; - if (currState == -1) - { - currState = findInitialState(loEncode[0], reversedVec[0]) - TANS_TABLE_SIZE; - startIndex = 1; - } - for (size_t i = startIndex; i < reversedVec.size(); i++) - currState = encodeSingleSymbol(loEncode[currState], reversedVec[i], &bitstream) - TANS_TABLE_SIZE; - } - std::vector reversedBitstream(bitstream.size()); - for (size_t i = 0; i < bitstream.size(); i++) - reversedBitstream[reversedBitstream.size() - 1 - i] = bitstream[i]; - bitstream = reversedBitstream; - std::vector checkBits = bitstream; - int checkState = currState; - - size_t currBitIndex = 0; - if (loEncoded) - { - std::vector checkLoNibbles(loNibbles.size()); - currBitIndex = decodeNibbles(loDecode, &checkBits, &checkState, &checkLoNibbles, currBitIndex, loNibbles.size()); - for (size_t i = 0; i < loNibbles.size(); i++) - if (loNibbles[i] != checkLoNibbles[i]) - { - fprintf(stderr, "LO Mismatch\n"); - break; - } - } - if (symEncoded) - { - std::vector checkSymNibbles(symNibbles.size()); - currBitIndex = decodeNibbles(symDecode, &checkBits, &checkState, &checkSymNibbles, currBitIndex, symNibbles.size()); - for (size_t i = 0; i < symNibbles.size(); i++) - if (symNibbles[i] != checkSymNibbles[i]) - { - fprintf(stderr, "Symbol Mismatch\n"); - break; - } - } - - std::vector tANSbits; - unsigned int currInt = 0; - for (size_t i = 0; i < bitstream.size(); i++) - { - currInt += bitstream[i] << (i%32); - if ((i+1) % 32 == 0) - { - tANSbits.push_back(currInt); - currInt = 0; - } - } - if (bitstream.size() % 32 != 0) - tANSbits.push_back(currInt); - image.headers = getNewHeaders(mode, imageBytes, symVec.size(), currState, tANSbits.size(), loVec.size()); - image.tANSbits = tANSbits; - image.symVec = symVec; - image.loVec = loVec; - image.initialState = currState; - return image; -} - -size_t decodeNibbles(std::vector decodeTable, std::vector *bits, int *currState, std::vector *nibbleVec, size_t currBitIndex, size_t numNibbles) -{ - for (size_t i = 0; i < numNibbles; i++) - { - (*nibbleVec)[i] = decodeTable[*currState].symbol; - int currK = decodeTable[*currState].k; - int nextState = decodeTable[*currState].y << currK; - for (size_t j = 0; j < currK; j++) - { - nextState += (*bits)[currBitIndex] << j; - currBitIndex++; - } - *currState = nextState - TANS_TABLE_SIZE; - } - return currBitIndex; -} - -std::vector readRawDataVecs(std::vector *pInput) -{ - std::vector imageVec; - CompressedImage readImage = readNewHeader(pInput); - bool loEncoded = isModeLoEncoded(readImage.mode); - bool symEncoded = isModeSymEncoded(readImage.mode); - bool symDelta = isModeSymDelta(readImage.mode); - size_t readIndex = 2; - std::vector tANSbits; - std::vector allBits; - std::vector symVec(readImage.symSize); - std::vector loVec(readImage.loSize); - std::vector symDecode(TANS_TABLE_SIZE); - std::vector loDecode(TANS_TABLE_SIZE); - int currState = readImage.initialState; - if (loEncoded) - { - for (size_t i = 0; i < 3; i++) - readImage.loFreqs[i] = (*pInput)[readIndex + i]; - readIndex += 3; - std::vector loFreqs = unpackFrequencies(&readImage.loFreqs[0]); - size_t currCol = 0; - for (size_t i = 0; i < 16; i++) - { - for (size_t j = 0; j < loFreqs[i]; j++) - { - loDecode[currCol].state = TANS_TABLE_SIZE + currCol; - loDecode[currCol].symbol = i; - loDecode[currCol].y = loFreqs[i] + j; - int currK = 0; - while ((loDecode[currCol].y << currK) < TANS_TABLE_SIZE) - currK++; - loDecode[currCol].k = currK; - currCol++; - } - } - } - if (symEncoded) - { - for (size_t i = 0; i < 3; i++) - readImage.symFreqs[i] = (*pInput)[readIndex + i]; - readIndex += 3; - std::vector symFreqs = unpackFrequencies(&readImage.symFreqs[0]); - size_t currCol = 0; - for (size_t i = 0; i < 16; i++) - { - for (size_t j = 0; j < symFreqs[i]; j++) - { - symDecode[currCol].state = TANS_TABLE_SIZE + currCol; - symDecode[currCol].symbol = i; - symDecode[currCol].y = symFreqs[i] + j; - int currK = 0; - while ((symDecode[currCol].y << currK) < TANS_TABLE_SIZE) - currK++; - symDecode[currCol].k = currK; - currCol++; - } - } - } - if (loEncoded || symEncoded) - { - tANSbits.resize(readImage.bitreamSize); - for (size_t i = 0; i < readImage.bitreamSize; i++) - tANSbits[i] = (*pInput)[readIndex + i]; - readIndex += readImage.bitreamSize; - allBits.resize(tANSbits.size()*32); - size_t currIndex = 0; - for (unsigned int ui : tANSbits) - { - for (size_t i = 0; i < 32; i++) - { - unsigned int currVal = (ui >> i) & 0x1; - allBits[currIndex] = currVal; - currIndex++; - } - } - } - bool leftOverValues = false; - if (!symEncoded) - { - for (size_t i = 0; i < readImage.symSize; i++) - { - symVec[i] = ((*pInput)[readIndex] >> (16*(i%2))) & 0xffff; - if ((i+1) % 2 == 0) - { - readIndex++; - leftOverValues = false; - } - else - { - leftOverValues = true; - } - } - } - if (!loEncoded) - { - size_t offsetMod = 0; - if (leftOverValues) - offsetMod = 2; - for (size_t i = 0; i < readImage.loSize; i++) - { - loVec[i] = ((*pInput)[readIndex] >> (8*((i%4) + offsetMod))) & 0xff; - if ((i+offsetMod+1) % 4 == 0) - readIndex++; - } - } - size_t bitIndex = 0; - if (loEncoded) - { - std::vector loNibbles(readImage.loSize*2); - bitIndex = decodeNibbles(loDecode, &allBits, &currState, &loNibbles, bitIndex, readImage.loSize*2); - for (size_t i = 0; i < readImage.loSize; i++) - loVec[i] = loNibbles[2*i] + (loNibbles[2*i + 1] << 4); - } - if (symEncoded) - { - std::vector symNibbles(readImage.symSize*4); - bitIndex = decodeNibbles(symDecode, &allBits, &currState, &symNibbles, bitIndex, readImage.symSize*4); - if (symDelta) - deltaDecode(&symNibbles, symNibbles.size()); - for (size_t i = 0; i < readImage.symSize; i++) - for (size_t j = 0; j < 4; j++) - symVec[i] += (unsigned short)symNibbles[i*4 + j] << (4*j); - } - imageVec = decodeBytesShort(&loVec, &symVec); - return imageVec; -} - -void deltaEncode(std::vector *buffer, int length) -{ - unsigned char last = 0; - for (int i = 0; i < length; i++) - { - unsigned char current =(*buffer)[i]; - (*buffer)[i] = (current-last) & 0xf; - last = current; - } -} - -void deltaDecode(std::vector *buffer, int length) -{ - unsigned char last = 0; - for (int i = 0; i < length; i++) - { - unsigned char delta =(*buffer)[i]; - (*buffer)[i] = (delta+last) & 0xf; - last = (*buffer)[i]; - } -} - std::vector getTestFreqs(std::vector freqs, std::string name) { if (name.find("test/compression/table_") == std::string::npos) @@ -1465,3 +549,783 @@ std::vector getTestFreqs(std::vector freqs, std::string name) } return freqs; } + +void deltaEncode(std::vector *buffer, int length) +{ + unsigned char last = 0; + for (int i = 0; i < length; i++) + { + unsigned char current =(*buffer)[i]; + (*buffer)[i] = (current-last) & 0xf; + last = current; + } +} + +void deltaDecode(std::vector *buffer, int length) +{ + unsigned char last = 0; + for (int i = 0; i < length; i++) + { + unsigned char delta =(*buffer)[i]; + (*buffer)[i] = (delta+last) & 0xf; + last = (*buffer)[i]; + } +} + +int findInitialState(EncodeCol *encodeCol, unsigned char firstSymbol) +{ + for (size_t i = 0; i < encodeCol->symbols.size(); i++) + { + if (encodeCol->symbols[i].symbol == firstSymbol) + return encodeCol->symbols[i].nextState; + } + return -1; +} + +size_t decodeNibbles(std::vector decodeTable, std::vector *bits, int *currState, std::vector *nibbleVec, size_t currBitIndex, size_t numNibbles, bool lastThing) +{ + for (size_t i = 0; i < numNibbles; i++) + { + (*nibbleVec)[i] = decodeTable[*currState].symbol; + + if (i + 1 == nibbleVec->size() && lastThing) + return currBitIndex; + + int currK = decodeTable[*currState].k; + int nextState = decodeTable[*currState].y << currK; + for (size_t j = 0; j < currK; j++) + { + nextState += (*bits)[currBitIndex] << j; + currBitIndex++; + } + *currState = nextState - TANS_TABLE_SIZE; + } + return currBitIndex; +} + +std::vector getNewHeaders(CompressionMode mode, size_t imageSize, size_t symLength, int initialState, size_t bitstreamSize, size_t loLength) +{ + if (initialState == -1) + initialState = 0; + std::vector returnVec(2); + + returnVec[0] += (unsigned int)mode; // 4 bits + returnVec[0] += (imageSize/IMAGE_SIZE_MODIFIER) << IMAGE_SIZE_OFFSET; // 14 bits + returnVec[0] += (symLength) << SYM_SIZE_OFFSET; // 14 bits + // 32 total + + returnVec[1] += initialState; // 6 bits + returnVec[1] += bitstreamSize << BITSTREAM_SIZE_OFFSET; // 13 bits + returnVec[1] += loLength << LO_SIZE_OFFSET; // 13 bits + // 32 total + + return returnVec; +} + +bool fillCompressVec(std::vector *pLoVec, std::vector *pSymVec, CompressionMode mode, size_t imageBytes, std::string name, CompressedImage *pOutput) +{ + bool loEncoded = isModeLoEncoded(mode); + bool symEncoded = isModeSymEncoded(mode); + bool symDelta = isModeSymDelta(mode); + + std::vector loNibbles(2 * pLoVec->size()); + std::vector symNibbles(4 * pSymVec->size()); + std::vector loDecode; + std::vector symDecode; + std::vector symbols = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + + if (loEncoded) + { + for (size_t i = 0; i < pLoVec->size(); i++) + { + loNibbles[2*i] = (*pLoVec)[i] & 0xf; + loNibbles[2*i + 1] = ((*pLoVec)[i] >> 4) & 0xf; + } + std::vector loCounts(16); + for (unsigned char uc : loNibbles) + loCounts[uc]++; + std::vector loFreqs = getNormalizedCounts(&loCounts); + std::vector freqs = getFreqWriteInts(loFreqs); + pOutput->loFreqs[0] = freqs[0]; + pOutput->loFreqs[1] = freqs[1]; + pOutput->loFreqs[2] = freqs[2]; + loDecode = createDecodingTable(symbols, loFreqs); + } + + if (symEncoded) + { + for (size_t i = 0; i < pSymVec->size(); i++) + { + for (size_t j = 0; j < 4; j++) + symNibbles[4*i + j] = ((*pSymVec)[i] >> (4*j)) & 0xf; + } + std::vector symCounts(16); + if (symDelta) + deltaEncode(&symNibbles, symNibbles.size()); + for (unsigned char uc : symNibbles) + symCounts[uc]++; + std::vector symFreqs = getNormalizedCounts(&symCounts); + symFreqs = getTestFreqs(symFreqs, name); + std::vector freqs = getFreqWriteInts(symFreqs); + pOutput->symFreqs[0] = freqs[0]; + pOutput->symFreqs[1] = freqs[1]; + pOutput->symFreqs[2] = freqs[2]; + symDecode = createDecodingTable(symbols, symFreqs); + } + + int currState = -1; + std::vector bitstream; + if (symEncoded) + { + std::vector reversedVec; + for (size_t i = 0; i < symNibbles.size(); i++) + reversedVec.push_back(symNibbles[symNibbles.size() - i - 1]); + std::vector symEncode = createEncodingTable(symDecode, symbols); + size_t startIndex = 0; + if (currState == -1) + { + currState = findInitialState(&symEncode[0], reversedVec[0]) - TANS_TABLE_SIZE; + startIndex = 1; + } + for (size_t i = startIndex; i < reversedVec.size(); i++) + currState = encodeSingleSymbol(symEncode[currState], reversedVec[i], &bitstream) - TANS_TABLE_SIZE; + } + + if (loEncoded) + { + std::vector reversedVec; + for (size_t i = 0; i < loNibbles.size(); i++) + reversedVec.push_back(loNibbles[loNibbles.size() - i - 1]); + std::vector loEncode = createEncodingTable(loDecode, symbols); + size_t startIndex = 0; + if (currState == -1) + { + currState = findInitialState(&loEncode[0], reversedVec[0]) - TANS_TABLE_SIZE; + startIndex = 1; + } + for (size_t i = startIndex; i < reversedVec.size(); i++) + currState = encodeSingleSymbol(loEncode[currState], reversedVec[i], &bitstream) - TANS_TABLE_SIZE; + } + + std::vector reversedBitstream(bitstream.size()); + for (size_t i = 0; i < bitstream.size(); i++) + reversedBitstream[reversedBitstream.size() - 1 - i] = bitstream[i]; + bitstream = reversedBitstream; + std::vector checkBits = bitstream; + int checkState = currState; + size_t currBitIndex = 0; + + if (loEncoded) + { + bool lastThing = !symEncoded; + std::vector checkLoNibbles(loNibbles.size()); + currBitIndex = decodeNibbles(loDecode, &checkBits, &checkState, &checkLoNibbles, currBitIndex, loNibbles.size(), lastThing); + for (size_t i = 0; i < loNibbles.size(); i++) + { + if (loNibbles[i] != checkLoNibbles[i]) + { + fprintf(stderr, "LO Mismatch\n"); + break; + } + } + } + + if (symEncoded) + { + std::vector checkSymNibbles(symNibbles.size()); + currBitIndex = decodeNibbles(symDecode, &checkBits, &checkState, &checkSymNibbles, currBitIndex, symNibbles.size(), true); + for (size_t i = 0; i < symNibbles.size(); i++) + { + if (symNibbles[i] != checkSymNibbles[i]) + { + fprintf(stderr, "Symbol Mismatch\n"); + break; + } + } + } + + std::vector tANSbits; + unsigned int currInt = 0; + for (size_t i = 0; i < bitstream.size(); i++) + { + currInt += bitstream[i] << (i%32); + if ((i+1) % 32 == 0) + { + tANSbits.push_back(currInt); + currInt = 0; + } + } + if (bitstream.size() % 32 != 0) + tANSbits.push_back(currInt); + + pOutput->headers = getNewHeaders(mode, imageBytes, pSymVec->size(), currState, tANSbits.size(), pLoVec->size()); + pOutput->tANSbits = tANSbits; + pOutput->symVec = *pSymVec; + pOutput->loVec = *pLoVec; + pOutput->initialState = currState; + return true; +} + +bool verifyCompressionShort(CompressedImage *pInput, std::vector *pImage) +{ + std::vector decodedImage = decodeImageShort(pInput); + return compareVectorsShort(&decodedImage, pImage); +} + +std::vector decodeImageShort(CompressedImage *pInput) +{ + DataVecs dataVecs = decodeDataVectorsNew(pInput); + return decodeBytesShort(&dataVecs.loVec, &dataVecs.symVec); +} + +std::vector decodeBytesShort(std::vector *pLoVec, std::vector *pSymVec) +{ + std::vector decodedImage; + size_t loIndex = 0; + size_t symIndex = 0; + while (loIndex < pLoVec->size()) + { + size_t currLength = 0; + size_t currOffset = 0; + currLength += (*pLoVec)[loIndex] & LO_LOW_BITS_MASK; + loIndex++; + if (((*pLoVec)[loIndex-1] & LO_CONTINUE_BIT) == LO_CONTINUE_BIT) + { + currLength += (*pLoVec)[loIndex] << LO_NUM_LOW_BITS; + loIndex++; + } + currOffset += (*pLoVec)[loIndex] & LO_LOW_BITS_MASK; + loIndex++; + if (((*pLoVec)[loIndex-1] & LO_CONTINUE_BIT) == LO_CONTINUE_BIT) + { + currOffset += (*pLoVec)[loIndex] << LO_NUM_LOW_BITS; + loIndex++; + } + if (currLength != 0) + { + decodedImage.push_back((*pSymVec)[symIndex]); + symIndex++; + for (size_t i = 0; i < currLength; i++) + decodedImage.push_back(decodedImage[decodedImage.size() - currOffset]); + } + else + { + for (size_t i = 0; i < currOffset; i++) + { + decodedImage.push_back((*pSymVec)[symIndex]); + symIndex++; + } + } + } + return decodedImage; +} + +DataVecs decodeDataVectorsNew(CompressedImage *pInput) +{ + CompressedImage headerValues; + readNewHeader(&pInput->headers, &headerValues); + size_t loSize = headerValues.loSize; + size_t symSize = headerValues.symSize; + CompressionMode mode = headerValues.mode; + bool loEncoded = isModeLoEncoded(mode); + bool symEncoded = isModeSymEncoded(mode); + bool symDelta = isModeSymDelta(mode); + std::vector symbols = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + std::vector loDecode; + std::vector symDecode; + + std::vector loVec(loSize); + std::vector symVec(symSize); + std::vector loNibbles(loSize*2); + std::vector symNibbles(symSize*4); + + if (loEncoded) + { + std::vector loFreqs = unpackFrequencies(&pInput->loFreqs[0]); + loDecode = createDecodingTable(symbols, loFreqs); + } + if (symEncoded) + { + std::vector symFreqs = unpackFrequencies(&pInput->symFreqs[0]); + symDecode = createDecodingTable(symbols, symFreqs); + } + + std::vector allBits(pInput->tANSbits.size()*32); + size_t currIndex = 0; + for (unsigned int ui : pInput->tANSbits) + for (size_t i = 0; i < 32; i++) + { + unsigned int currVal = (ui >> i) & 0x1; + allBits[currIndex] = currVal; + currIndex++; + } + + if (!symEncoded) + for (size_t i = 0; i < symSize; i++) + symVec[i] = pInput->symVec[i]; + if (!loEncoded) + for (size_t i = 0; i < loSize; i++) + loVec[i] = pInput->loVec[i]; + + size_t bitIndex = 0; + int currState = pInput->initialState; + if (loEncoded) + { + bitIndex = decodeNibbles(loDecode, &allBits, &currState, &loNibbles, bitIndex, loSize*2, !symEncoded); + } + if (loEncoded) + for (size_t i = 0; i < loVec.size(); i++) + loVec[i] = loNibbles[2*i] + (loNibbles[2*i + 1] << 4); + + if (symEncoded) + { + bitIndex = decodeNibbles(symDecode, &allBits, &currState, &symNibbles, bitIndex, symSize*4, true); + if (symDelta) + deltaDecode(&symNibbles, symNibbles.size()); + } + if (symEncoded) + for (size_t i = 0; i < symVec.size(); i++) + for (size_t j = 0; j < 4; j++) + symVec[i] += (unsigned short)symNibbles[i*4 + j] << (j*4); + + DataVecs returnData; + returnData.loVec = loVec; + returnData.symVec = symVec; + return returnData; +} + +void readNewHeader(std::vector *pInput, CompressedImage *pOutput) +{ + std::vector headers(2); + headers[0] = (*pInput)[0]; + headers[1] = (*pInput)[1]; + pOutput->mode = (CompressionMode)(headers[0] & MODE_MASK); + pOutput->rawNumBytes = ((headers[0] >> IMAGE_SIZE_OFFSET) & IMAGE_SIZE_MASK) * IMAGE_SIZE_MODIFIER; + pOutput->symSize = ((headers[0] >> SYM_SIZE_OFFSET) & SYM_SIZE_MASK); + + pOutput->initialState = headers[1] & INITIAL_STATE_MASK; + pOutput->bitreamSize = (headers[1] >> BITSTREAM_SIZE_OFFSET) & BITSTREAM_SIZE_MASK; + pOutput->loSize = (headers[1] >> LO_SIZE_OFFSET) & LO_SIZE_MASK; + pOutput->headers = headers; +} + +std::vector unpackFrequencies(unsigned int pInts[3]) +{ + std::vector returnVec; + int freq15 = 0; + for (size_t i = 0; i < 3; i++) + { + for (size_t j = 0; j < 5; j++) + { + int currVal = (pInts[i] >> (6*j)) & 0x3f; + returnVec.push_back(currVal); + } + freq15 += ((pInts[i] >> 30) & 0x3) << (2*i); + } + returnVec.push_back(freq15); + return returnVec; +} + +void getUIntVecFromData(CompressedImage *pImage, std::vector *pOutput) +{ + CompressedImage otherImage; + readNewHeader(&pImage->headers, &otherImage); + pOutput->push_back(pImage->headers[0]); + pOutput->push_back(pImage->headers[1]); + if (isModeLoEncoded(otherImage.mode)) + for (size_t i = 0; i < 3; i++) + pOutput->push_back(pImage->loFreqs[i]); + if (isModeSymEncoded(otherImage.mode)) + for (size_t i = 0; i < 3; i++) + pOutput->push_back(pImage->symFreqs[i]); + if (isModeLoEncoded(otherImage.mode) || isModeSymEncoded(otherImage.mode)) + for (unsigned int ui : pImage->tANSbits) + pOutput->push_back(ui); + unsigned int currInt = 0; + unsigned int currOffset = 0; + if (!isModeSymEncoded(otherImage.mode)) + { + for (size_t i = 0; i < pImage->symVec.size(); i++) + { + currInt += pImage->symVec[i] << (8*(currOffset % 4)); + currOffset += 2; + if (currOffset % 4 == 0) + { + pOutput->push_back(currInt); + currInt = 0; + } + } + } + if (!isModeLoEncoded(otherImage.mode)) + { + for (size_t i = 0; i < pImage->loVec.size(); i++) + { + currInt += 0; + currInt += pImage->loVec[i] << (8*(currOffset % 4)); + currOffset++; + if (currOffset % 4 == 0) + { + pOutput->push_back(currInt); + currInt = 0; + } + } + } + if (currOffset != 0) + pOutput->push_back(currInt); +} + +void readRawDataVecs(std::vector *pInput, std::vector *pOutput) +{ + CompressedImage readImage; + readNewHeader(pInput, &readImage); + bool loEncoded = isModeLoEncoded(readImage.mode); + bool symEncoded = isModeSymEncoded(readImage.mode); + bool symDelta = isModeSymDelta(readImage.mode); + size_t readIndex = 2; + std::vector tANSbits; + std::vector allBits; + std::vector symVec(readImage.symSize); + std::vector loVec(readImage.loSize); + std::vector symDecode(TANS_TABLE_SIZE); + std::vector loDecode(TANS_TABLE_SIZE); + int currState = readImage.initialState; + if (loEncoded) + { + for (size_t i = 0; i < 3; i++) + readImage.loFreqs[i] = (*pInput)[readIndex + i]; + readIndex += 3; + std::vector loFreqs = unpackFrequencies(&readImage.loFreqs[0]); + size_t currCol = 0; + for (size_t i = 0; i < 16; i++) + { + for (size_t j = 0; j < loFreqs[i]; j++) + { + loDecode[currCol].state = TANS_TABLE_SIZE + currCol; + loDecode[currCol].symbol = i; + loDecode[currCol].y = loFreqs[i] + j; + int currK = 0; + while ((loDecode[currCol].y << currK) < TANS_TABLE_SIZE) + currK++; + loDecode[currCol].k = currK; + currCol++; + } + } + } + if (symEncoded) + { + for (size_t i = 0; i < 3; i++) + readImage.symFreqs[i] = (*pInput)[readIndex + i]; + readIndex += 3; + std::vector symFreqs = unpackFrequencies(&readImage.symFreqs[0]); + size_t currCol = 0; + for (size_t i = 0; i < 16; i++) + { + for (size_t j = 0; j < symFreqs[i]; j++) + { + symDecode[currCol].state = TANS_TABLE_SIZE + currCol; + symDecode[currCol].symbol = i; + symDecode[currCol].y = symFreqs[i] + j; + int currK = 0; + while ((symDecode[currCol].y << currK) < TANS_TABLE_SIZE) + currK++; + symDecode[currCol].k = currK; + currCol++; + } + } + } + if (loEncoded || symEncoded) + { + tANSbits.resize(readImage.bitreamSize); + for (size_t i = 0; i < readImage.bitreamSize; i++) + tANSbits[i] = (*pInput)[readIndex + i]; + readIndex += readImage.bitreamSize; + allBits.resize(tANSbits.size()*32); + size_t currIndex = 0; + for (unsigned int ui : tANSbits) + { + for (size_t i = 0; i < 32; i++) + { + unsigned int currVal = (ui >> i) & 0x1; + allBits[currIndex] = currVal; + currIndex++; + } + } + } + bool leftOverValues = false; + if (!symEncoded) + { + for (size_t i = 0; i < readImage.symSize; i++) + { + symVec[i] = ((*pInput)[readIndex] >> (16*(i%2))) & 0xffff; + if ((i+1) % 2 == 0) + { + readIndex++; + leftOverValues = false; + } + else + { + leftOverValues = true; + } + } + } + if (!loEncoded) + { + size_t offsetMod = 0; + if (leftOverValues) + offsetMod = 2; + for (size_t i = 0; i < readImage.loSize; i++) + { + loVec[i] = ((*pInput)[readIndex] >> (8*((i%4) + offsetMod))) & 0xff; + if ((i+offsetMod+1) % 4 == 0) + readIndex++; + } + } + size_t bitIndex = 0; + if (loEncoded) + { + std::vector loNibbles(readImage.loSize*2); + bitIndex = decodeNibbles(loDecode, &allBits, &currState, &loNibbles, bitIndex, readImage.loSize*2, !symEncoded); + for (size_t i = 0; i < readImage.loSize; i++) + loVec[i] = loNibbles[2*i] + (loNibbles[2*i + 1] << 4); + } + if (symEncoded) + { + std::vector symNibbles(readImage.symSize*4); + bitIndex = decodeNibbles(symDecode, &allBits, &currState, &symNibbles, bitIndex, readImage.symSize*4, true); + if (symDelta) + deltaDecode(&symNibbles, symNibbles.size()); + for (size_t i = 0; i < readImage.symSize; i++) + for (size_t j = 0; j < 4; j++) + symVec[i] += (unsigned short)symNibbles[i*4 + j] << (4*j); + } + *pOutput = decodeBytesShort(&loVec, &symVec); +} + +InputSettings::InputSettings() {} + +InputSettings::InputSettings(bool canEncodeLO, bool canEncodeSyms, bool canDeltaSyms) +{ + this->canEncodeLO = canEncodeLO; + this->canEncodeSyms = canEncodeSyms; + this->canDeltaSyms = canDeltaSyms; +} + +bool readFileAsUC(std::string filePath, std::vector *pFileData) +{ + std::ifstream iStream; + iStream.open(filePath.c_str(), std::ios::binary); + if (!iStream.is_open()) + { + fprintf(stderr, "Error: Couldn't open %s for reading bytes\n", filePath.c_str()); + return false; + } + + iStream.ignore( std::numeric_limits::max() ); + std::streamsize size = iStream.gcount(); + iStream.clear(); + iStream.seekg( 0, std::ios_base::beg ); + pFileData->resize(size); + iStream.read((char *)(pFileData->data()), size); + iStream.close(); + return true; +} + +bool readFileAsUInt(std::string filePath, std::vector *pFileData) +{ + std::ifstream iStream; + iStream.open(filePath.c_str(), std::ios::binary); + if (!iStream.is_open()) + { + fprintf(stderr, "Error: Couldn't open %s for reading bytes\n", filePath.c_str()); + return false; + } + + iStream.ignore( std::numeric_limits::max() ); + std::streamsize size = iStream.gcount(); + iStream.clear(); + iStream.seekg( 0, std::ios_base::beg ); + pFileData->resize(size/4); + iStream.read((char *)(pFileData->data()), size); + iStream.close(); + return true; +} + +CompressedImage processImage(std::string fileName, InputSettings settings) +{ + CompressedImage image; + std::vector input; + if (!readFileAsUC(fileName, &input)) + { + fprintf(stderr, "Compression failure\n"); + return image; + } + if (!processImageData(&input, &image, settings, fileName)) + { + fprintf(stderr, "Fail\n"); + } + return image; +} + +// Not implemented yet +CompressedImage processImageFrames(std::string fileName, InputSettings settings) +{ + CompressedImage image; + return image; +} + +bool processImageData(std::vector *pInput, CompressedImage *pImage, InputSettings settings, std::string fileName) +{ + CompressionMode someMode; + bool hasImage = false; + bool byteFail = false; + bool copyFail = false; + bool compressionFail = false; + bool uIntConversionFail = false; + + std::vector usBase(pInput->size() / 2); + memcpy(usBase.data(), pInput->data(), pInput->size()); + + std::vector bestLO; + std::vector bestSym; + std::vector bestInstructions; + + for (size_t minCodeLength = 2; minCodeLength <= 15; minCodeLength++) + { + std::vector shortCopies; + if (!getShortCopies(&usBase, minCodeLength, &shortCopies)) + { + copyFail = true; + printf("ERROR: %zu\n", minCodeLength); + continue; + } + + std::vector shortInstructions; + if (!getShortInstructions(&shortCopies, &shortInstructions, &usBase)) + { + printf("ERROR\n"); + return false; + } + + std::vector loVec; + std::vector symVec; + getLosFromInstructions(&shortInstructions, &loVec); + getSymsFromInstructions(&shortInstructions, &symVec); + if (!verifyBytesShort(&loVec, &symVec, &usBase)) + { + byteFail = true; + printf("Byte veri\n"); + continue; + } + + CompressionMode mode = BASE_ONLY; + std::vector modesToUse = {BASE_ONLY, ENCODE_SYMS, ENCODE_DELTA_SYMS, ENCODE_LO, ENCODE_BOTH, ENCODE_BOTH_DELTA_SYMS}; + if (fileName.find("test/compression/") != std::string::npos) + { + if (fileName.find("mode_0.4bpp") != std::string::npos) + modesToUse = {BASE_ONLY}; + else if (fileName.find("mode_1.4bpp") != std::string::npos) + modesToUse = {ENCODE_SYMS}; + else if (fileName.find("mode_2.4bpp") != std::string::npos) + modesToUse = {ENCODE_DELTA_SYMS}; + else if (fileName.find("mode_3.4bpp") != std::string::npos) + modesToUse = {ENCODE_LO}; + else if (fileName.find("mode_4.4bpp") != std::string::npos) + modesToUse = {ENCODE_BOTH}; + else if (fileName.find("mode_5.4bpp") != std::string::npos) + modesToUse = {ENCODE_BOTH_DELTA_SYMS}; + else if (fileName.find("test/compression/table_") != std::string::npos) + modesToUse = {ENCODE_SYMS}; + + if (modesToUse.size() == 1) + { + settings.canDeltaSyms = true; + settings.canEncodeLO = true; + settings.canEncodeSyms = true; + } + } + for (CompressionMode currMode : modesToUse) + { + CompressedImage currImg; + mode = currMode; + if (!settings.canDeltaSyms + && (mode == ENCODE_DELTA_SYMS + || mode == ENCODE_BOTH_DELTA_SYMS)) + continue; + if (!settings.canEncodeLO + && (mode == ENCODE_LO + || mode == ENCODE_BOTH + || mode == ENCODE_BOTH_DELTA_SYMS)) + continue; + if (!settings.canEncodeSyms + && (mode == ENCODE_SYMS + || mode == ENCODE_BOTH + || mode == ENCODE_DELTA_SYMS + || mode == ENCODE_BOTH_DELTA_SYMS)) + continue; + if (!fillCompressVec(&loVec, &symVec, mode, pInput->size(), fileName, &currImg)) + { + printf("ERROR\n"); + } + + if (!verifyCompressionShort(&currImg, &usBase)) + { + compressionFail = true; + printf("ERROR\n"); + continue; + } + std::vector uiVec; + getUIntVecFromData(&currImg, &uiVec); + std::vector decodedImage; + readRawDataVecs(&uiVec, &decodedImage); + if (!compareVectorsShort(&decodedImage, &usBase)) + { + uIntConversionFail = true; + printf("ERROR\n"); + continue; + } + currImg.compressedSize = uiVec.size() * 4; + if (!hasImage) + { + bestLO = loVec; + bestSym = symVec; + bestInstructions = shortInstructions; + *pImage = currImg; + hasImage = true; + pImage->writeVec = uiVec; + someMode = mode; + } + else if (currImg.compressedSize < pImage->compressedSize) + { + bestLO = loVec; + bestSym = symVec; + bestInstructions = shortInstructions; + *pImage = currImg; + hasImage = true; + pImage->writeVec = uiVec; + someMode = mode; + } + } + } + pImage->mode = someMode; + pImage->fileName = fileName; + pImage->rawNumBytes = pInput->size(); + if (hasImage) + { + pImage->isValid = true; + } + else + { + fprintf(stderr, "Failed to compress image %s\nErrors: ", fileName.c_str()); + if (copyFail) + fprintf(stderr, "CopyProcessing "); + if (byteFail) + fprintf(stderr, "ByteConversion "); + if (compressionFail) + fprintf(stderr, "Compression "); + if (uIntConversionFail) + fprintf(stderr, "uIntConversion "); + printf("\n"); + } + + return true; +} diff --git a/tools/compresSmol/compressAlgo.h b/tools/compresSmol/compressAlgo.h index e27c599cef..6c2ff9373e 100644 --- a/tools/compresSmol/compressAlgo.h +++ b/tools/compresSmol/compressAlgo.h @@ -54,20 +54,7 @@ struct ShortCopy { size_t length; size_t offset; unsigned short firstSymbol; - std::vector usSequence; - ShortCopy(); - ShortCopy(size_t index, size_t length, size_t offset, std::vector usSequence); -}; - -struct CompressionInstruction { - size_t length; - size_t offset; - size_t index; - unsigned char firstSymbol; - std::vector symbols; - std::vector bytes; - void buildBytes(); - bool verifyInstruction(); + ShortCopy(size_t index, size_t length, size_t offset, unsigned short firstSymbol); }; struct ShortCompressionInstruction { @@ -75,20 +62,9 @@ struct ShortCompressionInstruction { size_t offset; size_t index; unsigned short firstSymbol; - std::vector symbols; std::vector loBytes; std::vector symShorts; - void buildBytes(); - bool verifyInstruction(); -}; - -struct SortedShortElement { - size_t index; - ShortCopy copy; - bool isRun = false; - bool isCopy = false; - SortedShortElement(); - SortedShortElement(size_t index, ShortCopy copy); + void buildBytes(std::vector *pInput); }; struct CompressedImage { @@ -116,7 +92,6 @@ struct InputSettings { bool canEncodeLO = true; bool canEncodeSyms = true; bool canDeltaSyms = true; - bool shouldCompare = false; bool useFrames = false; InputSettings(); InputSettings(bool canEncodeLO, bool canEncodeSyms, bool canDeltaSyms); @@ -127,45 +102,38 @@ struct DataVecs { std::vector symVec; }; -void analyzeImages(std::vector *allImages, std::mutex *imageMutex, FileDispatcher *dispatcher, std::mutex *dispatchMutex, InputSettings settings); - CompressedImage processImage(std::string fileName, InputSettings settings); CompressedImage processImageFrames(std::string fileName, InputSettings settings); -CompressedImage processImageData(std::vector input, InputSettings settings, std::string fileName); +bool processImageData(std::vector *pInput, CompressedImage *pImage, InputSettings settings, std::string fileName); -std::vector readFileAsUInt(std::string filePath); +bool readFileAsUInt(std::string filePath, std::vector *pFileData); -size_t getCompressedSize(CompressedImage *pImage); - -std::vector getShortCopies(std::vector input, size_t minLength); +bool getShortCopies(std::vector *pInput, size_t minLength, std::vector *pShortCopies); bool verifyShortCopies(std::vector *pCopies, std::vector *pImage); std::vector getNormalizedCounts(std::vector input); std::vector getFreqWriteInts(std::vector input); std::vector getNewHeaders(CompressionMode mode, size_t imageSize, size_t symLength, int initialState, size_t bitstreamSize, size_t loLength); -int findInitialState(EncodeCol encodeCol, unsigned char firstSymbol); -CompressedImage fillCompressVecNew(std::vector loVec, std::vector symVec, CompressionMode mode, size_t imageBytes, std::string name); -std::vector getShortInstructions(std::vector copies, size_t lengthMod); -std::vector getLosFromInstructions(std::vector instructions); -std::vector getSymsFromInstructions(std::vector instructions); +int findInitialState(EncodeCol *encodeCol, unsigned char firstSymbol); + +bool fillCompressVec(std::vector *pLoVec, std::vector *pSymVec, CompressionMode mode, size_t imageBytes, std::string name, CompressedImage *pOutput); + +bool getShortInstructions(std::vector *pCopies, std::vector *pInstructions, std::vector *pInput); +void getLosFromInstructions(std::vector *pInstructions, std::vector *pOutput); +void getSymsFromInstructions(std::vector *pInstructions, std::vector *pOutput); std::vector unpackFrequencies(unsigned int pInts[3]); -CompressedImage getDataFromUIntVec(std::vector *pInput); -CompressedImage readNewHeader(std::vector *pInput); -std::vector getUIntVecFromData(CompressedImage *pImage); +void readNewHeader(std::vector *pInput, CompressedImage *pOutput); +void getUIntVecFromData(CompressedImage *pImage, std::vector *pOutput); std::vector decodeBytesShort(std::vector *pLoVec, std::vector *pSymVec); std::vector decodeImageShort(CompressedImage *pInput); DataVecs decodeDataVectorsNew(CompressedImage *pInput); - -size_t decodeNibbles(std::vector decodeTable, std::vector *bits, int *currState, std::vector *nibbleVec, size_t currBitIndex, size_t numNibbles); - bool compareVectorsShort(std::vector *pVec1, std::vector *pVec2); bool verifyCompressionShort(CompressedImage *pInput, std::vector *pImage); -bool verifyBytesShort(std::vector *pLoVec, std::vector *pSymVec, std::vector *pImage); -bool verifyUIntVecShort(std::vector *pInput, std::vector *pImage); -std::vector readRawDataVecs(std::vector *pInput); +bool verifyBytesShort(std::vector *pLoVec, std::vector *pSymVec, std::vector *pImage); +void readRawDataVecs(std::vector *pInput, std::vector *pOutput); bool isModeLoEncoded(CompressionMode mode); bool isModeSymEncoded(CompressionMode mode); @@ -175,4 +143,5 @@ void deltaEncode(std::vector *buffer, int length); void deltaDecode(std::vector *buffer, int length); std::vector getTestFreqs(std::vector freqs, std::string name); + #endif diff --git a/tools/compresSmol/compressSmolTiles.cpp b/tools/compresSmol/compressSmolTiles.cpp index f8c14e7b38..b4a7ff0fcf 100644 --- a/tools/compresSmol/compressSmolTiles.cpp +++ b/tools/compresSmol/compressSmolTiles.cpp @@ -91,15 +91,19 @@ std::vector decompressVector(std::vector *pVec) CompressVectors compressVector(std::vector *pVec) { CompressVectors vecs; - std::vector shortCopies = getShortCopies(*pVec, 2); + std::vector shortCopies; + getShortCopies(pVec, 2, &shortCopies); if (!verifyShortCopies(&shortCopies, pVec)) { fprintf(stderr, "Error getting tile-number compression\n"); return vecs; } - std::vector shortInstructions = getShortInstructions(shortCopies, 0); - std::vector loVec = getLosFromInstructions(shortInstructions); - std::vector symVec = getSymsFromInstructions(shortInstructions); + std::vector shortInstructions; + getShortInstructions(&shortCopies, &shortInstructions, pVec); + std::vector loVec; + getLosFromInstructions(&shortInstructions, &loVec); + std::vector symVec; + getSymsFromInstructions(&shortInstructions, &symVec); if (!verifyBytesShort(&loVec, &symVec, pVec)) { diff --git a/tools/compresSmol/mainTiles.cpp b/tools/compresSmol/mainTiles.cpp index 1fa083446a..29626ceae3 100644 --- a/tools/compresSmol/mainTiles.cpp +++ b/tools/compresSmol/mainTiles.cpp @@ -19,6 +19,7 @@ int main(int argc, char *argv[]) fileOut.close(); return 0; } + /* else if (argc == 2) { std::filesystem::path filePath = argv[1]; @@ -66,6 +67,7 @@ int main(int argc, char *argv[]) printf("New size: %zu\n", totalSize); return 0; } + */ else { return 0; diff --git a/tools/compresSmol/tANS.cpp b/tools/compresSmol/tANS.cpp index 40abd8a658..36d84cabab 100644 --- a/tools/compresSmol/tANS.cpp +++ b/tools/compresSmol/tANS.cpp @@ -1,4 +1,5 @@ #include "tANS.h" +#include std::vector createDecodingTable(std::vector symbols, std::vector frequencies) { @@ -8,6 +9,7 @@ std::vector createDecodingTable(std::vector symbols, s { for (size_t j = 0; j < frequencies[i]; j++) { + //printf("%zu %zu\n", table.size(), currCol); table[currCol].state = TANS_TABLE_SIZE + currCol; table[currCol].symbol = symbols[i]; table[currCol].y = frequencies[i] + j;