/** * Copyright (c) 2017-present, Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #include #include #include #include #include #include #include /// This is an example demonstrating how to use auto-generated bundles and /// create standalone executables that can perform neural network computations. /// This example loads and runs the compiled lenet_mnist network model. #define DEFAULT_HEIGHT 28 #define DEFAULT_WIDTH 28 //===----------------------------------------------------------------------===// // Image processing helpers //===----------------------------------------------------------------------===// std::vector inputImageFilenames; /// \returns the index of the element at x,y,z,w. size_t getXYZW(const size_t *dims, size_t x, size_t y, size_t z, size_t w) { return (x * dims[1] * dims[2] * dims[3]) + (y * dims[2] * dims[3]) + (z * dims[3]) + w; } /// \returns the index of the element at x,y,z. size_t getXYZ(const size_t *dims, size_t x, size_t y, size_t z) { return (x * dims[1] * dims[2]) + (y * dims[2]) + z; } /// Reads a PNG image from a file into a newly allocated memory block \p imageT /// representing a WxHxNxC tensor and returns it. The client is responsible for /// freeing the memory block. bool readPngImage(const char *filename, std::pair range, float *&imageT, size_t *imageDims) { unsigned char header[8]; // open file and test for it being a png. FILE *fp = fopen(filename, "rb"); // Can't open the file. if (!fp) { return true; } // Validate signature. size_t fread_ret = fread(header, 1, 8, fp); if (fread_ret != 8) { return true; } if (png_sig_cmp(header, 0, 8)) { return true; } // Initialize stuff. png_structp png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, nullptr, nullptr, nullptr); if (!png_ptr) { return true; } png_infop info_ptr = png_create_info_struct(png_ptr); if (!info_ptr) { return true; } if (setjmp(png_jmpbuf(png_ptr))) { return true; } png_init_io(png_ptr, fp); png_set_sig_bytes(png_ptr, 8); png_read_info(png_ptr, info_ptr); size_t width = png_get_image_width(png_ptr, info_ptr); size_t height = png_get_image_height(png_ptr, info_ptr); int color_type = png_get_color_type(png_ptr, info_ptr); int bit_depth = png_get_bit_depth(png_ptr, info_ptr); const bool isGray = color_type == PNG_COLOR_TYPE_GRAY; const size_t numChannels = 1; (void)bit_depth; assert(bit_depth == 8 && "Invalid image"); assert(isGray && "Invalid image"); (void)isGray; bool hasAlpha = (color_type == PNG_COLOR_TYPE_RGB_ALPHA); int number_of_passes = png_set_interlace_handling(png_ptr); (void)number_of_passes; assert(number_of_passes == 1 && "Invalid image"); png_read_update_info(png_ptr, info_ptr); // Error during image read. if (setjmp(png_jmpbuf(png_ptr))) { return true; } auto *row_pointers = (png_bytep *)malloc(sizeof(png_bytep) * height); for (size_t y = 0; y < height; y++) { row_pointers[y] = (png_byte *)malloc(png_get_rowbytes(png_ptr, info_ptr)); } png_read_image(png_ptr, row_pointers); png_read_end(png_ptr, info_ptr); imageDims[0] = width; imageDims[1] = height; imageDims[2] = numChannels; imageT = static_cast( calloc(1, width * height * numChannels * sizeof(float))); float scale = ((range.second - range.first) / 255.0); float bias = range.first; for (size_t row_n = 0; row_n < height; row_n++) { png_byte *row = row_pointers[row_n]; for (size_t col_n = 0; col_n < width; col_n++) { png_byte *ptr = &(row[col_n * (hasAlpha ? (numChannels + 1) : numChannels)]); imageT[getXYZ(imageDims, row_n, col_n, 0)] = float(ptr[0]) * scale + bias; } } for (size_t y = 0; y < height; y++) { free(row_pointers[y]); } free(row_pointers); png_destroy_read_struct(&png_ptr, &info_ptr, (png_infopp)NULL); fclose(fp); printf("Loaded image: %s\n", filename); return false; } /// Loads and normalizes all PNGs into a tensor memory block \p resultT in the /// NCHW 1x28x28 format. static void loadImagesAndPreprocess(const std::vector &filenames, float *&resultT, size_t *resultDims) { assert(filenames.size() > 0 && "There must be at least one filename in filenames"); std::pair range = std::make_pair(0., 1.0); unsigned numImages = filenames.size(); // N x C x H x W resultDims[0] = numImages; resultDims[1] = 1; resultDims[2] = DEFAULT_HEIGHT; resultDims[3] = DEFAULT_WIDTH; size_t resultSizeInBytes = numImages * DEFAULT_HEIGHT * DEFAULT_WIDTH * sizeof(float); resultT = static_cast(malloc(resultSizeInBytes)); // We iterate over all the png files, reading them all into our result tensor // for processing for (unsigned n = 0; n < numImages; n++) { float *imageT{nullptr}; size_t dims[3]; bool loadSuccess = !readPngImage(filenames[n].c_str(), range, imageT, dims); assert(loadSuccess && "Error reading input image."); (void)loadSuccess; assert((dims[0] == DEFAULT_HEIGHT && dims[1] == DEFAULT_WIDTH) && "All images must have the same Height and Width"); // Convert to BGR, as this is what NN is expecting. for (unsigned y = 0; y < dims[1]; y++) { for (unsigned x = 0; x < dims[0]; x++) { resultT[getXYZW(resultDims, n, 0, x, y)] = imageT[getXYZ(dims, x, y, 0)]; } } } printf("Loaded images size in bytes is: %lu\n", resultSizeInBytes); } /// Parse images file names into a vector. void parseCommandLineOptions(int argc, char **argv) { int arg = 1; while (arg < argc) { inputImageFilenames.push_back(argv[arg++]); } } //===----------------------------------------------------------------------===// // Wrapper code for executing a bundle //===----------------------------------------------------------------------===// /// Type describing a symbol table entry of a generated bundle. struct SymbolTableEntry { // Name of a variable. const char *name; // Offset of the variable inside the memory area. uint64_t offset; // The number of elements inside this variable. uint64_t size; // The kind of the variable. 1 if it is a mutable variable, 0 otherwise. char kind; }; /// Type describing the config of a generated bundle. struct BundleConfig { // Size of the constant weight variables memory area. uint64_t constantWeightVarsMemSize; // Size of the mutable weight variables memory area. uint64_t mutableWeightVarsMemSize; // Size of the activations memory area. uint64_t activationsMemSize; // Alignment to be used for weights and activations. uint64_t alignment; // Number of symbols in the symbol table. uint64_t numSymbols; // Symbol table. const SymbolTableEntry *symbolTable; }; // These two external symbols are auto-generated by means of the -bundle option. extern "C" void lenet_mnist(uint8_t *constantWeightVars, uint8_t *mutableWeightVars, uint8_t *activations); extern "C" BundleConfig lenet_mnist_config; /// Find in the bundle's symbol table a weight variable whose name starts with /// \p name. const SymbolTableEntry *getWeightVar(const BundleConfig &config, const char *name) { for (unsigned i = 0, e = config.numSymbols; i < e; ++i) { if (!strncmp(config.symbolTable[i].name, name, strlen(name))) { return &config.symbolTable[i]; } } return nullptr; } /// Find in the bundle's symbol table a mutable weight variable whose name /// starts with \p name. const SymbolTableEntry &getMutableWeightVar(const BundleConfig &config, const char *name) { const SymbolTableEntry *mutableWeightVar = getWeightVar(config, name); if (!mutableWeightVar) { printf("Expected to find variable '%s'\n", name); } assert(mutableWeightVar && "Expected to find a mutable weight variable"); assert(mutableWeightVar->kind != 0 && "Weight variable is expected to be mutable"); return *mutableWeightVar; } /// Allocate an aligned block of memory. void *alignedAlloc(const BundleConfig &config, size_t size) { void *ptr; // Properly align the memory region. int res = posix_memalign(&ptr, config.alignment, size); assert(res == 0 && "posix_memalign failed"); assert((size_t)ptr % config.alignment == 0 && "Wrong alignment"); memset(ptr, 0, size); (void)res; return ptr; } /// Initialize the constant weights memory block by loading the weights from the /// weights file. static uint8_t *initConstantWeights(const char *weightsFileName, const BundleConfig &config) { // Load weights. FILE *weightsFile = fopen(weightsFileName, "rb"); if (!weightsFile) { fprintf(stderr, "Could not open the weights file: %s\n", weightsFileName); exit(1); } fseek(weightsFile, 0, SEEK_END); size_t fileSize = ftell(weightsFile); fseek(weightsFile, 0, SEEK_SET); uint8_t *baseConstantWeightVarsAddr = static_cast(alignedAlloc(config, fileSize)); printf("Allocated weights of size: %lu\n", fileSize); printf("Expected weights of size: %" PRIu64 "\n", config.constantWeightVarsMemSize); assert(fileSize == config.constantWeightVarsMemSize && "Wrong weights file size"); int result = fread(baseConstantWeightVarsAddr, fileSize, 1, weightsFile); if (result != 1) { perror("Could not read the weights file"); } else { printf("Loaded weights of size: %lu from the file %s\n", fileSize, weightsFileName); } fclose(weightsFile); return baseConstantWeightVarsAddr; } /// The assumed layout of the area for mutable WeightVars is: /// data | results static uint8_t *allocateMutableWeightVars(const BundleConfig &config) { auto *weights = static_cast( alignedAlloc(config, config.mutableWeightVarsMemSize)); printf("Allocated mutable weight variables of size: %" PRIu64 "\n", config.mutableWeightVarsMemSize); return weights; } /// Dump the result of the inference by looking at the results vector and /// finding the index of the max element. static void dumpInferenceResults(const BundleConfig &config, uint8_t *mutableWeightVars) { const SymbolTableEntry &outputWeights = getMutableWeightVar(config, "save_softmax"); int maxIdx = 0; float maxValue = 0; float *results = (float *)(mutableWeightVars + outputWeights.offset); for (int i = 0; i < outputWeights.size; ++i) { if (results[i] > maxValue) { maxValue = results[i]; maxIdx = i; } } printf("Result: %u\n", maxIdx); } /// The assumed layout of the area for mutable WeightVars is: /// data | results static uint8_t *initMutableWeightVars(const BundleConfig &config) { uint8_t *mutableWeightVarsAddr = allocateMutableWeightVars(config); size_t inputDims[4]; float *inputT{nullptr}; loadImagesAndPreprocess(inputImageFilenames, inputT, inputDims); // Copy image data into the data input variable in the mutableWeightVars area. size_t imageDataSizeInBytes = inputDims[0] * inputDims[1] * inputDims[2] * inputDims[3] * sizeof(float); printf("Copying image data into mutable weight vars: %lu bytes\n", imageDataSizeInBytes); const SymbolTableEntry &inputDataVar = getMutableWeightVar(config, "data"); memcpy(mutableWeightVarsAddr + inputDataVar.offset, inputT, imageDataSizeInBytes); return mutableWeightVarsAddr; } static uint8_t *initActivations(const BundleConfig &config) { return static_cast( alignedAlloc(config, config.activationsMemSize)); } int main(int argc, char **argv) { parseCommandLineOptions(argc, argv); // Allocate and initialize constant and mutable weights. uint8_t *constantWeightVarsAddr = initConstantWeights("lenet_mnist.weights", lenet_mnist_config); uint8_t *mutableWeightVarsAddr = initMutableWeightVars(lenet_mnist_config); uint8_t *activationsAddr = initActivations(lenet_mnist_config); // Perform the computation. lenet_mnist(constantWeightVarsAddr, mutableWeightVarsAddr, activationsAddr); // Report the results. dumpInferenceResults(lenet_mnist_config, mutableWeightVarsAddr); // Free all resources. free(activationsAddr); free(constantWeightVarsAddr); free(mutableWeightVarsAddr); }