Assistance for Cifar-10 on MAX78000FTHR, mod to utilize the onboard camera

[MagnusOvason]

Hi. I am modifying the cigar-10 program to utilize captured images from the onboard camera instead of the sample input. And it is not working now, as it predicts a cat in each image. I don't know what the issue is, but it probably has something to do with how I capture the image, process it, or load it into the CNN, or all of them simultaneously.

I have shared the code with you guys, and ANY HELP would be highly appreciated!
Any general understanding is also very much welcome, as I would like to deploy other models as well, such as a tinyissimoYOLO model or something similar.

Fingers crossed. Cheers!

#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include "mxc.h"
#include "fcr_regs.h"
#include "icc.h"
#include "led.h"
#include "tmr.h"
#include "dma.h"
#include "cnn.h"
#include "weights.h"
#include "camera.h"

#define IMAGE_SIZE_X (32)
#define IMAGE_SIZE_Y (32)

#define CAMERA_FREQ (5 * 1000 * 1000)

const char classes[CNN_NUM_OUTPUTS][11] = {"airplane", "automobile", "bird", "cat", "deer", "dog", "frog", "horse", "ship", "truck"};

// Classification layer:
static int32_t ml_data[CNN_NUM_OUTPUTS];
static q15_t ml_softmax[CNN_NUM_OUTPUTS];

// CNN inference time variable
volatile uint32_t cnn_time; // Stopwatch

// Buffers for camera image
static uint32_t input_0[IMAGE_SIZE_X * IMAGE_SIZE_Y]; // buffer for camera image

// DMA channel for camera interface
int dma_channel;
int g_dma_channel_tft = 1;

void cnn_load_input(void)
{
    memcpy32((uint32_t *)0x50400000, input_0, (IMAGE_SIZE_X * IMAGE_SIZE_Y));
}

void softmax_layer(void)
{
    cnn_unload((uint32_t *)ml_data);
    cnn_stop();
    softmax_q17p14_q15((const q31_t *)ml_data, CNN_NUM_OUTPUTS, ml_softmax);
}

void capture_process_camera(void)
{
    uint8_t *raw;
    uint32_t imgLen;
    uint32_t w, h;

    int cnt = 0;

    uint8_t r, g, b;

    uint8_t *data = NULL;
    stream_stat_t *stat;

    camera_start_capture_image();

    // Get the details of the image from the camera driver.
    camera_get_image(&raw, &imgLen, &w, &h);
    printf("\nW:%d H:%d L:%d \n", w, h, imgLen);

    // Get image line by line
    for (int row = 0; row < h; row++)
    {
        // Wait until camera streaming buffer is full
        while ((data = get_camera_stream_buffer()) == NULL)
        {
            if (camera_is_image_rcv())
            {
                break;
            }
        }

        for (int k = 0; k < 4 * w; k += 4)
        {
            // data format: 0x00bbggrr
            r = data[k];
            g = data[k + 1];
            b = data[k + 2];
            // skip k+3 because it is not used

            // change the range from [0,255] to [-128,127] and store in buffer for CNN
            input_0[cnt++] = ((b << 16) | (g << 8) | r) ^ 0x00808080;
        }

        // LED_Toggle(LED2);
        //  Release stream buffer
        release_camera_stream_buffer();
    }

    // camera_sleep(1);
    stat = get_camera_stream_statistic();

    if (stat->overflow_count > 0)
    {
        printf("OVERFLOW DISP = %d\n", stat->overflow_count);
        LED_On(LED2); // Turn on red LED if overflow detected
        while (1)
        {
        }
    }
}

int main(void)
{
    int i;
    int digs, tens;
    int ret = 0;
    int result[CNN_NUM_OUTPUTS] = {0}; // = {0} ensures that all elements are initialized to 0
    int dma_channel;

    MXC_Delay(SEC(2)); // Wait for 2 seconds

    // Enable camera power
    printf("Enable Camera Power...\n");
    Camera_Power(POWER_ON);

    printf("\nCifar-10 Feather Demo\n");

    // Enable cache
    MXC_ICC_Enable(MXC_ICC0);

    // Switch to 100 MHz clock
    MXC_SYS_Clock_Select(MXC_SYS_CLOCK_IPO);
    SystemCoreClockUpdate();

    printf("Waiting...\n");

    // Enable peripheral, enable CNN interrupt, turn on CNN clock
    // CNN clock: APB (50 MHz) div 1
    cnn_enable(MXC_S_GCR_PCLKDIV_CNNCLKSEL_PCLK, MXC_S_GCR_PCLKDIV_CNNCLKDIV_DIV1);

    // Configure P2.5, turn on the CNN Boost, which helps with CNN performance
    // cnn_boost_enable(MXC_GPIO2, MXC_GPIO_PIN_5);

    printf("Init CNN...\n");
    cnn_init();         // Bring state machine into consistent state
    cnn_load_weights(); // Load kernels
    cnn_load_bias();    // Load bias values
    cnn_configure();    // Configure state machine

    // Initialize DMA for camera interface
    MXC_DMA_Init();
    dma_channel = MXC_DMA_AcquireChannel();

    // Initialize camera
    printf("Init Camera...\n");
    camera_init(CAMERA_FREQ);

    // Set up camera
    ret = camera_setup(IMAGE_SIZE_X, IMAGE_SIZE_Y, PIXFORMAT_RGB888, FIFO_THREE_BYTE, STREAMING_DMA,
                       dma_channel);
    if (ret != STATUS_OK)
    {
        printf("Error returned from setting up camera. Error %d\n", ret);
        return -1;
    }

    camera_write_reg(0x11, 0x0); // Set the camera to sleep mode

    printf("********** Next snapshot in 1 second **********\r\n");
    MXC_Delay(SEC(1));

    // Enable CNN clock
    MXC_SYS_ClockEnable(MXC_SYS_PERIPH_CLOCK_CNN);

    while (1)
    {
        LED_Off(LED1);
        LED_Off(LED2);

        // Capture image from camera
        printf("\nCapture and process camera image.\n");
        capture_process_camera();

        // Start CNN processing
        printf("\n*** CNN Inference Test ***\n");
        cnn_start();

        // Load camera image into CNN input buffer
        printf("Load CNN input...\n");
        cnn_load_input();

        // Wait for CNN
        SCB->SCR &= ~SCB_SCR_SLEEPDEEP_Msk; // SLEEPDEEP=0
        while (cnn_time == 0)
            __WFI();

        // Softmax layer
        softmax_layer();

        // Print inference time
        printf("Time for CNN: %d us\n\n", cnn_time);

        // Print classification results
        printf("\nClassification results:\n");
        for (i = 0; i < CNN_NUM_OUTPUTS; i++)
        {
            digs = (1000 * ml_softmax[i] + 0x4000) >> 15;
            tens = digs % 10;
            digs = digs / 10;
            result[i] = digs;
            printf("[%7d] -> Class %d %11s: %d.%d%%\r\n\n", ml_data[i], i, classes[i], result[i],
                   tens);
        }

        printf("\n");

        // Wait for 1 second
        printf("********** Next snapshot in 1 second **********\r\n");
        MXC_Delay(SEC(1));
    }

    return 0;
}

[MagnusOvason]

Managed to get it working with some extra mods. Final code can be seen below:

/*******************************************************************************
 * Copyright (C) 2019-2023 Maxim Integrated Products, Inc., All rights Reserved.
 *
 * This software is protected by copyright laws of the United States and
 * of foreign countries. This material may also be protected by patent laws
 * and technology transfer regulations of the United States and of foreign
 * countries. This software is furnished under a license agreement and/or a
 * nondisclosure agreement and may only be used or reproduced in accordance
 * with the terms of those agreements. Dissemination of this information to
 * any party or parties not specified in the license agreement and/or
 * nondisclosure agreement is expressly prohibited.
 *
 * The above copyright notice and this permission notice shall be included
 * in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
 * IN NO EVENT SHALL MAXIM INTEGRATED BE LIABLE FOR ANY CLAIM, DAMAGES
 * OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Except as contained in this notice, the name of Maxim Integrated
 * Products, Inc. shall not be used except as stated in the Maxim Integrated
 * Products, Inc. Branding Policy.
 *
 * The mere transfer of this software does not imply any licenses
 * of trade secrets, proprietary technology, copyrights, patents,
 * trademarks, maskwork rights, or any other form of intellectual
 * property whatsoever. Maxim Integrated Products, Inc. retains all
 * ownership rights.
 *******************************************************************************/

// cifar-10-max78000fthr
// This file was @generated by ai8xize.py --test-dir sdk/Examples/MAX78000/CNN --prefix cifar-10-max78000fthr --board-name FTHR_RevA --checkpoint-file trained/ai85-cifar10-qat8-q.pth.tar --config-file networks/cifar10-nas.yaml --fifo --softmax --device MAX78000 --timer 0 --display-checkpoint --verbose --overwrite

#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include <stdio.h>
#include "board.h"
#include "mxc.h"
#include "cnn.h"
#include "sampledata.h"
#include "sampleoutput.h"
#include "camera.h"
#include "dma.h"
#include "pb.h"

#define CAMERA_FREQ (5 * 1000 * 1000) // 5 MHz
#define IMAGE_XRES (32)               // X resolution
#define IMAGE_YRES (32)               // Y resolution

const char classes[CNN_NUM_OUTPUTS][16] = {
    "airplane",
    "automobile",
    "bird",
    "cat",
    "deer",
    "dog",
    "frog",
    "horse",
    "ship",
    "truck"};

volatile uint32_t cnn_time; // Stopwatch

// Data input: HWC 3x32x32 (3072 bytes total / 1024 bytes per channel): IMAGE_XRES * IMAGE_YRES = 32 * 32 = 1024
static uint32_t input_from_camera[IMAGE_XRES * IMAGE_YRES];
void load_input(void)
{
  int i;
  const uint32_t *in0 = input_from_camera;

  for (i = 0; i < (IMAGE_XRES * IMAGE_YRES); i++)
  {
    // Remove the following line if there is no risk that the source would overrun the FIFO:
    while (((*((volatile uint32_t *)0x50000004) & 1)) != 0)
      ;                                          // Wait for FIFO 0
    *((volatile uint32_t *)0x50000008) = *in0++; // Write FIFO 0
  }
}

// Classification layer:
static int32_t ml_data[CNN_NUM_OUTPUTS];
static q15_t ml_softmax[CNN_NUM_OUTPUTS];

void softmax_layer(void)
{
  cnn_unload((uint32_t *)ml_data);
  softmax_q17p14_q15((const q31_t *)ml_data, CNN_NUM_OUTPUTS, ml_softmax);
}

void capture_process_camera(void)
{
  uint8_t *rawData;
  uint32_t imageLength, imageWidth, imageHeight;
  uint8_t *data = NULL;
  uint8_t r, g, b;
  stream_stat_t *stat;
  int cnt = 0;

  camera_start_capture_image(); // Start camera capture

  camera_get_image(&rawData, &imageLength, &imageWidth, &imageHeight); // Get image from camera
  printf("\nRaw Data: %d\n", rawData);
  printf("Image Length: %d\n", imageLength);
  printf("Image Width: %d\n", imageWidth);
  printf("Image Height: %d\n", imageHeight);

  // Get image line by line
  for (int row = 0; row < imageHeight; row++)
  {
    // Wait until camera streaming buffer is full
    while ((data = get_camera_stream_buffer()) == NULL)
    {
      if (camera_is_image_rcv())
      {
        break;
      }
    }

    for (int k = 0; k < 4 * imageWidth; k += 4)
    {
      // data format: 0x00bbggrr
      r = data[k];
      g = data[k + 1];
      b = data[k + 2];
      // skip k+3

      // change the range from [0,255] to [-128,127] and store in buffer for CNN
      // input_from_camera[cnt++] = ((b << 16) | (g << 8) | r) ^ 0x00808080;
      input_from_camera[cnt] = ((b << 16) | (g << 8) | r) ^ 0x00808080;
      cnt++;

      release_camera_stream_buffer(); // Release camera streaming buffer
    }

    stat = get_camera_stream_statistic();

    if (stat->overflow_count > 0)
    {
      printf("OVERFLOW DISP = %d\n", stat->overflow_count);
      LED_On(LED_RED); // Turn on red LED if overflow detected
      while (1)
      {
      }
    }
  }

  return;
}

int main(void)
{
  int i;
  int digs, tens;
  int result[CNN_NUM_OUTPUTS];
  int dma_channel;
  int status;

  MXC_Delay(SEC(2)); // Wait for PMIC 1.8V to stabilize, about 180ms after power up

  Camera_Power(POWER_ON); // Power on camera

  printf("\n*** Cifar-10 Feather Program ***\n");

  MXC_ICC_Enable(MXC_ICC0); // Enable cache

  // Switch to 100 MHz clock
  MXC_SYS_Clock_Select(MXC_SYS_CLOCK_IPO);
  SystemCoreClockUpdate();

  // Enable peripheral, enable CNN interrupt, turn on CNN clock
  // CNN clock: APB (50 MHz) div 1
  cnn_enable(MXC_S_GCR_PCLKDIV_CNNCLKSEL_PCLK, MXC_S_GCR_PCLKDIV_CNNCLKDIV_DIV1);

  /* Configure P2.5, turn on the CNN Boost */
  cnn_boost_enable(MXC_GPIO2, MXC_GPIO_PIN_5);

  /* Bring CNN state machine into consistent state */
  cnn_init();
  /* Load CNN kernels */
  cnn_load_weights();
  /* Load CNN bias */
  cnn_load_bias();
  /* Configure CNN state machine */
  cnn_configure();

  printf("Waiting...\n");
  MXC_Delay(SEC(1));

  MXC_DMA_Init();                         // Initialize DMA
  dma_channel = MXC_DMA_AcquireChannel(); // Acquire DMA channel
  printf("DMA setup has been completed\n");
  printf("Init Camera.\n");
  camera_init(CAMERA_FREQ); // Initialize camera

  status = camera_setup(IMAGE_XRES, IMAGE_YRES, PIXFORMAT_RGB888, FIFO_THREE_BYTE, STREAMING_DMA, dma_channel); // Setup camera
  if (status != STATUS_OK)
  {
    printf("Error returned from setting up camera. Error %d\n", status);
    return -1;
  }

  camera_write_reg(0x11, 0x0); // set camera clock prescaller to prevent streaming overflow

  printf("********** Press PB1(SW1) to capture an image **********\r\n");
  LED_On(LED_BLUE);
  while (!PB_Get(0))
  {
  }
  MXC_Delay(MSEC(200)); // Debounce

  printf("\n*** CNN Inference Test cifar-10-max78000fthr ***\n");

  // Enable CNN clock
  MXC_SYS_ClockEnable(MXC_SYS_PERIPH_CLOCK_CNN);

  while (1)
  {
    LED_Off(LED_BLUE);
    LED_On(LED_GREEN);
    LED_Off(LED_RED);

    capture_process_camera(); // Capture image from camera

    cnn_start();         // Start CNN processing
    load_input();        // Load input data

    SCB->SCR &= ~SCB_SCR_SLEEPDEEP_Msk; // SLEEPDEEP=0
    while (cnn_time == 0)
    {
      __WFI(); // Wait for CNN interrupt
    }

    // Unload CNN data
    cnn_unload((uint32_t *)ml_data);
    cnn_stop();

    // Softmax
    softmax_q17p14_q15((const q31_t *)ml_data, CNN_NUM_OUTPUTS, ml_softmax);

    printf("Time for CNN: %d us\n\n", cnn_time);

    printf("Classification results:\n");

    for (i = 0; i < CNN_NUM_OUTPUTS; i++)
    {
      digs = (1000 * ml_softmax[i] + 0x4000) >> 15;
      tens = digs % 10;
      digs = digs / 10;
      result[i] = digs;
      printf("[%7d] -> Class %d %11s: %d.%d%%\n", ml_data[i], i, classes[i], result[i],
             tens);
    }

    printf("\n");

    LED_On(LED_BLUE);
    LED_Off(LED_GREEN);

    printf("********** Press PB1(SW1) to capture an image **********\r\n");
    while (!PB_Get(0))
    {
    }
    MXC_Delay(MSEC(200)); // Debounce
  }

  return 0;
}

/*
  SUMMARY OF OPS
  Hardware: 36,481,536 ops (36,180,992 macc; 300,544 comp; 0 add; 0 mul; 0 bitwise)
    Layer 0: 1,835,008 ops (1,769,472 macc; 65,536 comp; 0 add; 0 mul; 0 bitwise)
    Layer 1: 2,129,920 ops (2,097,152 macc; 32,768 comp; 0 add; 0 mul; 0 bitwise)
    Layer 2: 18,939,904 ops (18,874,368 macc; 65,536 comp; 0 add; 0 mul; 0 bitwise)
    Layer 3: 4,792,320 ops (4,718,592 macc; 73,728 comp; 0 add; 0 mul; 0 bitwise)
    Layer 4: 540,672 ops (524,288 macc; 16,384 comp; 0 add; 0 mul; 0 bitwise)
    Layer 5: 4,743,168 ops (4,718,592 macc; 24,576 comp; 0 add; 0 mul; 0 bitwise)
    Layer 6: 1,056,768 ops (1,048,576 macc; 8,192 comp; 0 add; 0 mul; 0 bitwise)
    Layer 7: 1,188,864 ops (1,179,648 macc; 9,216 comp; 0 add; 0 mul; 0 bitwise)
    Layer 8: 1,181,696 ops (1,179,648 macc; 2,048 comp; 0 add; 0 mul; 0 bitwise)
    Layer 9: 68,096 ops (65,536 macc; 2,560 comp; 0 add; 0 mul; 0 bitwise)
    Layer 10: 5,120 ops (5,120 macc; 0 comp; 0 add; 0 mul; 0 bitwise)

  RESOURCE USAGE
  Weight memory: 301,760 bytes out of 442,368 bytes total (68.2%)
  Bias memory:   842 bytes out of 2,048 bytes total (41.1%)
*/