Raspberry Pi Forums

Posted: **Sat Aug 15, 2015 3:09 pm**

Hello all,
I'm trying to use the camera in a project where real-time response is important. I want to play sounds in response to movements.
Focused in the problem of acquiring images fast and processing it, and after many tests, I have finnally developed two utility classes, based on the advanced recipe proposed in picamera docs.
Please, analyse/use/test it, any feedback to improve it will be welcome.

Note than the condicions are:

Only one frame is proccessed at a time
If a new frame is captured and another is waiting processing, the "old" frame is discarded.
While capturing a frame (before yield is called) if image processor thread detects is finalyced, the frame is processed inmediatelly. Again, "old" frames could be discarded.

The main class (a thread):

Code: Select all

import threading, time, io
import picamera, picamera.array, cv2
import imageprocessor

class RTOpenCV(threading.Thread):
    # stop condition for the thread
    stop = False
    status_lock = threading.Lock()
    # possible frame status: Empty, Capturing, Ready, Processing
    status = ['Empty','Empty','Empty']

    def __init__(self, camera, callback):
        super(RTOpenCV, self).__init__()
        RTOpenCV.camera = camera
        # three streams will be used
        #   only one is in Processing/Capturing at a time
        RTOpenCV.streams = [picamera.array.PiRGBArray(self.camera),
                            picamera.array.PiRGBArray(self.camera),
                            picamera.array.PiRGBArray(self.camera)]
        RTOpenCV.processor = imageprocessor.ImageProcessor()
        RTOpenCV.callback = staticmethod(callback)
        self.start()

    def run(self):
        RTOpenCV.camera.capture_sequence(RTOpenCV._streams(), use_video_port=True, format='bgr')

    def close(self):
        RTOpenCV.stop = True
        self.join()
        RTOpenCV.processor.join()

    @staticmethod
    def _discard_frames(last_frame):
        # check if another frame is Ready
        # in that case, discard it (only last frame is usefull)
        if RTOpenCV.status[(last_frame+1)%3] == 'Ready':
            RTOpenCV.status[(last_frame+1)%3] = 'Empty'
        if RTOpenCV.status[(last_frame+2)%3] == 'Ready':
            RTOpenCV.status[(last_frame+2)%3] = 'Empty'

    @staticmethod
    def _streams():
        # yield a stream for capture_sequence
        frames=0
        # first select an non used stream
        while not RTOpenCV.stop:
            free_stream = -1
            with RTOpenCV.status_lock:
                for i in range(3):
                    if RTOpenCV.status[i] == 'Empty':
                        RTOpenCV.status[i] = 'Capturing'
                        free_stream = i
                        break
                if free_stream == -1:
                    for i in range(3):
                        if RTOpenCV.status[i] == 'Ready':
                            RTOpenCV.status[i] = 'Capturing'
                            free_stream = i
                            break
                RTOpenCV.streams[free_stream].seek(0)

            yield RTOpenCV.streams[free_stream]

            # if stream have not been used, mark it ready
            with RTOpenCV.status_lock:
                if RTOpenCV.status[free_stream]=='Capturing':
                    RTOpenCV.status[free_stream]='Ready'
                    RTOpenCV._discard_frames(free_stream)
            frames+=1

The processor class (a thread too):

Code: Select all

import threading, time
import cv2

class ImageProcessor(threading.Thread):
    def __init__(self):
        super(ImageProcessor, self).__init__()
        import rtopencv
        self.frame_size = rtopencv.RTOpenCV.camera.resolution[0]*\
                        rtopencv.RTOpenCV.camera.resolution[1]*3
        self.num_frames = 0
        self.start()

    def run(self):
        import rtopencv
        while not rtopencv.RTOpenCV.stop:
            # search the last captured frame
            ready = -1
            with rtopencv.RTOpenCV.status_lock:
                for i in range(3):
                    if rtopencv.RTOpenCV.streams[i].tell()==self.frame_size\
                            and rtopencv.RTOpenCV.status[i] == 'Capturing':
                        rtopencv.RTOpenCV.status[i] = 'Processing'
                        rtopencv.RTOpenCV._discard_frames(i)
                        ready = i
                        break
                if ready == -1:
                    for i in range(3):
                        if rtopencv.RTOpenCV.status[i] == 'Ready':
                            rtopencv.RTOpenCV.status[i] = 'Processing'
                            rtopencv.RTOpenCV._discard_frames(i)
                            ready = i
                            break
            # if frame found, process it
            if ready > -1:
                rtopencv.RTOpenCV.callback(rtopencv.RTOpenCV, rtopencv.RTOpenCV.streams[ready].array)
                self.num_frames+=1
                # when processed, mark stream empty
                rtopencv.RTOpenCV.status[ready]='Empty'
            else:
                #no frame available
                time.sleep(0.002)

Test code:

Code: Select all

import time, io
import picamera
import cv2
import rtopencv

# process de frame , rtopencv and frame a passed as parameters
def process_image(rtopencv_class, frame):
    cv2.imshow("Frames",frame)
    cv2.waitKey(1)

with picamera.PiCamera() as camera:
    # initialice the camera as desired
    camera.resolution = (640, 480)
    camera.framerate = 20
    time.sleep(1)
    # create an instance (two thread ar created,
    #   one for capturing, other for processing)
    opencv = rtopencv.RTOpenCV(camera, process_image)
    # make other things
    time.sleep(10)
    # when want to finish image processing, call close()
    opencv.close()

In this particular sample, some times a frame seams to be processed unordered and a flick is observed. Haven't found why.

Raspberry Pi Forums

Real time processing images with OpenCV

Real time processing images with OpenCV