python使用opencv实现手势识别并控制ppt

news2024/11/15 8:18:37

需要使用到的包

from collections import deque

import cv2
import numpy as np
import math
import shutil

import sys
import os
import time

#这个求出现频率最高的太慢了，所以把它放弃了
from collections import Counter

准备好安装包后需要获取图片

def star():
    while camera.isOpened():
        global frame
        ret, frame = camera.read()
        frame = cv2.flip(frame, 1)
        cv2.imshow('ori', frame)
        #
        # img=frame[0:int(0.8 * frame.shape[0]),
        #       int(0.5 * frame.shape[1]):frame.shape[1]][0:int(0.8 * frame.shape[0]),
        #       int(0.5 * frame.shape[1]):frame.shape[1]]
        frame,ndefects=grdetect(frame)
        # cv2.imshow('min', img)
        k = cv2.waitKey(1)
        if k == 27:
            camera.release()
            cv2.destroyAllWindows()
            break

写进方法start中
视频也是图片构成的，只是在不同帧展示不同的图片而已。这里根据自己电脑的性能选择取图片的频率。

然后一处图片的噪点

def _remove_background(frame):
    fgbg = cv2.createBackgroundSubtractorMOG2()  # 利用BackgroundSubtractorMOG2算法消除背景
    fgmask = fgbg.apply(frame)
    kernel = np.ones((3, 3), np.uint8)
    fgmask = cv2.erode(fgmask, kernel, iterations=1)
    res = cv2.bitwise_and(frame, frame, mask=fgmask)
    # cv2.imshow('res',fgmask)
    return res

最终是给机器看的，在让他处理之前尽量降低影响条件。

再根据皮肤识别获取手的大致形状

def _bodyskin_detetc(frame):
    # 肤色检测: YCrCb之Cr分量 + OTSU二值化
    ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)  # 分解为YUV图像,得到CR分量
    (_, cr, _) = cv2.split(ycrcb)
    cr1 = cv2.GaussianBlur(cr, (5, 5), 0)  # 高斯滤波
    _, skin = cv2.threshold(cr1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)  # OTSU图像二值化
    # cv2.imshow('skin',skin)
    return skin

获取大致轮廓

def _get_contours(array):
        # 利用findContours检测图像中的轮廓, 其中返回值contours包含了图像中所有轮廓的坐标点
        contours, _ = cv2.findContours(array, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        return contours

接下来准对这个大致图形进行处理
这里利用到了凹凸图处理（腐蚀，另外一个名词想不起来了！）


# 根据图像中凹凸点中的 (开始点, 结束点, 远点)的坐标, 利用余弦定理计算两根手指之间的夹角, 其必为锐角, 根据锐角的个数判别手势.
def _get_defects_count(array, contour,defects, verbose = False):
    ndefects = 0
    list=[]
    for i in range(defects.shape[0]):
        s,e,f,_ = defects[i,0]
        beg     = tuple(contour[s][0])
        end     = tuple(contour[e][0])
        far     = tuple(contour[f][0])
        a = math.sqrt((beg[0] - end[0]) ** 2 + (beg[1] - end[1]) ** 2)
        b = math.sqrt((beg[0] - far[0]) ** 2 + (beg[1] - far[1]) ** 2)
        c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2)
        angle   = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) # * 57
        if angle <= math.pi/2 :#90:
            if far[1]>350:#有的不需要的杂点在这里给他筛出
                break
            ndefects = ndefects + 1
            if verbose:
                cv2.circle(array, far, 3, [255,0,0], -1)
        if verbose:
            cv2.line(array, beg, end, [0,255,0], 1)
            cv2.circle(array, beg, 3, [255, 255, 0], -1)
            list.append(beg)

        cv2.imshow('arry',array)
    return array,ndefects,list

根据处理咱们可以获取得到五个数据，咱们需要利用其中的三个数据 (开始点, 结束点, 远点)通过反余弦定理求出手指岔开角度，进而判断伸出手指的个数。
为了使系统稳定运行这里家里arry进行缓存50个前50帧图像处理结果然后综合判断此时此刻的识别结果。

然后就是利用手势操作ppt了


def grdetect(array):
    global num_ppt
    strDath=os.getcwd()+'/'+str(num_ppt)+'.JPG'

    print('strDath',strDath)
    #读取ppt
    ppt1 = cv2.imread(strDath)

    copy       = array.copy()
    array = _remove_background(array) # 移除背景, add by wnavy
    thresh = _bodyskin_detetc(array)
    contours= _get_contours(thresh.copy()) # 计算图像的轮廓
    largecont  = max(contours, key = lambda contour: cv2.contourArea(contour))
    hull           = cv2.convexHull(largecont, returnPoints = False) # 计算轮廓的凸点
    try:
        defects        = cv2.convexityDefects(largecont, hull) # 计算轮廓的凹点
    except:
        print('凸点有问题')
        defects=None
    if defects is not None:
        # 利用凹陷点坐标, 根据余弦定理计算图像中锐角个数
        ndefects=''
        copy,ndefects,list = _get_defects_count(copy, largecont, defects, verbose = True)
        # 根据锐角个数判断手势, 会有一定的误差

        if ndefects == 0:

            num1.append(0)

            num2.append(0)

            min_lin=10000
            min_point=(0,0)
            if (sum(num1)<50 and sum(num2)<50):
                for i in range(len(list)):
                    if min_lin>list[i][1]:
                        min_lin=list[i][1]
                        min_point=list[i]
                # print('min_point=',min_point)
                dx.append(min_point[0])
                dy.append(min_point[1])
                # print('ppt大小',ppt1.shape)
                #ppt大小 (720, 960, 3)
                cv2.rectangle(copy, (0, 0),
                              (int(0.8 * frame.shape[1]), int(0.6 * frame.shape[0])), (255, 0, 0), 2)
                # print('宽：',0.8 * frame.shape[1],'高：',0.6 * frame.shape[0])
                #宽： 512.0 高： 288.0

                cv2.circle(copy, (sum(dx)//5,sum(dy)//5), 6, [255,0 , 255], -1)
                #宽512:960
                witch = int(np.interp(sum(dx)//5, [0, 512], [0, 960]))
                #高288:720
                height = int(np.interp(sum(dy)//5, [0, 288], [0, 720]))
                cv2.circle(ppt1, (witch, height), 6, [255, 0, 255], -1)
                cv2.imshow('copy',copy)
                try:
                    cv2.imshow('ppt', ppt1)
                except:
                    print('少一个手指一')
            print(0)
        elif ndefects == 1:

            num1.append(2)
            num2.append(0)
            if (sum(num1) > 50 and sum(num2) < 50):

                min_lin = 10000
                min_point = (0, 0)
                for i in range(len(list)):
                    if min_lin > list[i][1]:
                        min_lin = list[i][1]
                        min_point = list[i]
                # print('min_point=', min_point)
                dx.append(min_point[0])
                dy.append(min_point[1])
                # 宽512:960
                witch = int(np.interp(sum(dx) // 5, [0, 512], [0, 960]))
                # 高288:720
                height = int(np.interp(sum(dy) // 5, [0, 288], [0, 720]))
                print('sum(num1)=',sum(num1))
                if sum(num1)>50 and sum(num2) <50:
                    global imgCanvas
                    global xp,yp
                    if xp==0 and yp==0:
                        xp,yp=witch,height
                    cv2.line(ppt1, (xp,yp),(witch, height),[0, 255, 0], 15)
                    cv2.line(imgCanvas, (xp, yp), (witch, height), [155,155 , 0], 15)
                    xp, yp = witch, height
                    cv2.imshow('ppt',ppt1)
                    cv2.imshow('imgCanvas',imgCanvas)
            elif 20<sum(num1)<50 or sum(num2) != 0:
                img = cv2.putText(ppt1, "pencil loading....", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1.2, (0, 255, 0), 2)
                try:
                    cv2.imshow('ppt', img)
                except:
                    print('少一个手指二')
            print(1+1)