matthew Champion
Published © GPL3+

Xbox kinect Movement based midi instrument

A small step in the dream of creating a full 3D body movement midi interpreter.

IntermediateWork in progress2 hours13
Xbox kinect Movement based midi instrument

Things used in this project

Hardware components

Xbox Kinnect
×1

Software apps and online services

midi

Story

Read more

Schematics

Instrument-1

instrument-2

Instrument-3

Code

midi-main.py

Python
You must run this code from a terminal or command interface via pyhton. "python midi-main.py"
#!/usr/bin/env python
import freenect
import cv2
import frame_convert2
import numpy as np
import time
import mido
import rtmidi
from functions import *


dorian = [3, 2, 2, 3, 2]

midiout = rtmidi.MidiOut()
available_ports = midiout.get_ports()

kernel = np.ones((5, 5), np.uint8)
current_depth = 100
max_area = 0
threshold = 760

font = cv2.FONT_HERSHEY_COMPLEX


sw = 0
itt = 0
hand2 = 0
msw = 0
i = 0
ab = 0


notesw = 1
notesw1 = 1
notesw2 = 1
notesw3 = 1
notesw4 = 1
notesw5 = 1

midicount = 1
lengthset = 10

ooc = ""
ooc1 = ""
ooc2 = ""
ooc3 = ""

scalesw = 0
scalesw2 = 0
scrollSwitch =0
cX = 0
cY = 0
rX = 0
rY = 0

if available_ports:
    midiout.open_port(0)
else:
    midiout.open_virtual_port("My virtual output")


def show_depth():
    global current_depth
    global threshold
    global mou
    global df
    thrii = threshold / 100.0
    thi   = 1.0 / thrii
    mou   = thi * 100
    th    = thi * (thi * 150000.0)
    t     = thi * (thi * 25000.0)
    if (max_area > th*2 ):
       threshold = threshold - 5
    if (max_area < t):
       threshold = threshold + 1
       if (threshold > 750):
           threshold = 750

    depth, timestamp = freenect.sync_get_depth()
    depth = 255 * np.logical_and(depth >= current_depth - threshold,
                                 depth <= current_depth + threshold)
    depth = depth.astype(np.uint8)
    df =  depth


while 1:
    show_depth()
    canvas = np.zeros(df.shape, np.uint8)
    canvas2 = np.zeros(df.shape, np.uint8)
    blur = cv2.GaussianBlur(df,(15,45),0)
    ret,thresh1 = cv2.threshold(blur,160,255,cv2.THRESH_BINARY)
    contours, hierarchy = cv2.findContours(thresh1,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)

    if hierarchy is None:
       max_area = 450
       hand2 = 0
       scrollSwitch = 0
    else:
       cnt = contours[0]
       max_area = cv2.contourArea(cnt)
       if (max_area > 400):
          if len(contours) > 1:
              cntt = contours[1]
              for cont2 in contours[1]:
                 approx1 = cv2.approxPolyDP(cont2, 0.035*cv2.arcLength(cont2, True), True)
                 cv2.drawContours(canvas2, [approx1], 0, (0), 5)
                 hand2 = 1
          else:
                 hand2 = 0
                 scrollSwitch = 0
          for cont in contours:
             approx = cv2.approxPolyDP(cont, 0.035*cv2.arcLength(cont, True), True)
             x = approx.ravel()[0]
             y = approx.ravel()[1]

          for cont in contours[0]:
             approx2 = cv2.approxPolyDP(cont, 0.035*cv2.arcLength(cont, True), True)
             cv2.drawContours(canvas, [approx2], 0, (0), 5)

             if cv2.contourArea(cont) > 2000:
                 cnt = cont
                 max_area = cv2.contourArea(cont)
                                                     # define main island contour approx. and hull
       if(hand2 ==1):
          dleft = abs(cX) / 5
          dright = abs(rX) / 5
          dnote = dright - dleft
          scalesw = 0                                            #filter linier input into musical scale
          ab = abs(dnote)
          w = 0

          for x in range(0, 15):
             for x in range(0, len(dorian)):
                if ab == w + dorian[x]:
                    scalesw = 1
                w = w + dorian[x]
          if scalesw == 0:
             ab = ab + (dorian[x] - 1)


          dnote_on = mido.Message('note_on', channel=2, note= ab, velocity=112).bytes()
          dnote_off = mido.Message('note_off', channel=2, note= ab, velocity=0).bytes()

          cv2.drawContours(canvas2, [cntt], -1, (56, 255, 0), 2)
          M2 = cv2.moments(canvas2)
          rx = rX
          ry = rY
          rX = int(M2["m10"] / M2["m00"])
          rY = int(M2["m01"] / M2["m00"])
          if(cx > cX + 2.7) or (cx < cX -2.7) or (rX > rx +2.7) or (rX < rx -2.7):
             if (cx > cX & notesw == 1):
                 midiout.send_message(dnote_on)
                 if notesw4 == 1:
                    ooc4 = note_off
                    notesw4 = 0

             if (rX > rx & notesw1 == 1):
                midiout.send_message(dnote_on)
                if notesw5 == 1:
                    ooc5 = note_offy
                    notesw5 = 0


                                                      #ret,canvas = cv2.threshold(canvas,70,255,cv2.THRESH_BINARY_INV)
       cv2.drawContours(canvas, [cnt], -1, (56, 255, 0), 2)
       M = cv2.moments(canvas)
       cx = cX
       cy = cY                                  	# calculate x,y coordinate of center
       cX = int(M["m10"] / M["m00"])
       cY = int(M["m01"] / M["m00"])

       scalesw = 0                                            #filter linier input into musical scale
       a = abs(cX)  / 5
       w = 0
       for x in range(0, 15):
          for x in range(0, len(dorian)):
             if a == w + dorian[x]:
                 scalesw = 1
             w = w + dorian[x]
       if scalesw == 0:
          a = a + (dorian[x] - 1)

       scalesw2 = 0
       b = abs(cY)  / 5
       w = 0
       for x in range(0, 15):
          for x in range(0, len(dorian)):
             if b == w + dorian[x]:
                 scalesw2 = 1
             w = w + dorian[x]
       if scalesw2 == 0:
          b = b + (dorian[x] - 1)
                                                                                  #define midi messages to send later
       note_on = mido.Message('note_on', channel=9, note= a, velocity=112).bytes()
       note_off = mido.Message('note_off', channel=9, note= a, velocity=0).bytes()
       note_ony = mido.Message('note_on', channel=8, note= b, velocity=112).bytes()
       note_offy = mido.Message('note_off', channel=8, note= b, velocity=0).bytes()
       if scrollSwitch == 1:
          if (cy > cY + 30):
             lengthset = lengthset + 1      # set the length before notes are turned off if you move fast on y axis
             print(lengthset)
          if (cy < cY - 30):
             lengthset = lengthset - 1



       if hand2 == 0:
          if (cy > cY + 20) or (cy < cY -20):
             sw = 0
          else:
             if(cx > cX + 2.7) or (cx < cX -2.7) or (cy > cY +2.7) or (cy < cY -2.7):  # movement detect for note switch on
                if (cx > cX & notesw == 1):
                   midiout.send_message(note_on)
                   if notesw == 1:
                      ooc = note_off
                      notesw = 0

                if (cy > cY & notesw1 == 1):
                   midiout.send_message(note_ony)
                   if notesw1 == 1:
                      ooc1 = note_offy
                      notesw1 = 0

                if (cx < cX & notesw2 == 1):
                   midiout.send_message(note_on)
                   if notesw2 == 1:
                      ooc2 = note_off
                      notesw2 = 0


                if (cy < cY & notesw3 == 1):
                   midiout.send_message(note_ony)
                   if notesw3 == 1:
                      ooc3 = note_offy
                      notesw3 = 0

       canvas = cv2.cvtColor(canvas, cv2.COLOR_RGB2BGR)
       cv2.circle(canvas, (cX, cY), 5, (255, 255, 255), -1)
       if(hand2 == 1):
          cv2.drawContours(canvas, [cntt], -1, (255, 2, 0), 2)
          cv2.circle(canvas, (rX, rY), 5, (255, 255, 255), -1)
          cv2.putText(canvas, "ce2", (rX - 25, rY - 25),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
       cv2.putText(canvas, "ce", (cX - 25, cY - 25),cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255, 255, 255), 2)
       cv2.drawContours(canvas, [cnt], -1, (56, 255, 0), 2)
       scrollSwitch = 1
    midicount = midicount + 1
    if midicount > lengthset:
       if notesw == 0:
          midiout.send_message(ooc)
       if notesw1 == 0:
          midiout.send_message(ooc1)
       if notesw2 == 0:
          midiout.send_message(ooc2)
       if notesw3 == 0:
          midiout.send_message(ooc3)
       if notesw4 == 0:
          midiout.send_message(ooc4)
       if notesw5 == 0:
          midiout.send_message(ooc5)

       midicount = 0
       notesw =  1
       notesw1 = 1
       notesw2 = 1
       notesw3 = 1
       notesw4 = 1
       notesw5 = 1
    cv2.imshow("Contour", canvas)
    if cv2.waitKey(10) == 27:
        break

Credits

matthew Champion

matthew Champion

4 projects • 4 followers
Llandovery, Wales

Comments