Paul DeCarlo
Published © MIT

Custom Object Detection with CSI IR Camera on NVIDIA Jetson

Detect any thing at any time using a Camera Serial Interface Infrared Camera on an NVIDIA Jetson Nano with Azure IoT and Cognitive Services.

AdvancedFull instructions provided4 hours2,491
Custom Object Detection with CSI IR Camera on NVIDIA Jetson

Things used in this project

Hardware components

NVIDIA Jetson Nano Developer Kit
NVIDIA Jetson Nano Developer Kit
×1
IMX219-77IR 8-Megapixels Infrared Night Vision IR Camera Module
×1

Software apps and online services

Azure IoT Edge
Microsoft Azure IoT Edge
Microsoft Azure
Microsoft Azure

Story

Read more

Code

nvidia-jetson-nano-dcm.json

JSON
The NVIDIA Jetson Nano Device Capability Model
{
    "@id": "urn:AzureIoT:NVIDIAJetsonNano:1",
    "@type": "CapabilityModel",
    "implements": [],
    "displayName": {
        "en": "NVIDIA Jetson Nano DCM"
    },
    "contents": [
        {
            "@id": "urn:AzureIoT:NVIDIAJetsonNano:NVIDIANanoIotCModule:1",
            "@type": [
                "Relationship",
                "SemanticType/EdgeModule"
            ],
            "displayName": {
                "en": "NVIDIA Jetson Nano"
            },
            "name": "NVIDIANanoIotCModule",
            "maxMultiplicity": 1,
            "target": [
                {
                    "@id": "urn:AzureIoT:NVIDIANanoIotCModule:1",
                    "@type": "CapabilityModel",
                    "implements": [
                        {
                            "@type": "InterfaceInstance",
                            "displayName": {
                                "en": "Settings"
                            },
                            "name": "Settings",
                            "schema": {
                                "@id": "urn:AzureIoT:NVIDIANanoIotCModule:ISettings:1",
                                "@type": "Interface",
                                "displayName": {
                                    "en": "Settings"
                                },
                                "contents": [
                                    {
                                        "@type": "Property",
                                        "displayName": {
                                            "en": "Primary Detection Class"
                                        },
                                        "name": "wpPrimaryDetectionClass",
                                        "writable": true,
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Property",
                                        "displayName": {
                                            "en": "Secondary Detection Class"
                                        },
                                        "name": "wpSecondaryDetectionClass",
                                        "writable": true,
                                        "schema": "string"
                                    }
                                ]
                            }
                        },
                        {
                            "@type": "InterfaceInstance",
                            "name": "DeviceInformation",
                            "displayName": {
                                "en": "Device information"
                            },
                            "schema": {
                                "@id": "urn:azureiot:DeviceManagement:DeviceInformation:1",
                                "@type": "Interface",
                                "displayName": {
                                    "en": "Device information"
                                },
                                "contents": [
                                    {
                                        "@type": "Property",
                                        "comment": "Company name of the device manufacturer. This could be the same as the name of the original equipment manufacturer (OEM). Ex. Contoso.",
                                        "displayName": {
                                            "en": "Manufacturer"
                                        },
                                        "name": "manufacturer",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Property",
                                        "comment": "Device model name or ID. Ex. Surface Book 2.",
                                        "displayName": {
                                            "en": "Device model"
                                        },
                                        "name": "model",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Property",
                                        "comment": "Version of the software on your device. This could be the version of your firmware. Ex. 1.3.45",
                                        "displayName": {
                                            "en": "Software version"
                                        },
                                        "name": "swVersion",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Property",
                                        "comment": "Name of the operating system on the device. Ex. Windows 10 IoT Core.",
                                        "displayName": {
                                            "en": "Operating system name"
                                        },
                                        "name": "osName",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Property",
                                        "comment": "Architecture of the processor on the device. Ex. x64 or ARM.",
                                        "displayName": {
                                            "en": "Processor architecture"
                                        },
                                        "name": "processorArchitecture",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Property",
                                        "comment": "Name of the manufacturer of the processor on the device. Ex. Intel.",
                                        "displayName": {
                                            "en": "Processor manufacturer"
                                        },
                                        "name": "processorManufacturer",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Property",
                                        "comment": "Total available storage on the device in kilobytes. Ex. 2048000 kilobytes.",
                                        "displayName": {
                                            "en": "Total storage"
                                        },
                                        "name": "totalStorage",
                                        "displayUnit": {
                                            "en": "kilobytes"
                                        },
                                        "schema": "long"
                                    },
                                    {
                                        "@type": "Property",
                                        "comment": "Total available memory on the device in kilobytes. Ex. 256000 kilobytes.",
                                        "displayName": {
                                            "en": "Total memory"
                                        },
                                        "name": "totalMemory",
                                        "displayUnit": {
                                            "en": "kilobytes"
                                        },
                                        "schema": "long"
                                    }
                                ]
                            }
                        },
                        {
                            "@type": "InterfaceInstance",
                            "name": "ModuleInformation",
                            "displayName": {
                                "en": "Module Information"
                            },
                            "schema": {
                                "@id": "urn:AzureIoT:NVIDIANanoIotCModule:IModuleInformation:1",
                                "@type": "Interface",
                                "displayName": {
                                    "en": "Module Information"
                                },
                                "contents": [
                                    {
                                        "@type": "Telemetry",
                                        "displayName": {
                                            "en": "System Heartbeat"
                                        },
                                        "name": "tlSystemHeartbeat",
                                        "schema": "integer"
                                    },
                                    {
                                        "@type": "Telemetry",
                                        "displayName": {
                                            "en": "Primary Detection Count"
                                        },
                                        "name": "tlPrimaryDetectionCount",
                                        "schema": "integer"
                                    },
                                    {
                                        "@type": "Telemetry",
                                        "displayName": {
                                            "en": "Secondary Detection Count"
                                        },
                                        "name": "tlSecondaryDetectionCount",
                                        "schema": "integer"
                                    },
                                    {
                                        "@type": "Telemetry",
                                        "displayName": {
                                            "en": "Inference"
                                        },
                                        "name": "tlInference",
                                        "schema": {
                                            "@type": "Object",
                                            "displayName": {
                                                "en": "Object"
                                            },
                                            "fields": [
                                                {
                                                    "@type": "SchemaField",
                                                    "displayName": {
                                                        "en": "cameraId"
                                                    },
                                                    "name": "cameraId",
                                                    "schema": "string"
                                                },
                                                {
                                                    "@type": "SchemaField",
                                                    "displayName": {
                                                        "en": "trackingId"
                                                    },
                                                    "name": "trackingId",
                                                    "schema": "integer"
                                                },
                                                {
                                                    "@type": "SchemaField",
                                                    "displayName": {
                                                        "en": "className"
                                                    },
                                                    "name": "className",
                                                    "schema": "integer"
                                                },
                                                {
                                                    "@type": "SchemaField",
                                                    "displayName": {
                                                        "en": "roi"
                                                    },
                                                    "name": "roi",
                                                    "schema": {
                                                        "@type": "Object",
                                                        "displayName": {
                                                            "en": "Object"
                                                        },
                                                        "fields": [
                                                            {
                                                                "@type": "SchemaField",
                                                                "displayName": {
                                                                    "en": "left"
                                                                },
                                                                "name": "left",
                                                                "schema": "double"
                                                            },
                                                            {
                                                                "@type": "SchemaField",
                                                                "displayName": {
                                                                    "en": "top"
                                                                },
                                                                "name": "top",
                                                                "schema": "double"
                                                            },
                                                            {
                                                                "@type": "SchemaField",
                                                                "displayName": {
                                                                    "en": "right"
                                                                },
                                                                "name": "right",
                                                                "schema": "double"
                                                            },
                                                            {
                                                                "@type": "SchemaField",
                                                                "displayName": {
                                                                    "en": "bottom"
                                                                },
                                                                "name": "bottom",
                                                                "schema": "double"
                                                            }
                                                        ]
                                                    }
                                                }
                                            ]
                                        }
                                    },
                                    {
                                        "@type": [
                                            "Telemetry",
                                            "SemanticType/State"
                                        ],
                                        "displayName": {
                                            "en": "Module State"
                                        },
                                        "name": "stModuleState",
                                        "schema": {
                                            "@type": "Enum",
                                            "valueSchema": "string",
                                            "enumValues": [
                                                {
                                                    "@type": "EnumValue",
                                                    "displayName": {
                                                        "en": "inactive"
                                                    },
                                                    "enumValue": "inactive",
                                                    "name": "inactive"
                                                },
                                                {
                                                    "@type": "EnumValue",
                                                    "displayName": {
                                                        "en": "active"
                                                    },
                                                    "enumValue": "active",
                                                    "name": "active"
                                                }
                                            ]
                                        }
                                    },
                                    {
                                        "@type": [
                                            "Telemetry",
                                            "SemanticType/State"
                                        ],
                                        "displayName": {
                                            "en": "Pipeline State"
                                        },
                                        "name": "stPipelineState",
                                        "schema": {
                                            "@type": "Enum",
                                            "valueSchema": "string",
                                            "enumValues": [
                                                {
                                                    "@type": "EnumValue",
                                                    "displayName": {
                                                        "en": "inactive"
                                                    },
                                                    "enumValue": "inactive",
                                                    "name": "inactive"
                                                },
                                                {
                                                    "@type": "EnumValue",
                                                    "displayName": {
                                                        "en": "active"
                                                    },
                                                    "enumValue": "active",
                                                    "name": "active"
                                                }
                                            ]
                                        }
                                    },
                                    {
                                        "@type": [
                                            "Telemetry",
                                            "SemanticType/Event"
                                        ],
                                        "displayName": {
                                            "en": "Processing Started"
                                        },
                                        "name": "evVideoStreamProcessingStarted",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": [
                                            "Telemetry",
                                            "SemanticType/Event"
                                        ],
                                        "displayName": {
                                            "en": "Processing Stopped"
                                        },
                                        "name": "evVideoStreamProcessingStopped",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": [
                                            "Telemetry",
                                            "SemanticType/Event"
                                        ],
                                        "displayName": {
                                            "en": "Device Restart"
                                        },
                                        "name": "evDeviceRestart",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Telemetry",
                                        "displayName": {
                                            "en": "Free Memory"
                                        },
                                        "name": "tlFreeMemory",
                                        "schema": "long"
                                    },
                                    {
                                        "@type": "Telemetry",
                                        "displayName": {
                                            "en": "Inference Rate"
                                        },
                                        "name": "tlInferenceRate",
                                        "schema": "double"
                                    },
                                    {
                                        "@type": "Telemetry",
                                        "displayName": {
                                            "en": "Primary Detection Class"
                                        },
                                        "name": "tlPrimaryDetectionClass",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Telemetry",
                                        "displayName": {
                                            "en": "Secondary Detection Class"
                                        },
                                        "name": "tlSecondaryDetectionClass",
                                        "schema": "string"
                                    },
                                    {
                                        "@type": "Command",
                                        "commandType": "asynchronous",
                                        "displayName": {
                                            "en": "Restart DeepStream"
                                        },
                                        "name": "cmRestartDeepStream"
                                    },
                                    {
                                        "@type": "Command",
                                        "commandType": "asynchronous",
                                        "request": {
                                            "@type": "SchemaField",
                                            "displayName": {
                                                "en": "Timeout"
                                            },
                                            "name": "cmpRestartDeviceTimeout",
                                            "schema": "integer"
                                        },
                                        "displayName": {
                                            "en": "Restart Device"
                                        },
                                        "name": "cmRestartDevice"
                                    }
                                ]
                            }
                        }
                    ],
                    "displayName": {
                        "en": "NVIDIAJetsonNano Module"
                    },
                    "contents": [],
                    "@context": [
                        "http://azureiot.com/v1/contexts/IoTModel.json"
                    ]
                }
            ]
        }
    ],
    "@context": [
        "http://azureiot.com/v1/contexts/IoTModel.json"
    ]
}

deployment.manifest.json

JSON
The IoT Edge deployment manifest
{
  "modulesContent": {
    "$edgeAgent": {
      "properties.desired": {
        "schemaVersion": "1.0",
        "runtime": {
          "type": "docker",
          "settings": {
            "minDockerVersion": "v1.25",
            "loggingOptions": "",
            "registryCredentials": {}
          }
        },
        "systemModules": {
          "edgeAgent": {
            "type": "docker",
            "settings": {
              "image": "mcr.microsoft.com/azureiotedge-agent:1.0",
              "createOptions": "{}"
            }
          },
          "edgeHub": {
            "type": "docker",
            "status": "running",
            "restartPolicy": "always",
            "settings": {
              "image": "mcr.microsoft.com/azureiotedge-hub:1.0",
              "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"5671/tcp\":[{\"HostPort\":\"5671\"}],\"8883/tcp\":[{\"HostPort\":\"8883\"}],\"443/tcp\":[{\"HostPort\":\"443\"}]}}}"
            }
          }
        },
        "modules": {
          "NVIDIADeepStreamSDK": {
            "version": "1.0",
            "type": "docker",
            "status": "running",
            "restartPolicy": "always",
            "settings": {
              "image": "marketplace.azurecr.io/nvidia/deepstream-iot2-l4t:latest",
              "createOptions": "{\"Entrypoint\":[\"/usr/bin/deepstream-test5-app\",\"-c\",\"DSConfig.txt\"],\"HostConfig\":{\"runtime\":\"nvidia\",\"NetworkMode\":\"host\",\"Binds\":[\"/data/misc/storage:/data/misc/storage\",\"/tmp/argus_socket:/tmp/argus_socket\",\"/tmp/.X11-unix/:/tmp/.X11-unix/\"],\"IpcMode\":\"host\",\"Devices\":[{\"PathOnHost\":\"/dev/video0\",\"PathInContainer\":\"/dev/video0\",\"CgroupPermissions\":\"rwm\"}]},\"NetworkingConfig\":{\"EndpointsConfig\":{\"host\":{}}},\"WorkingDir\":\"/data/misc/storage\"}"
            },
            "env": {
              "DISPLAY": {
                "value": ":0"
              }
            }
          },
          "NVIDIANanoIotCModule": {
            "settings": {
              "image": "toolboc/nvidia-nano-iotc-module",
              "createOptions": "{\"HostConfig\":{\"PortBindings\":{\"9014/tcp\":[{\"HostPort\":\"9014\"}]},\"Binds\":[\"/data/misc:/data/misc\",\"/run/systemd:/run/systemd\",\"/var/run/docker.sock:/var/run/docker.sock\"]}}"
            },
            "type": "docker",
            "env": {
              "DEBUG_TELEMETRY": {
                "value": "1"
              }
            },
            "status": "running",
            "restartPolicy": "always",
            "version": "1.0"
          }
        }
      }
    },
    "$edgeHub": {
      "properties.desired": {
        "schemaVersion": "1.0",
        "routes": {
          "DeepstreamToFilter": "FROM /messages/modules/NVIDIADeepStreamSDK/outputs/* INTO BrokeredEndpoint(\"/modules/NVIDIANanoIotCModule/inputs/dsmessages\")",
          "filterToIoTHub": "FROM /messages/* INTO $upstream"
        },
        "storeAndForwardConfiguration": {
          "timeToLiveSecs": 7200
        }
      }
    }
  }
}

Credits

Paul DeCarlo

Paul DeCarlo

25 projects • 203 followers
Paul DeCarlo is a prof @ #Bauer college of Business @UniversityOfHouston and Software Engineer @Microsoft focused on IoT, Cloud, and Mobile.

Comments