Add feature to detect anomaly

src/main.py

@@ -5,6 +5,7 @@ time python src/_detector.py --input ~/Desktop/5min.mp4 -l
 
 """
 # import the necessary packages
+import sys
 import time
 import argparse
 import pprint
@@ -18,6 +19,7 @@ from utils import (
     get_heading,
     get_avg_heading,
     OBJ_LEAVING_COND,
+    DONTCARE,
 )
 
 pp = pprint.PrettyPrinter(indent=2)
@@ -73,6 +75,7 @@ print("[INFO] loading YOLO from disk...")
 net = cv2.dnn.readNetFromDarknet(configPath, weightsPath)
 ln = net.getLayerNames()
 ln = [ln[i[0] - 1] for i in net.getUnconnectedOutLayers()]
+start = end = 0
 
 
 def detect_stuffs(_frame):
@@ -163,6 +166,11 @@ except:
 _frame_count = 0
 tracker_counter = 1
 
+
+for xx in range(3996):
+    (grabbed, frame) = vs.read()
+    _frame_count += 1
+
 # loop over frames from the video file stream
 while True:
     # read the next frame from the file
@@ -186,119 +194,169 @@ while True:
     # print("boxes", boxes)
     print(f"[{_frame_count:08d}] ::")
 
+    # quit if unable to read the video file
+    if not success:
+        print("Failed to read video", grabbed, frame)
+
+        cv_trackers.clear()
+        cv_trackers = cv2.MultiTracker_create()
+        # print("  INIT AGAIN cv_tracker", type(cv_trackers), cv_trackers)
+        # get rid of them in trackers too
+        trackers = [_ for _ in trackers if _["id"] not in ut_ids]
+        # add tracker again
+        for _trckr in trackers:
+            __tkr = OPENCV_OBJECT_TRACKERS["csrt"]()
+            cv_trackers.add(__tkr, frame, tuple(_trckr["curr_position"]))
+        (success, boxes) = cv_trackers.update(frame)
+        # print('SUCCESS? ', success)
+
     untracking = []
     # loop over the bounding boxes and draw then on the frame
-    if success:
-        obj_cnt = len(boxes)
-        # print(f"obj_cnt: ", obj_cnt, ' | tracker #', len(trackers))
-        for idx in range(obj_cnt):
-            GONE = False
-
-            box = boxes[idx]
-            (x, y, w, h) = [int(v) for v in box]
-            _last_pos = trackers[idx]["curr_position"]
-            (_x, _y, _w, _h) = _last_pos
-            curr_distance = box_distance(box, _last_pos)
-            last_distance = trackers[idx]["distance"]
-            trackers[idx]["distance"] = curr_distance
-            trackers[idx]["curr_position"] = box  # [int(v) for v in box]
-
-            _heading = get_heading(_x, _y, x, y)
-
-            STILL_DIST_PX = 2
-            if last_distance < STILL_DIST_PX and curr_distance < STILL_DIST_PX:
-                trackers[idx]["still"] += 1
-            else:
-                trackers[idx]["still"] = 0
-
-            if trackers[idx]["still"] == 0:
-                trackers[idx]["heading"] = [_heading] + trackers[idx]["heading"]
-
-            trackers[idx]["heading"] = trackers[idx]["heading"][:20]
-
-            if trackers[idx]["still"] > 30 or x < 5 or x > 1250:
-                untracking.append(trackers[idx])
-
-            tk = trackers[idx]
-            # check if it's hit the first
-            avg_heading = get_avg_heading(trackers[idx]["heading"])
-            for lvng_cnd in OBJ_LEAVING_COND:
-                _origin = tk["origin"]["id"]
-                cond = dict(lvng_cnd)
-                if cond["origin_id"] != _origin or avg_heading != cond["heading"]:
-                    continue
-                REACH_condition = False
-                if cond["x"] != -1:
-                    if (cond["heading"] in "NE" and x + w > cond["x"]) or (
-                        cond["heading"] in "SW" and x < cond["x"]
-                    ):
-                        # print("REACH condition: X", x, cond["x"])
-                        REACH_condition = True
 
-                elif cond["y"] != -1:
+    obj_cnt = len(boxes)
-                    # Don't have one yet
+    # print(f"obj_cnt: ", obj_cnt, ' | tracker #', len(trackers))
-                    if (cond["heading"] in "NE" and x + w > cond["y"]) or (
+    for idx in range(obj_cnt):
-                        cond["heading"] in "SW" and x < cond["y"]
+        GONE = False
-                    ):
-                        # print("REACH condition: Y")
-                        REACH_condition = True
-                if not REACH_condition:
-                    continue
 
-                # print("MATCH COND")
+        box = boxes[idx]
-                # pp.pprint(cond)
+        (x, y, w, h) = [int(v) for v in box]
-                # TODO: should be a loop here if next has > 1
+
-                _nid = f"id_{cond['next_area'][0]}"
+        # check if size is growing? if more than twice then, untrack it
-                # print(f"#{tk['id']} origin:#{_origin} to#{_nid}", end="")
+        ow, oh = trackers[idx]["size"]
-                print(f"         {tk['id']} LEFT from {_origin} -> {_nid}")
+        if ow / w > 2 or oh / h > 2:
-                if _nid not in W4A:
+            print(f"      {tk['id']} GROW_TOO_BIG")
-                    # print(f">>add AREA {_nid} to W4A", end="")
+            trackers[idx]["status"] = "grow-too-big"
-                    W4A[_nid] = {"objects": []}
+            untracking.append(trackers[idx])
-                # put this object to W4A for next area if doesn't exist
+            continue
-                has_this = [_ for _ in W4A[_nid]["objects"] if _[0]["id"] == tk["id"]]
+
-                if not has_this:
+        _last_pos = trackers[idx]["curr_position"]
-                    GONE = True
+        (_x, _y, _w, _h) = _last_pos
-                    # unit in frame
+        curr_distance = box_distance(box, _last_pos)
-                    _expected_frame = _frame_count + cond["duration_to_next"]
+        last_distance = trackers[idx]["distance"]
-                    W4A[_nid]["objects"].append((tk, _frame_count, _expected_frame))
+        trackers[idx]["distance"] = curr_distance
-                    untracking.append(tk)
+        trackers[idx]["curr_position"] = box  # [int(v) for v in box]
-                    # print(f'>>GONE - W#{len(W4A[_nid]["objects"])}')
+
-                    # print(f'     {_nid} objs: ')
+        _heading = get_heading(_x, _y, x, y)
-                    # pp.pprint(W4A[_nid]["objects"])
+        tk = trackers[idx]
-                    # print(f'     {_nid} untracking: ', [_['id'] for _ in untracking])
+
-
+        STILL_DIST_PX = 1
-            if GONE:
+        if last_distance < STILL_DIST_PX and curr_distance < STILL_DIST_PX:
+            trackers[idx]["still"] += 1
+        else:
+            trackers[idx]["still"] = 0
+
+        if trackers[idx]["still"] == 0:
+            trackers[idx]["heading"] = [_heading] + trackers[idx]["heading"]
+
+        # trackers[idx]["heading"] = trackers[idx]["heading"][:20]
+
+        if trackers[idx]["still"] > 30 or x < 5 or x > 1250:
+            untracking.append(trackers[idx])
+
+        if trackers[idx]["still"] > 10 and len(trackers[idx]["heading"]) < 5:
+            print(f"      {tk['id']} LEFT - short-life")
+            trackers[idx]["status"] = "short-life"
+            untracking.append(trackers[idx])
+
+        # check if it's hit the first
+        avg_heading = get_avg_heading(trackers[idx]["heading"])
+
+        h_count = len(tk["heading"])
+        if 5 < h_count and h_count < 10:
+            # enough to validate, but never an established one
+            DONTCARE_IDS = [_[0][1] for _ in DONTCARE]
+            if tk["origin"]["id"] in DONTCARE_IDS:
+                dc_id = DONTCARE_IDS.index(tk["origin"]["id"])
+                dc_dict = dict(DONTCARE[dc_id])
+                print(DONTCARE_IDS, tk["origin"]["id"], dc_id, dc_dict["heading"])
+                if avg_heading in dc_dict["heading"]:
+                    print(f"      {tk['id']} DONT-CARE condition")
+                    trackers[idx]["status"] = "dont-care"
+                    untracking.append(trackers[idx])
+
+        for lvng_cnd in OBJ_LEAVING_COND:
+            _origin = tk["origin"]["id"]
+            cond = dict(lvng_cnd)
+            if cond["origin_id"] != _origin or avg_heading != cond["heading"]:
+                continue
+            REACH_condition = False
+            if cond["x"] != -1:
+                # print("REACH condition: X", x, cond["x"])
+                if cond["heading"] in "NE":
+                    _cond = cond["x"](y + w) if callable(cond["x"]) else cond["x"]
+                    if x + w > _cond:
+                        REACH_condition = True
+                elif cond["heading"] in "SW":
+                    _cond = cond["x"](y + h) if callable(cond["x"]) else cond["x"]
+                    if x < _cond:
+                        REACH_condition = True
+            elif cond["y"] != -1:
+                _cond = cond["y"](x + h) if callable(cond["y"]) else cond["y"]
+                # Don't have one yet
+                if cond["heading"] in "SW" and y + h > _cond:
+                    print("REACH condition: Y", y + h, _cond)
+                    REACH_condition = True
+                # pass
+            if not REACH_condition:
                 continue
 
-            # print(
+            # print("MATCH COND")
-            #     f"[{tk['id']}-{tk['type']}] (x,y)=({x},{y})"
+            # pp.pprint(cond)
-            #     f" | still #{tk['still']} | distance:  "
+            _nid = f"id_{cond['next_area'][0]}"
-            #     f"{last_distance:.3f} -> {curr_distance:.3f}",
+            # print(f"#{tk['id']} origin:#{_origin} to#{_nid}", end="")
-            #     end="",
+            print(f"         {tk['id']} LEFT from {_origin} -> {_nid}")
-            # )
+            if _nid not in W4A:
-            # _htxt = ",".join(trackers[idx]["heading"])
+                # print(f">>add AREA {_nid} to W4A", end="")
-            # print(f" ------  heading: {_htxt}", end='')
+                W4A[_nid] = {"objects": []}
-            # print(f" ------  avg heading: {avg_heading}", end="")
+            # put this object to W4A for next area if doesn't exist
-            # print("")
+            has_this = [_ for _ in W4A[_nid]["objects"] if _[0]["id"] == tk["id"]]
-
+            if not has_this:
-            # DRAW on FRAME
+                GONE = True
-            color = [int(c) for c in COLORS[0]]
+                # unit in frame
-            cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
+                _expected_frame = _frame_count + cond["duration_to_next"]
-            cv2.putText(
+                W4A[_nid]["objects"].append((tk, _frame_count, _expected_frame))
-                frame,
+                untracking.append(tk)
-                f"{tk['id']} - {tk['type']}",
+                # print(f'>>GONE - W#{len(W4A[_nid]["objects"])}')
-                (x, y - 5),
+                # print(f'     {_nid} objs: ')
-                cv2.FONT_HERSHEY_SIMPLEX,
+                # pp.pprint(W4A[_nid]["objects"])
-                0.5,
+                # print(f'     {_nid} untracking: ', [_['id'] for _ in untracking])
-                color,
+
-                2,
+        if GONE:
-            )
+            continue
+
+        print(
+            f"[{tk['id']}{tk['type'][:1]}] o#{tk['origin']['id']} (x,y)=({x},{y},{w},{h})"
+            f" | still #{tk['still']} | distance:  "
+            f"{last_distance:.3f} -> {curr_distance:.3f}",
+            end="",
+        )
+        _htxt = ",".join(trackers[idx]["heading"][:18])
+        print(f"|heading:{_htxt}", end="")
+        print(f"|avg:{avg_heading}", end="")
+        print("")
+
+        # DRAW on FRAME
+        color = [int(c) for c in COLORS[0]]
+        cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
+        cv2.putText(
+            frame,
+            f"{tk['id']} - {tk['type']}",
+            (x, y - 5),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            0.5,
+            color,
+            2,
+        )
 
     # Cleanup Tracker
     if untracking:
         ut_ids = [_["id"] for _ in untracking]  # untracking ids
+        # cv_trackers.clear()
         cv_trackers.clear()
+        cv_trackers = None
+        time.sleep(1)
         cv_trackers = cv2.MultiTracker_create()
+        # print("  cv_tracker", type(cv_trackers), cv_trackers)
         # get rid of them in trackers too
         trackers = [_ for _ in trackers if _["id"] not in ut_ids]
         # add tracker again
@@ -313,6 +371,12 @@ while True:
     if _frame_count % 15 == 1:
         idxs, boxes, confidences, classIDs, start, end = detect_stuffs(frame)
         # loop over the indexes we are keeping
+        if len(idxs) == 0:
+            print("CAN NOT DETECT STUFFS??")
+            continue
+        if isinstance(idxs, tuple):
+            pp.pprint(idxs)
+
         for i in idxs.flatten():
             # extract the bounding box coordinates
             (x, y) = (boxes[i][0], boxes[i][1])
@@ -323,6 +387,10 @@ while True:
             if not found_at:
                 continue
 
+            # only do for person/wheelchair
+            if _class not in ["person", "wheelchair", "bicycle"]:
+                continue
+
             # (1) check whether it's the same object as one in trackers
             is_same = False
             for t in trackers:
@@ -360,7 +428,9 @@ while True:
                 t = {
                     "id": tracker_counter if gid is None else gid,
                     "type": _class,
+                    "status": "",
                     "curr_position": bbox,
+                    "size": (w, h),
                     "heading": [],
                     "origin": found_at,
                     "distance": -1,
@@ -383,8 +453,9 @@ while True:
     # CLEANUP
     cleanups = []
     for k in W4A:
+        print(f"   CLEANUP[area {k}] ", end="")
         for _o in W4A[k]["objects"]:
-            print(f"   CLEANUP: {_o[0]['id']} - {_o[1]} - {_o[2]}", end="\r\n")
+            print(f"   {_o[0]['id']} - {_o[1]} - {_o[2]}", end="")
             if _o[2] < _frame_count:
                 continue
             wf = _o[2] - _o[1]
@@ -396,6 +467,7 @@ while True:
                 W4A[k]["objects"] = [
                     _ for _ in W4A[_area_id]["objects"] if _[0]["id"] != _o[0]["id"]
                 ]
+        print(f" | tracked #{len(trackers)}")
     for obj, _s, _e in cleanups:
         print(
             f"         {obj['id']} CLEANED UP should found at {_e - _frame_count} ago"

src/utils.py

@@ -4,6 +4,7 @@ from collections import namedtuple
 
 Rectangle = namedtuple("Rectangle", "xmin ymin xmax ymax")
 
+
 def area(a, b):
     # returns None if rectangles don't intersect
     dx = min(a.xmax, b.xmax) - max(a.xmin, b.xmin)
@@ -28,13 +29,13 @@ AREAS = [
     ],
     [
         ("id", 3),
-        ("area", ((38, 340), (99, 482))),
+        ("area", ((38, 340), (130, 482))),
         ("target", ["person", "wheelchair"]),
         ("next", [5]),
     ],
     [
         ("id", 4),
-        ("area", ((106, 310), (164, 461))),
+        ("area", ((96, 310), (145, 461))),
         ("target", ["person", "wheelchair"]),
     ],
     [
@@ -55,9 +56,15 @@ AREAS = [
         ("target", ["person", "wheelchair", "bicycle"]),
         ("next", [4]),
     ],
+    [
+        ("id", 11),
+        ("area", ((875, 179), (945, 278))),
+        ("target", ["person", "wheelchair"]),
+        ("next", [9]),
+    ],
     [
         ("id", 8),
-        ("area", ((901, 223), (956, 342))),
+        ("area", ((901, 223), (976, 342))),
         ("target", ["person", "wheelchair"]),
         ("next", [7, 5]),
     ],
@@ -73,12 +80,6 @@ AREAS = [
         ("target", ["person", "wheelchair"]),
         ("next", [9]),
     ],
-    [
-        ("id", 11),
-        ("area", ((875, 179), (915, 278))),
-        ("target", ["person", "wheelchair"]),
-        ("next", [9]),
-    ],
 ]
 
 OBJ_LEAVING_COND = [
@@ -93,19 +94,19 @@ OBJ_LEAVING_COND = [
     [
         ("origin_id", 3),
         ("heading", "N"),
-        ("x", 175),
+        ("x", lambda y: (y - 80.5) / 2.19),
-        ("y", -1),
-        ("next_area", [5]),
-        ("duration_to_next", 30 * 4),
-    ],
-    [
-        ("origin_id", 4),
-        ("heading", "N"),
-        ("x", 175),
         ("y", -1),
         ("next_area", [5]),
         ("duration_to_next", 30 * 4),
     ],
+    # [
+    #     ("origin_id", 4),
+    #     ("heading", "N"),
+    #     ("x", 175),
+    #     ("y", -1),
+    #     ("next_area", [5]),
+    #     ("duration_to_next", 30 * 4),
+    # ],
     [
         ("origin_id", 5),
         ("heading", "N"),
@@ -114,6 +115,14 @@ OBJ_LEAVING_COND = [
         ("next_area", [11]),
         ("duration_to_next", 30 * 3),
     ],
+    [
+        ("origin_id", 7),
+        ("heading", "S"),
+        ("x", -1),
+        ("y", lambda x: 0.4 * x + 152),
+        ("next_area", [4]),
+        ("duration_to_next", 30 * 4),
+    ],
     # [
     #     ('origin_id', ),
     #     ('heading', ''),
@@ -124,10 +133,18 @@ OBJ_LEAVING_COND = [
     # ],
 ]
 
+DONTCARE = (
+    (("origin_id", 5), ("heading", ["S", "W"])),
+    (("origin_id", 4), ("heading", ["S", "W"])),
+    (("origin_id", 7), ("heading", ["N", "E"])),
+    (("origin_id", 10), ("heading", ["N", "E"])),
+)
+
 
 def get_avg_heading(headings):
     latest = headings[:15]
-    chars = collections.Counter("".join(latest)).most_common(10)
+    _h = "".join(latest).replace('W', '').replace('E', '')
+    chars = collections.Counter(_h).most_common(10)
     if chars:
         return chars[0][0]
     return None
@@ -189,7 +206,7 @@ def is_it_the_same_obj(x1, y1, w1, h1, i1, j1, w2, h2, **kwargs):
     #     print(" :: check against id:", _id, end="")
 
     # if first coords are pretty much the same spot, then they are the same
-    if abs(x1 - i1)/x1 < 0.05 and abs(y1 - j1)/y1 < 0.05:
+    if abs(x1 - i1) / x1 < 0.05 and abs(y1 - j1) / y1 < 0.05:
         # print(" same 1st coords")
         return True