Merge commit 'b89052eeead1e369f6c71a0ad2aaf79b4b274141' into feature/…

…machine-learning/cicd

machine_learning/src/face_detect_model/config.yaml

@@ -16,17 +16,17 @@ face_detect:
   clip_size: { width: 100, height: 100 }
 dataset:
   augmentation:
-    num_variations: 10
+    num_variations: 30
   ratio:
     train: 0.8
     valid: 0.1
     test: 0.1
 train:
-  epoch: 3
-  learning_rate: 0.001
+  epoch: 5
+  learning_rate: 0.0005
   train_batch_size: 1
   valid_batch_size: 1
   test_batch_size: 1
-  validate_interval: 3
+  validate_interval: 6
   save_model_dir: "src/face_detect_model/pickle/mock_model/"
   test_model_path: "src/face_detect_model/pickle/mock_model/best_model.pth"

machine_learning/src/face_detect_model/data/faceDetectDataset.py

@@ -2,13 +2,13 @@
 
 import torch
 from PIL import Image
-from torchvision import transforms
 
 from face_detect_model.util import (
     load_image_from_remote,
 )
 
 from face_detect_model.gcp_util import get_bucket, get_blobs
+from face_detect_model.util import get_augment_transform, get_default_transforms
 
 
 # (child_id, image)のタプルを返す
@@ -18,7 +18,7 @@ def __init__(self, args, config, client):
         self.args = args
         self.config = config
 
-        default_transform = self.get_default_transforms()
+        default_transform = get_default_transforms()
 
         bucket_name = os.environ.get("BUCKET_NAME")
         bucket = get_bucket(client, bucket_name)
@@ -62,41 +62,9 @@ def _add_label(self, label: str):
         self.name_label_dict[label] = self.label_num
         self.label_num += 1
 
-    def get_default_transforms(self):
-        return transforms.Compose(
-            [
-                transforms.ToTensor(),
-                transforms.Normalize(
-                    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
-                ),
-            ]
-        )
-
-    def get_augment_transform(self):
-        return transforms.Compose(
-            [
-                transforms.RandomCrop((100, 100)),
-                transforms.RandomHorizontalFlip(p=0.5),
-                transforms.RandomVerticalFlip(p=0.5),
-                transforms.RandomApply([transforms.RandomRotation(degrees=180)], p=0.5),
-                transforms.RandomApply(
-                    [
-                        transforms.RandomAffine(
-                            degrees=0, translate=(0.1, 0.1), scale=(0.8, 1.2)
-                        )
-                    ],
-                    p=0.5,
-                ),
-                transforms.ToTensor(),
-                transforms.Normalize(
-                    mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
-                ),
-            ]
-        )
-
     def augment_image(self, image, num_variations=100):
         # ランダムな変換を適用するための拡張設定を強化
-        transformations = self.get_augment_transform()
+        transformations = get_augment_transform()
         augmented_images = []
         for _ in range(num_variations):
             augmented_image = transformations(image)

machine_learning/src/face_detect_model/main.py

@@ -37,11 +37,6 @@ def TrainAndTest(args: argparse.Namespace, config: dict):
         ],
     )
 
-    # modelの読み込み
-    logger.info("model Initializing")
-    num_classes = dataset.get_label_num()
-    image_shape = dataset.get_image_shape()
-
     idx_to_label = dataset.get_idx_label_dict()
     if args.mode == "train":
         bucket = get_bucket(
@@ -55,6 +50,11 @@ def TrainAndTest(args: argparse.Namespace, config: dict):
         )
         logger.info("idx_to_label is saved")
 
+    # modelの読み込み
+    logger.info("model Initializing")
+    num_classes = dataset.get_label_num()
+    image_shape = dataset.get_image_shape()
+
     face_detect_model = FaceDetectModel(
         config, num_classes=num_classes, input_dim=image_shape
     )
@@ -100,7 +100,7 @@ def main(args: argparse.Namespace):
         nargs="*",
     )
     parser.add_argument("--bus_id", type=str)
-    parser.add_argument("--bus_type", type=str)
+    parser.add_argument("--bus_type", type=int)
 
     args = parser.parse_args()
 

machine_learning/src/face_detect_model/pred.py

@@ -1,4 +1,161 @@
-def main():
-    gray = cv2.cvtColor(captureBuffer, cv2.COLOR_BGR2GRAY)
-    pred_child_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+import os
+import argparse
+from dotenv import load_dotenv
+import yaml
+import torch
+
+load_dotenv("secrets/.env")
+
+from face_detect_model.util import (
+    load_image_from_binary,
+    switch_to_bus_type,
+    load_pickle_to_gcs,
+    get_default_transforms,
+    logger,
+)
+from face_detect_model.gcp_util import init_client, get_bucket
+from face_detect_model.model.faceDetectModel import FaceDetectModel
+from face_detect_model.DetectFaceAndClip.detectFaceUtil import (
+    detect_face,
+    load_cascade,
+    clip_and_resize_face,
+)
+
+
+def load_model(model_pickle, config, num_classes, input_shape) -> FaceDetectModel:
+    try:
+        model = FaceDetectModel(config, num_classes, input_shape)
+        model.load_state_dict(model_pickle)
+    except Exception as e:
+        raise ValueError(f"Failed to load model due to an error: {e}")
+    return model
+
+
+def get_clipped_face(image, faces, image_size):
+    clipped_faces = []
+    for face in faces:
+        clipped_face = clip_and_resize_face(face, image, image_size)
+        clipped_faces.append(clipped_face)
+    return clipped_faces
+
+
+def detect_face_and_clip_from_image(image, config):
+    face_cascade_path = config["face_detect"]["cascade_path"]
+    image_size = (
+        config["face_detect"]["clip_size"]["height"],
+        config["face_detect"]["clip_size"]["width"],
+    )
+
+    # Haar Cascadeの読み込み
+    face_cascade = load_cascade(face_cascade_path)
+
+    faces = detect_face(
+        image,
+        face_cascade,
+        scaleFactor=config["face_detect"]["scale_factor"],
+        minNeighbors=config["face_detect"]["min_neighbors"],
+        minSize=(
+            config["face_detect"]["min_size"]["height"],
+            config["face_detect"]["min_size"]["width"],
+        ),
+    )
+
+    clipped_faces = get_clipped_face(image, faces, image_size)
+    return clipped_faces
+
+
+def convert_to_tensor_from_images(cliped_face_images):
+    transforms = get_default_transforms()
+    image_tensors = []
+    for cliped_face_image in cliped_face_images:
+        image_tensor = transforms(cliped_face_image)
+        image_tensors.append(image_tensor)
+    return image_tensors
+
+
+def get_cliped_faces_from_images(video_chunk_list, config):
+    all_faces = []
+    for video_chunk in video_chunk_list:
+        image = load_image_from_binary(video_chunk)
+        clipped_faces = detect_face_and_clip_from_image(image, config)
+        all_faces.extend(clipped_faces)
+    return all_faces
+
+
+def get_model_input_shape(tensor):
+    return tensor.shape
+
+
+def pred_child_from_tensor(model, input_image_tensor):
+    output = model(input_image_tensor.unsqueeze(0))
+    logger.info(f"Predicted present: {torch.nn.functional.softmax(output, dim=1)}")
+    pred = output.argmax(dim=1)
+    return pred
+
+
+def get_pred_child(model, input_image_tensors, idx_to_label_dict):
+    pred_child_ids = set()
+
+    # TODO: 複数枚をバッチで処理する
+    for input_image_tensor in input_image_tensors:
+        pred = pred_child_from_tensor(model, input_image_tensor)
+        child_id = idx_to_label_dict[pred.item()]
+        logger.info(f"Predicted child id: {child_id}")
+        pred_child_ids.add(child_id)
+
     return pred_child_ids
+
+
+def pred_child_from_images(args, config):
+    client = init_client()
+    bucket_name = os.environ.get("BUCKET_NAME_FOR_MODEL")
+    bucket = get_bucket(client, bucket_name)
+    idx_to_label_dict_bucket_path = f"{args.nursery_id}/{args.bus_id}/idx_to_label_{switch_to_bus_type(args.bus_type)}.pth"
+
+    idx_to_label_dict = load_pickle_to_gcs(bucket, idx_to_label_dict_bucket_path)
+    model_class_num = len(idx_to_label_dict)
+
+    clipped_faces = get_cliped_faces_from_images(args.video_chunk, config)
+    input_image_tensors = convert_to_tensor_from_images(clipped_faces)
+
+    model_bucket_path = (
+        f"{args.nursery_id}/{args.bus_id}/model_{switch_to_bus_type(args.bus_type)}.pth"
+    )
+    input_shape = get_model_input_shape(input_image_tensors[0])
+    model_pickle = load_pickle_to_gcs(bucket, model_bucket_path)
+    model = load_model(
+        model_pickle=model_pickle,
+        config=config,
+        num_classes=model_class_num,
+        input_shape=input_shape,
+    )
+    model.eval()
+
+    logger.info("Start predicting child id")
+    pred_child_ids = get_pred_child(model, input_image_tensors, idx_to_label_dict)
+    logger.info(f"Predicted child ids: {pred_child_ids}")
+    logger.info("Done")
+
+    return list(pred_child_ids)
+
+
+def main(args):
+    with open("src/face_detect_model/config.yaml", "r") as f:
+        config = yaml.safe_load(f)
+    pred_child_ids_list = pred_child_from_images(args, config)
+    return pred_child_ids_list
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--nursery_id", type=str, required=True)
+    parser.add_argument("--bus_id", type=str, required=True)
+    parser.add_argument("--bus_type", type=int, required=True)
+
+    args = parser.parse_args()
+    args.video_chunk = [
+        open("/Users/mizuki/Desktop/test.jpg", "rb").read(),
+        open("/Users/mizuki/Desktop/test1.jpg", "rb").read(),
+    ]
+
+    main(args)

machine_learning/src/face_detect_model/util.py

@@ -4,6 +4,7 @@
 import random
 import torch
 from google.cloud.storage import Bucket
+from torchvision import transforms
 import os
 import numpy as np
 
@@ -72,14 +73,62 @@ def load_image_from_remote(blobs: list):
     return images
 
 
+def load_image_from_binary(binary: bytes):
+    image_array = np.frombuffer(binary, dtype=np.uint8)
+    image = cv2.imdecode(image_array, cv2.IMREAD_COLOR)
+    image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)  # BGRからRGBに変換
+    if image is None:
+        raise ValueError("Can not load image from binary.")
+    return image
+
+
+def get_default_transforms():
+    return transforms.Compose(
+        [
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+
+
+def get_augment_transform():
+    return transforms.Compose(
+        [
+            transforms.RandomCrop((100, 100)),
+            transforms.RandomHorizontalFlip(p=0.5),
+            transforms.RandomVerticalFlip(p=0.5),
+            transforms.RandomApply([transforms.RandomRotation(degrees=180)], p=0.5),
+            transforms.RandomApply(
+                [
+                    transforms.RandomAffine(
+                        degrees=0, translate=(0.1, 0.1), scale=(0.8, 1.2)
+                    )
+                ],
+                p=0.5,
+            ),
+            transforms.ToTensor(),
+            transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
+        ]
+    )
+
+
 def _is_valid_file(file_name):
     VALID_EXTENSIONS = {".png"}
     return os.path.splitext(file_name)[1].lower() in VALID_EXTENSIONS
 
 
 def save_pickle_to_gcs(bucket: Bucket, upload_model_path: str, obj: object):
-    logger.info(f"Saving model to {upload_model_path}")
+    logger.info(f"Saving pickle to {upload_model_path}")
     blob = bucket.blob(upload_model_path)
     with blob.open("wb", ignore_flush=True) as f:
         torch.save(obj, f)
-    logger.info(f"Model saved to {upload_model_path}")
+    logger.info(f"pickle saved to {upload_model_path}")
+
+
+def load_pickle_to_gcs(bucket: Bucket, obj_path: str) -> object:
+    logger.info(f"Loading pickle from {obj_path}")
+    blob = bucket.blob(obj_path)
+    with blob.open("rb") as f:
+        model = torch.load(f)
+    logger.info(f"pickle loaded from {obj_path}")
+    return model

machine_learning/src/generated/machine_learning/v1/func_args.py

@@ -0,0 +1,27 @@
+from dataclasses import dataclass
+
+
+@dataclass
+class Pred_Args:
+    nursery_id: str
+    bus_id: str
+    bus_type: int
+    video_chunk: str
+    vehicle_event: str
+
+
+@dataclass
+class Train_Args:
+    nursery_id: str
+    child_ids: str
+    bus_id: str
+    bus_type: int
+    seed: int
+    mode: str
+
+
+@dataclass
+class FaceDetectAndClip_Args:
+    nursery_id: str
+    child_id: str
+    env: str