OpenCV + PyTorch + faissã§åŠç¿ãªãé¡èå¥ïŒ
ã¯ããã«
ãä¹ ãã¶ãã§ãïŒååã¯ãŒãããYOLOv3ãäœããšããç¡è¬ãªããšãããŠããŸãããã»ã»ã»ã
æ®æ®µã¯ ã§æè¡ããã°ãæžãããããŠããŸãïŒè¯ããã°èªãã§ãã ããïœïœïœïœïŒ
ååãCenterNetã®èšäºãæ¬¡ã¯æžãïŒãšèšã£ãŠããã®ã§ããããªãéããã®ãæžããŠããŸãã
ä»åã«è³ã£ãŠã¯ã¿ã€ãã«ã«ïŒãå
¥ã£ã¡ãã£ãŠãŸãããããäœãªãã§ãããã»ã»ã»ã
ããŠãä»åã¯AIã¢ãã«ã®åŠç¿ç¡ãã§é¡èªèšŒçãªäœããäœããªããïŒãšããããšã§ãã£ã¬ã³ãžããŠã¿ãŸããã
å
ã«çµæããèŠãããŸãïŒ

å
¥åããç»åãšããããããç»åã䞊ã¹ãŠããŸãã1çªç®ã¯å
¥åããç»åã®äººã«ãªã£ãŠããããšãå€ãã®ã§ãå²ãšããæããããªãã§ããããïŒ
ç»è§ãšèæ¯ãã¡ãããšåãããã°ãã£ãšããŸããããããªæ°ãããŸããïŒ
ã§ã¯ãã©ããã£ãŠ ãã¿ãäœã£ãã®ããã®ã¢ã«ãŽãªãºã ãäœã£ãã®ã説æããŠãããŸãïŒ
åŠç¿ãªãé¡èå¥ã®èãæ¹
ãŸãã¯ä»åäœæããã¢ã«ãŽãªãºã ã®æŠå¿µå³ã瀺ããŸãã

倧ãã䜿ã£ãŠããã¢ããªã±ãŒã·ã§ã³ã¯ïŒã€ã§ãã
- OpenCVïŒç»ååŠçãpython
- PyTorchïŒåŠç¿æžã¿ã¢ãã«ã®å©çšã»ç¹åŸŽéæœåº
- faissïŒãã¯ãã«æ€çŽ¢
åŠç¿ãããªããŠæžãæå€§ã®ãã€ã³ãã¯ãåŠç¿æžã¿ã¢ãã«ãæœåºããç¹åŸŽéãå©çšããããšã§ãã
åŠç¿æžã¿ã¢ãã«ã¯åºæ¬1000ã¯ã©ã¹åé¡ãã§ããã¢ãã«ã§ãã
ããªãã¡ã1000ã¯ã©ã¹ããã现ãã«åããããã»ã©ãäžå¯§ã«ç»åããç¹åŸŽãæœåºããŠãããšãèããããšãã§ããŸãã
ã€ãŸãåŠç¿æžã¿ã¢ãã«ãç¹åŸŽæœåºåšãšããŠå©çšããã°ãåããã®ã¯äŒŒãç¹åŸŽããéããã®ã¯éã£ãç¹åŸŽãæœåºãããããšã«ãªããšèããããŸãã
ä»åã¯ResNeXtãç¹åŸŽæœåºåšãšããŠå©çšããŠããŸãã
ç¹åŸŽãæœåºããã¯ããããé¡äŒŒåºŠãããã³ã°ã¯ã©ãããŸããããïŒããæžãããšãã§ããŸãããã¡ãã£ãšåŠçãéãããªãããã§ããã
ãããªæã«äŸ¿å©ãªã®ãfaissãæ€çŽ¢ã©ã€ãã©ãªã®æåŒ·ããã§ãããããfacebookãäœã£ãŠããã§ãããPyTorchãšããfaissãšãããããfacebookã«è¶³ãåããŠå¯ãããŸããã»ã»ã»ãïŒfacebookèªäœã¯ïŒå¹Žä»¥äžæŽæ°ããŠãªãã®ã§ããïŒ
ãã®faissãšããã©ã€ãã©ãªã¯ããã¯ãã«ãå ¥åãããšãã®é¢ä¿æ§ãè¯ããªã«ããŠä¿æããŠãããŸãã
ãã®åŸãæ€çŽ¢ããããã¯ãã«ãå ¥åãããšãé«éã§ïŒ0.1msãããïŒã§ãã®ã€ã³ããã¯ã¹ãè¿åŽããŸãã
ä»åã¯faissã«å±±ç°ååŒãããšã ãããšã·ããã®é¡ãResNeXtã§ãã¯ãã«åãããããã¶ã¡ãã§ã¿ãŸããïŒ
å®è£
ã§ã¯å®éã«å®è£ ããŠã¿ãŸãããïŒ
é¡èªè
ãŸãã¯é¡èªèããå®è£ ããŸãã
é¡ã®ã¿ãåãåãçç±ãšããŠãé¡ä»¥å€ã®é åãæã£ããŸãŸç¹åŸŽæœåºåšã«å
¥ããŠããŸããšãé¢ä¿ã®ãªãéšåã®ç¹åŸŽãæœåºããŠããŸãããšã«ãªããŸãã
ãããããšãåã人ã§ãæãéããšæœåºããããã¯ãã«ãå€ãã£ãŠããŸããå¥ç©å€å®ãããŸãã
ãããã£ãåé¡ã解決ããããã«ãé¡ã ããåãæããŠããŸããŸãïŒ
ç°å¢æºåã¯ãã®èšäºãåèã«ããŠãã ããïŒ
ãŸãã¯ã¹ã¯ãªãŒã³ã«æ ã£ãŠããå 容ããé¡èªèããŠã¿ãŸãããïŒ
import cv2
from PIL import ImageGrab ,Image
from cv2 import bilateralFilter
import numpy as np
import random
import copy
cascade = cv2.CascadeClassifier('./haarcascade_frontalface_default.xml')
def random_face_opencv():
while True:
image = ImageGrab.grab()
image = cv2.cvtColor(np.array(image),cv2.COLOR_RGB2BGR)
image = cv2.resize(image,(int(image.shape[1]/2),int(image.shape[0]/2)))
gray = cv2.cvtColor(copy.deepcopy(image),cv2.COLOR_BGR2GRAY)
faces = cascade.detectMultiScale(gray)
for (x, y, w, h) in faces:
image = cv2.rectangle(image,(x,y),(x+w,y+h) , color = (0,0,255),thickness=3)
cv2.imshow('test',image)
k = cv2.waitKey(1)&0xFF
if k == ord('q'):
break
if __name__ == "__main__":
random_face_opencv()
ç§ã¯Googleã§ãããããšèª¿ã¹ãŠæ€åºããŠã¿ãŸããïŒããªããããã«æ€åºã§ããŠãŸããïŒ
å€åããã§ååã§ãããïŒ
ããŒã¿ã®æºå
ç¶ããŠé¡èªèããã人ã®åçãåéããŸãã
ç§ã¯Googleæ€çŽ¢ã§ãå±±ç°ååŒããã ãããšã·ããšå
¥åããŠãåããã ãåããŸããã
ãã£ãšæ¥œãªæ¹æ³ã§ååŸã§ããæ¹ã¯äººåã§è¡ãããã³ãŒãã§è¡ãã»ãããããšæããŸãïœïœïœ
åéããç»åã以äžã®ããã«ä¿åããŸãããïŒ

é¡é åã®ã«ããã£ã³ã°
ã§ã¯ïŒãåŸ ã¡ããïŒé¡ã«ããã£ã³ã°ã®ãæéã§ãïŒ
ãŸãã¯ä¿åå ãäœããŸãããïŒç§ã¯ãcutããšãããã©ã«ããäœæããŸãããã奜ããªããã«æ§ç¯ããŠãã ããïŒ
ã§ã¯ã«ããããŸãïŒ
import cv2
import copy
# èå¥ãããç»ååã ãæºåããŠãã ããïŒ
muro_list = glob.glob(os.path.join("./face/muro/","*"))
yamada_list = glob.glob(os.path.join("./face/yamada/","*"))
cascade = cv2.CascadeClassifier('./haarcascade_frontalface_default.xml')
# äžã«åãããŠæžãæããŠãã ããïŒïŒ
data_list = [muro_list , yamada_list]
# ã¯ã©ã¹åãèšèŒããŠãã ããïŒ
name_list = ["muro" , "yamada"]
for name , l in zip(name_list,data_list):
count = 1
for idx , path in enumerate(l):
image =cv2.imread(path)
if image is None:
continue
gray = cv2.cvtColor(copy.deepcopy(image),cv2.COLOR_BGR2GRAY)
faces = cascade.detectMultiScale(gray)
for (x, y, w, h) in faces:
cv2.imwrite(f"./cut/{name}_{count}.png",image[y:y+h,x:x+w])
count += 1
çµæã¯ã»ã»ã»ã»

ããŒãŒãŒãå§ãåããã¡ããã¡ãã誀æ€åºãæ··ãã£ãŠãŸãããæã§åé€ãã¡ãããŸãããïŒ

ãããã«ãªããŸãããïŒ
åãå§ã匷ãã»ã»ã»ããªããããã女ã®åãšãã«ããªãã£ãã®ãã»ã»ã»ã
奜ããªã¢ã€ãã«ããã£ã©ã¯ã¿ãŒã®é¡ã顿ã«ããæ¹ã¯ããã¶ããã®æ®µéã§æºè¶³ããŠãããããªãã§ããããïŒ ãã¯ãäœæ¯ã§ãè¡ãããã§ãã
ç¹åŸŽæœåºåšã«ããç¹åŸŽæœåº
ç¶ããŠç¹åŸŽæœåºåšãçšããŠãå ã»ã©ã®é¡ç»åããç¹åŸŽãæœåºããŸãã
æ©éæºåããŸãããïŒ
import torch
import torchvision.models as models
import torch.nn as nn
model = models.resnext50_32x4d()
vec_model = nn.Sequential(
*(
list(model.children())[:-1]
)
)
宿ã§ãïŒããã£ãæ°è¡ã§æºåã§ãã¡ãããã§ããã³ã£ããã»ã»ã»ã
ã§ã¯ç¹åŸŽæœåºãã¡ãããŸãããïœïœ
from PIL import Image
from torchvision import transforms
import torch
# åé¡ç»åã®åŒã³åºããè¡ã£ãŠããŸããyamada_ãmuro_ãåèªã®ååã§å€æŽããŠãã ããã
yamada_face = glob.glob(os.path.join("./cut/","yamada_*"))
muro_face = glob.glob(os.path.join("./cut/","muro_*"))
len(yamada_face),len(muro_face)
# faissã«ä¿åããç»åãšæ€èšŒã«å©çšããç»åãæ±ºããŠããŸãã
# trainãfaissçšãvalãæ€èšŒçšã§ãã
yamada_train = yamada_face[:int(len(yamada_face)*0.5)]
yamada_val = yamada_face[int(len(yamada_face)*0.5):]
muro_train = muro_face[:int(len(muro_face)*0.5)]
muro_val = muro_face[int(len(muro_face)*0.5):]
vec_list = []
face_data = [yamada_train , muro_train]
preprocess = transforms.Compose([
transforms.Resize((224,224)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
for data in face_data:
for path in data:
input_image = Image.open(path)
input_image = input_image.convert("RGB")
with torch.no_grad():
img = preprocess(input_image)
vec = vec_model(img.unsqueeze(0))
vec_list.append(vec.detach().cpu().numpy()[0,:,0,0])
faissã«ç»é²ãã
æœåºãããã¯ãã«ãã¡ãfaissã«ç»é²ããŸãã
faissã¯
pip install faiss-cpu
ã§ã€ã³ã¹ããŒã«ã§ããã®ã§ããŸã ã®æ¹ã¯å ¥ãã¡ãã£ãŠãã ããïŒ
ã§ã¯ãfaissã«ãã¯ãã«ãç»é²ããŠãããŸãããïŒ
import faiss
import numpy as np
# å±±ç°ååŒã®ç»åãã ãããšã·ã®ç»åããããããã«ããŒã¿ãæ®ã
ans = ["yamada"]*len(yamada_train) + ["muro"]*len(muro_list)
# ç»é²
index = faiss.IndexFlatL2(vec_list[0].shape[0])
index.add(np.array(vec_list, dtype=np.float32) )
print(index.ntotal)
ã§ã¯ãç»é²ããå å®¹ã§æ€çŽ¢ããŠã¿ãŸãããã
# 詊ãããç»åãéžæ
search_id = 0
print(f"ãã®ç»åã¯{ans[search_id]}ã§ã")
k = 3
D, I = index.search(np.array(vec_list[search_id:search_id+1]), k)
print(f"çµæã¯")
for n,i in enumerate(I[0]):
print(f"No{n+1}ã¯{ans[i]}")
#ãã®ç»åã¯yamadaã§ã
#çµæã¯
#No0ã¯yamada
#No1ã¯muro
#No2ã¯yamada
ã©ãã§ããããïŒ
é äœãé«ãã»ããé¡äŒŒããŠããŸãïŒãªãããããããã©æ¥œããïŒïŒçæ°ïŒ
å®éã«èå¥ããŠã¿ã
ã§ã¯ïŒãåŸ
ã¡ããã®èå¥ã¿ã€ã ã§ãïŒ
ããŠãã©ããªæãã«ãªãã§ããããïœïœïœïœïŒ
from PIL import Image
from torchvision import transforms
import torch
vec_list = []
face_data = [yamada_val , muro_val]
preprocess = transforms.Compose([
transforms.Resize((224,224)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
for data in face_data:
for path in data:
input_image = Image.open(path)
input_image = input_image.convert("RGB")
with torch.no_grad():
img = preprocess(input_image)
vec = vec_model(img.unsqueeze(0))
vec_list.append(vec.detach().cpu().numpy()[0,:,0,0])
import matplotlib.pyplot as plt
%matplotlib inline
def create_fig(id,path_list):
fig = plt.Figure(figsize=(30,10))
for i,path in enumerate(path_list):
plt.subplot(1,4,i+1)
img = Image.open(path)
plt.imshow(img)
plt.title("input "if i == 0 else f"No {i}")
plt.axis('off')
plt.savefig(f"{id}.png")
train_path_list = yamada_train + muro_train
val_path_list = yamada_val + muro_val
k = 3 # we want to see 4 nearest neighbors
for i in range(len(vec_list)):
D, I = index.search(np.array(vec_list[i:i+1]), k) # sanity check
I = I[0]
D = D[0]
print(I,D)
create_fig(i,val_path_list[i:i+1]+[train_path_list[idx] for idx in I])
å±±ç°ååŒã®å Žå

ã©ãã§ãããïŒå€ãã®å Žåã§å±±ç°ååŒãäžäœã«åºãŠããŠããããšãããããŸãïŒïŒã¡ããã£ãšã ãããšã·ãNo1ã®æããããŸããã»ã»ã»ïŒ
ã ãããšã·ã®å Žå

ãïŒå±±ç°ååŒã®é¡åŒ·ããïŒ 5/9ã§å±±ç°ååŒãããªãã§ããã»ã»ã»ã
ãŸãšã
ä»åã¯åŠç¿ãªãã§é¡èå¥ã«åãçµãã§ã¿ãŸããïŒ
æã£ãŠãããé£ããã£ãã§ããã»ã»ã»ã
ã§ããæãã€ããå®è£
ãã§ããã®ã§æºè¶³ã§ãïŒ
ç¹åŸŽãå©çšããç»åæ€çŽ¢ã¯ããè¡ãããŠããã®ã§ãäœãã®åèã«ãªãã°å¹žãã§ãïŒ
ã§ã¯ãŸãïŒæ¬¡åã¯äœã®èšäºã«ãªãã§ããããïœïœïœãïŒããCenterNetãšã¯èšããŸããã»ã»ã»ïŒ
Discussion
çŽ æŽãããèšäºãããããšãããããŸãïŒðð faissãé¢çœããã§ããïŒ