wandou
/
faceta-algo-server
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity
faceta-algo-server/benchmarks/Evaluate-Results.ipynb

1845 lines
175 KiB
Plaintext
Raw Normal View History

feature: face detect
2024-10-11 15:15:15 +00:00
{
"cells": [
{
"cell_type": "markdown",
"id": "59b076ef",
"metadata": {},
"source": [
"# Evaluate DeepFace's Results In The Big Picture"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "79200f8c",
"metadata": {},
"outputs": [],
"source": [
"import pandas as pd\n",
"from IPython.display import display, HTML\n",
"from sklearn import metrics\n",
"import matplotlib.pyplot as plt"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "bbc11592",
"metadata": {},
"outputs": [],
"source": [
"alignment = [False, True]\n",
"models = [\"Facenet512\", \"Facenet\", \"VGG-Face\", \"ArcFace\", \"Dlib\", \"GhostFaceNet\", \"SFace\", \"OpenFace\", \"DeepFace\", \"DeepID\"]\n",
"detectors = [\"retinaface\", \"mtcnn\", \"fastmtcnn\", \"dlib\", \"yolov8\", \"yunet\", \"centerface\", \"mediapipe\", \"ssd\", \"opencv\", \"skip\"]\n",
"distance_metrics = [\"euclidean\", \"euclidean_l2\", \"cosine\"]"
]
},
{
"cell_type": "markdown",
"id": "e0dabf1b",
"metadata": {},
"source": [
"# Main results"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "03b09fa3",
"metadata": {
"scrolled": false
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"euclidean for alignment False\n"
]
},
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Facenet512</th>\n",
" <th>Facenet</th>\n",
" <th>VGG-Face</th>\n",
" <th>ArcFace</th>\n",
" <th>Dlib</th>\n",
" <th>GhostFaceNet</th>\n",
" <th>SFace</th>\n",
" <th>OpenFace</th>\n",
" <th>DeepFace</th>\n",
" <th>DeepID</th>\n",
" </tr>\n",
" <tr>\n",
" <th>detector</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>retinaface</th>\n",
" <td>96.1</td>\n",
" <td>92.8</td>\n",
" <td>95.7</td>\n",
" <td>84.1</td>\n",
" <td>88.3</td>\n",
" <td>83.2</td>\n",
" <td>78.6</td>\n",
" <td>70.8</td>\n",
" <td>67.4</td>\n",
" <td>64.3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mtcnn</th>\n",
" <td>95.9</td>\n",
" <td>92.5</td>\n",
" <td>95.5</td>\n",
" <td>81.8</td>\n",
" <td>89.3</td>\n",
" <td>83.2</td>\n",
" <td>76.3</td>\n",
" <td>70.9</td>\n",
" <td>65.9</td>\n",
" <td>63.2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>fastmtcnn</th>\n",
" <td>96.3</td>\n",
" <td>93.0</td>\n",
" <td>96.0</td>\n",
" <td>82.2</td>\n",
" <td>90.0</td>\n",
" <td>82.7</td>\n",
" <td>76.8</td>\n",
" <td>71.2</td>\n",
" <td>66.5</td>\n",
" <td>64.3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>dlib</th>\n",
" <td>96.0</td>\n",
" <td>89.0</td>\n",
" <td>94.1</td>\n",
" <td>82.6</td>\n",
" <td>96.3</td>\n",
" <td>65.6</td>\n",
" <td>73.1</td>\n",
" <td>75.9</td>\n",
" <td>61.8</td>\n",
" <td>61.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yolov8</th>\n",
" <td>94.8</td>\n",
" <td>90.8</td>\n",
" <td>95.2</td>\n",
" <td>83.2</td>\n",
" <td>88.4</td>\n",
" <td>77.6</td>\n",
" <td>71.6</td>\n",
" <td>68.9</td>\n",
" <td>68.2</td>\n",
" <td>66.3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yunet</th>\n",
" <td>97.9</td>\n",
" <td>96.5</td>\n",
" <td>96.3</td>\n",
" <td>84.1</td>\n",
" <td>91.4</td>\n",
" <td>82.7</td>\n",
" <td>78.2</td>\n",
" <td>71.7</td>\n",
" <td>65.5</td>\n",
" <td>65.2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>centerface</th>\n",
" <td>97.4</td>\n",
" <td>95.4</td>\n",
" <td>95.8</td>\n",
" <td>83.2</td>\n",
" <td>90.3</td>\n",
" <td>82.0</td>\n",
" <td>76.5</td>\n",
" <td>69.9</td>\n",
" <td>65.7</td>\n",
" <td>62.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mediapipe</th>\n",
" <td>94.9</td>\n",
" <td>87.1</td>\n",
" <td>93.1</td>\n",
" <td>71.1</td>\n",
" <td>91.9</td>\n",
" <td>61.9</td>\n",
" <td>73.2</td>\n",
" <td>77.6</td>\n",
" <td>61.7</td>\n",
" <td>62.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>ssd</th>\n",
" <td>97.2</td>\n",
" <td>94.9</td>\n",
" <td>96.7</td>\n",
" <td>83.9</td>\n",
" <td>88.6</td>\n",
" <td>84.9</td>\n",
" <td>82.0</td>\n",
" <td>69.9</td>\n",
" <td>66.7</td>\n",
" <td>64.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>opencv</th>\n",
" <td>94.1</td>\n",
" <td>90.2</td>\n",
" <td>95.8</td>\n",
" <td>89.8</td>\n",
" <td>91.2</td>\n",
" <td>91.0</td>\n",
" <td>86.9</td>\n",
" <td>71.1</td>\n",
" <td>68.4</td>\n",
" <td>61.1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>skip</th>\n",
" <td>92.0</td>\n",
" <td>64.1</td>\n",
" <td>90.6</td>\n",
" <td>56.6</td>\n",
" <td>69.0</td>\n",
" <td>75.1</td>\n",
" <td>81.4</td>\n",
" <td>57.4</td>\n",
" <td>60.8</td>\n",
" <td>60.7</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<hr>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"euclidean_l2 for alignment False\n"
]
},
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Facenet512</th>\n",
" <th>Facenet</th>\n",
" <th>VGG-Face</th>\n",
" <th>ArcFace</th>\n",
" <th>Dlib</th>\n",
" <th>GhostFaceNet</th>\n",
" <th>SFace</th>\n",
" <th>OpenFace</th>\n",
" <th>DeepFace</th>\n",
" <th>DeepID</th>\n",
" </tr>\n",
" <tr>\n",
" <th>detector</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>retinaface</th>\n",
" <td>98.0</td>\n",
" <td>95.9</td>\n",
" <td>95.7</td>\n",
" <td>95.7</td>\n",
" <td>88.4</td>\n",
" <td>89.5</td>\n",
" <td>90.6</td>\n",
" <td>70.8</td>\n",
" <td>67.7</td>\n",
" <td>64.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mtcnn</th>\n",
" <td>97.8</td>\n",
" <td>96.2</td>\n",
" <td>95.5</td>\n",
" <td>95.9</td>\n",
" <td>89.2</td>\n",
" <td>88.0</td>\n",
" <td>91.1</td>\n",
" <td>70.9</td>\n",
" <td>67.0</td>\n",
" <td>64.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>fastmtcnn</th>\n",
" <td>97.7</td>\n",
" <td>96.6</td>\n",
" <td>96.0</td>\n",
" <td>95.9</td>\n",
" <td>89.6</td>\n",
" <td>87.8</td>\n",
" <td>89.7</td>\n",
" <td>71.2</td>\n",
" <td>67.8</td>\n",
" <td>64.2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>dlib</th>\n",
" <td>96.5</td>\n",
" <td>89.9</td>\n",
" <td>94.1</td>\n",
" <td>93.8</td>\n",
" <td>95.6</td>\n",
" <td>63.0</td>\n",
" <td>75.0</td>\n",
" <td>75.9</td>\n",
" <td>62.6</td>\n",
" <td>61.8</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yolov8</th>\n",
" <td>97.7</td>\n",
" <td>95.8</td>\n",
" <td>95.2</td>\n",
" <td>95.0</td>\n",
" <td>88.1</td>\n",
" <td>88.7</td>\n",
" <td>89.8</td>\n",
" <td>68.9</td>\n",
" <td>68.9</td>\n",
" <td>65.3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yunet</th>\n",
" <td>98.3</td>\n",
" <td>96.8</td>\n",
" <td>96.3</td>\n",
" <td>96.1</td>\n",
" <td>91.7</td>\n",
" <td>88.0</td>\n",
" <td>90.5</td>\n",
" <td>71.7</td>\n",
" <td>67.6</td>\n",
" <td>63.2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>centerface</th>\n",
" <td>97.4</td>\n",
" <td>96.3</td>\n",
" <td>95.8</td>\n",
" <td>95.8</td>\n",
" <td>90.2</td>\n",
" <td>86.8</td>\n",
" <td>89.3</td>\n",
" <td>69.9</td>\n",
" <td>68.4</td>\n",
" <td>63.1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mediapipe</th>\n",
" <td>96.3</td>\n",
" <td>90.0</td>\n",
" <td>93.1</td>\n",
" <td>89.3</td>\n",
" <td>91.8</td>\n",
" <td>65.6</td>\n",
" <td>74.6</td>\n",
" <td>77.6</td>\n",
" <td>64.9</td>\n",
" <td>61.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>ssd</th>\n",
" <td>97.9</td>\n",
" <td>97.0</td>\n",
" <td>96.7</td>\n",
" <td>96.6</td>\n",
" <td>89.4</td>\n",
" <td>91.5</td>\n",
" <td>93.0</td>\n",
" <td>69.9</td>\n",
" <td>68.7</td>\n",
" <td>64.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>opencv</th>\n",
" <td>96.2</td>\n",
" <td>92.9</td>\n",
" <td>95.8</td>\n",
" <td>93.2</td>\n",
" <td>91.5</td>\n",
" <td>93.3</td>\n",
" <td>91.7</td>\n",
" <td>71.1</td>\n",
" <td>68.3</td>\n",
" <td>61.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>skip</th>\n",
" <td>91.4</td>\n",
" <td>67.6</td>\n",
" <td>90.6</td>\n",
" <td>57.2</td>\n",
" <td>69.3</td>\n",
" <td>78.4</td>\n",
" <td>83.4</td>\n",
" <td>57.4</td>\n",
" <td>62.6</td>\n",
" <td>61.6</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<hr>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"cosine for alignment False\n"
]
},
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Facenet512</th>\n",
" <th>Facenet</th>\n",
" <th>VGG-Face</th>\n",
" <th>ArcFace</th>\n",
" <th>Dlib</th>\n",
" <th>GhostFaceNet</th>\n",
" <th>SFace</th>\n",
" <th>OpenFace</th>\n",
" <th>DeepFace</th>\n",
" <th>DeepID</th>\n",
" </tr>\n",
" <tr>\n",
" <th>detector</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>retinaface</th>\n",
" <td>98.0</td>\n",
" <td>95.9</td>\n",
" <td>95.7</td>\n",
" <td>95.7</td>\n",
" <td>88.4</td>\n",
" <td>89.5</td>\n",
" <td>90.6</td>\n",
" <td>70.8</td>\n",
" <td>67.7</td>\n",
" <td>63.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mtcnn</th>\n",
" <td>97.8</td>\n",
" <td>96.2</td>\n",
" <td>95.5</td>\n",
" <td>95.9</td>\n",
" <td>89.2</td>\n",
" <td>88.0</td>\n",
" <td>91.1</td>\n",
" <td>70.9</td>\n",
" <td>67.0</td>\n",
" <td>64.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>fastmtcnn</th>\n",
" <td>97.7</td>\n",
" <td>96.6</td>\n",
" <td>96.0</td>\n",
" <td>95.9</td>\n",
" <td>89.6</td>\n",
" <td>87.8</td>\n",
" <td>89.7</td>\n",
" <td>71.2</td>\n",
" <td>67.8</td>\n",
" <td>62.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>dlib</th>\n",
" <td>96.5</td>\n",
" <td>89.9</td>\n",
" <td>94.1</td>\n",
" <td>93.8</td>\n",
" <td>95.6</td>\n",
" <td>63.0</td>\n",
" <td>75.0</td>\n",
" <td>75.9</td>\n",
" <td>62.6</td>\n",
" <td>61.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yolov8</th>\n",
" <td>97.7</td>\n",
" <td>95.8</td>\n",
" <td>95.2</td>\n",
" <td>95.0</td>\n",
" <td>88.1</td>\n",
" <td>88.7</td>\n",
" <td>89.8</td>\n",
" <td>68.9</td>\n",
" <td>68.9</td>\n",
" <td>65.3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yunet</th>\n",
" <td>98.3</td>\n",
" <td>96.8</td>\n",
" <td>96.3</td>\n",
" <td>96.1</td>\n",
" <td>91.7</td>\n",
" <td>88.0</td>\n",
" <td>90.5</td>\n",
" <td>71.7</td>\n",
" <td>67.6</td>\n",
" <td>63.2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>centerface</th>\n",
" <td>97.4</td>\n",
" <td>96.3</td>\n",
" <td>95.8</td>\n",
" <td>95.8</td>\n",
" <td>90.2</td>\n",
" <td>86.8</td>\n",
" <td>89.3</td>\n",
" <td>69.9</td>\n",
" <td>68.4</td>\n",
" <td>62.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mediapipe</th>\n",
" <td>96.3</td>\n",
" <td>90.0</td>\n",
" <td>93.1</td>\n",
" <td>89.3</td>\n",
" <td>91.8</td>\n",
" <td>64.8</td>\n",
" <td>74.6</td>\n",
" <td>77.6</td>\n",
" <td>64.9</td>\n",
" <td>61.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>ssd</th>\n",
" <td>97.9</td>\n",
" <td>97.0</td>\n",
" <td>96.7</td>\n",
" <td>96.6</td>\n",
" <td>89.4</td>\n",
" <td>91.5</td>\n",
" <td>93.0</td>\n",
" <td>69.9</td>\n",
" <td>68.7</td>\n",
" <td>63.8</td>\n",
" </tr>\n",
" <tr>\n",
" <th>opencv</th>\n",
" <td>96.2</td>\n",
" <td>92.9</td>\n",
" <td>95.8</td>\n",
" <td>93.2</td>\n",
" <td>91.5</td>\n",
" <td>93.3</td>\n",
" <td>91.7</td>\n",
" <td>71.1</td>\n",
" <td>68.1</td>\n",
" <td>61.1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>skip</th>\n",
" <td>91.4</td>\n",
" <td>67.6</td>\n",
" <td>90.6</td>\n",
" <td>54.8</td>\n",
" <td>69.3</td>\n",
" <td>78.4</td>\n",
" <td>83.4</td>\n",
" <td>57.4</td>\n",
" <td>62.6</td>\n",
" <td>61.1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<hr>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"euclidean for alignment True\n"
]
},
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Facenet512</th>\n",
" <th>Facenet</th>\n",
" <th>VGG-Face</th>\n",
" <th>ArcFace</th>\n",
" <th>Dlib</th>\n",
" <th>GhostFaceNet</th>\n",
" <th>SFace</th>\n",
" <th>OpenFace</th>\n",
" <th>DeepFace</th>\n",
" <th>DeepID</th>\n",
" </tr>\n",
" <tr>\n",
" <th>detector</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>retinaface</th>\n",
" <td>95.9</td>\n",
" <td>93.5</td>\n",
" <td>95.8</td>\n",
" <td>85.2</td>\n",
" <td>88.9</td>\n",
" <td>85.9</td>\n",
" <td>80.2</td>\n",
" <td>69.4</td>\n",
" <td>67.0</td>\n",
" <td>65.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mtcnn</th>\n",
" <td>95.2</td>\n",
" <td>93.8</td>\n",
" <td>95.9</td>\n",
" <td>83.7</td>\n",
" <td>89.4</td>\n",
" <td>83.0</td>\n",
" <td>77.4</td>\n",
" <td>70.2</td>\n",
" <td>66.5</td>\n",
" <td>63.3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>fastmtcnn</th>\n",
" <td>96.0</td>\n",
" <td>93.4</td>\n",
" <td>95.8</td>\n",
" <td>83.5</td>\n",
" <td>91.1</td>\n",
" <td>82.8</td>\n",
" <td>77.7</td>\n",
" <td>69.4</td>\n",
" <td>66.7</td>\n",
" <td>64.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>dlib</th>\n",
" <td>96.0</td>\n",
" <td>90.8</td>\n",
" <td>94.5</td>\n",
" <td>88.6</td>\n",
" <td>96.8</td>\n",
" <td>65.7</td>\n",
" <td>66.3</td>\n",
" <td>75.8</td>\n",
" <td>63.4</td>\n",
" <td>60.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yolov8</th>\n",
" <td>94.4</td>\n",
" <td>91.9</td>\n",
" <td>95.0</td>\n",
" <td>84.1</td>\n",
" <td>89.2</td>\n",
" <td>77.6</td>\n",
" <td>73.4</td>\n",
" <td>68.7</td>\n",
" <td>69.0</td>\n",
" <td>66.5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yunet</th>\n",
" <td>97.3</td>\n",
" <td>96.1</td>\n",
" <td>96.0</td>\n",
" <td>84.9</td>\n",
" <td>92.2</td>\n",
" <td>84.0</td>\n",
" <td>79.4</td>\n",
" <td>70.9</td>\n",
" <td>65.8</td>\n",
" <td>65.2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>centerface</th>\n",
" <td>97.6</td>\n",
" <td>95.8</td>\n",
" <td>95.7</td>\n",
" <td>83.6</td>\n",
" <td>90.4</td>\n",
" <td>82.8</td>\n",
" <td>77.4</td>\n",
" <td>68.9</td>\n",
" <td>65.5</td>\n",
" <td>62.8</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mediapipe</th>\n",
" <td>95.1</td>\n",
" <td>88.6</td>\n",
" <td>92.9</td>\n",
" <td>73.2</td>\n",
" <td>93.1</td>\n",
" <td>63.2</td>\n",
" <td>72.5</td>\n",
" <td>78.7</td>\n",
" <td>61.8</td>\n",
" <td>62.2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>ssd</th>\n",
" <td>88.9</td>\n",
" <td>85.6</td>\n",
" <td>87.0</td>\n",
" <td>75.8</td>\n",
" <td>83.1</td>\n",
" <td>79.1</td>\n",
" <td>76.9</td>\n",
" <td>66.8</td>\n",
" <td>63.4</td>\n",
" <td>62.5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>opencv</th>\n",
" <td>88.2</td>\n",
" <td>84.2</td>\n",
" <td>87.3</td>\n",
" <td>73.0</td>\n",
" <td>84.4</td>\n",
" <td>83.8</td>\n",
" <td>81.1</td>\n",
" <td>66.4</td>\n",
" <td>65.5</td>\n",
" <td>59.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>skip</th>\n",
" <td>92.0</td>\n",
" <td>64.1</td>\n",
" <td>90.6</td>\n",
" <td>56.6</td>\n",
" <td>69.0</td>\n",
" <td>75.1</td>\n",
" <td>81.4</td>\n",
" <td>57.4</td>\n",
" <td>60.8</td>\n",
" <td>60.7</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<hr>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"euclidean_l2 for alignment True\n"
]
},
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Facenet512</th>\n",
" <th>Facenet</th>\n",
" <th>VGG-Face</th>\n",
" <th>ArcFace</th>\n",
" <th>Dlib</th>\n",
" <th>GhostFaceNet</th>\n",
" <th>SFace</th>\n",
" <th>OpenFace</th>\n",
" <th>DeepFace</th>\n",
" <th>DeepID</th>\n",
" </tr>\n",
" <tr>\n",
" <th>detector</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>retinaface</th>\n",
" <td>98.4</td>\n",
" <td>96.4</td>\n",
" <td>95.8</td>\n",
" <td>96.6</td>\n",
" <td>89.1</td>\n",
" <td>90.5</td>\n",
" <td>92.4</td>\n",
" <td>69.4</td>\n",
" <td>67.7</td>\n",
" <td>64.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mtcnn</th>\n",
" <td>97.6</td>\n",
" <td>96.8</td>\n",
" <td>95.9</td>\n",
" <td>96.0</td>\n",
" <td>90.0</td>\n",
" <td>89.8</td>\n",
" <td>90.5</td>\n",
" <td>70.2</td>\n",
" <td>66.4</td>\n",
" <td>64.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>fastmtcnn</th>\n",
" <td>98.1</td>\n",
" <td>97.2</td>\n",
" <td>95.8</td>\n",
" <td>96.4</td>\n",
" <td>91.0</td>\n",
" <td>89.5</td>\n",
" <td>90.0</td>\n",
" <td>69.4</td>\n",
" <td>67.4</td>\n",
" <td>64.1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>dlib</th>\n",
" <td>97.0</td>\n",
" <td>92.6</td>\n",
" <td>94.5</td>\n",
" <td>95.1</td>\n",
" <td>96.4</td>\n",
" <td>63.3</td>\n",
" <td>69.8</td>\n",
" <td>75.8</td>\n",
" <td>66.5</td>\n",
" <td>59.5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yolov8</th>\n",
" <td>97.3</td>\n",
" <td>95.7</td>\n",
" <td>95.0</td>\n",
" <td>95.5</td>\n",
" <td>88.8</td>\n",
" <td>88.9</td>\n",
" <td>91.9</td>\n",
" <td>68.7</td>\n",
" <td>67.5</td>\n",
" <td>66.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yunet</th>\n",
" <td>97.9</td>\n",
" <td>97.4</td>\n",
" <td>96.0</td>\n",
" <td>96.7</td>\n",
" <td>91.6</td>\n",
" <td>89.1</td>\n",
" <td>91.0</td>\n",
" <td>70.9</td>\n",
" <td>66.5</td>\n",
" <td>63.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>centerface</th>\n",
" <td>97.7</td>\n",
" <td>96.8</td>\n",
" <td>95.7</td>\n",
" <td>96.5</td>\n",
" <td>90.9</td>\n",
" <td>87.5</td>\n",
" <td>89.3</td>\n",
" <td>68.9</td>\n",
" <td>67.8</td>\n",
" <td>64.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mediapipe</th>\n",
" <td>96.1</td>\n",
" <td>90.6</td>\n",
" <td>92.9</td>\n",
" <td>90.3</td>\n",
" <td>92.6</td>\n",
" <td>64.4</td>\n",
" <td>75.4</td>\n",
" <td>78.7</td>\n",
" <td>64.7</td>\n",
" <td>63.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>ssd</th>\n",
" <td>88.7</td>\n",
" <td>87.5</td>\n",
" <td>87.0</td>\n",
" <td>86.2</td>\n",
" <td>83.3</td>\n",
" <td>82.2</td>\n",
" <td>84.6</td>\n",
" <td>66.8</td>\n",
" <td>64.1</td>\n",
" <td>62.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>opencv</th>\n",
" <td>87.6</td>\n",
" <td>84.8</td>\n",
" <td>87.3</td>\n",
" <td>84.6</td>\n",
" <td>84.0</td>\n",
" <td>85.0</td>\n",
" <td>83.6</td>\n",
" <td>66.4</td>\n",
" <td>63.8</td>\n",
" <td>60.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>skip</th>\n",
" <td>91.4</td>\n",
" <td>67.6</td>\n",
" <td>90.6</td>\n",
" <td>57.2</td>\n",
" <td>69.3</td>\n",
" <td>78.4</td>\n",
" <td>83.4</td>\n",
" <td>57.4</td>\n",
" <td>62.6</td>\n",
" <td>61.6</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<hr>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"cosine for alignment True\n"
]
},
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>Facenet512</th>\n",
" <th>Facenet</th>\n",
" <th>VGG-Face</th>\n",
" <th>ArcFace</th>\n",
" <th>Dlib</th>\n",
" <th>GhostFaceNet</th>\n",
" <th>SFace</th>\n",
" <th>OpenFace</th>\n",
" <th>DeepFace</th>\n",
" <th>DeepID</th>\n",
" </tr>\n",
" <tr>\n",
" <th>detector</th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>retinaface</th>\n",
" <td>98.4</td>\n",
" <td>96.4</td>\n",
" <td>95.8</td>\n",
" <td>96.6</td>\n",
" <td>89.1</td>\n",
" <td>90.5</td>\n",
" <td>92.4</td>\n",
" <td>69.4</td>\n",
" <td>67.7</td>\n",
" <td>64.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mtcnn</th>\n",
" <td>97.6</td>\n",
" <td>96.8</td>\n",
" <td>95.9</td>\n",
" <td>96.0</td>\n",
" <td>90.0</td>\n",
" <td>89.8</td>\n",
" <td>90.5</td>\n",
" <td>70.2</td>\n",
" <td>66.3</td>\n",
" <td>63.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>fastmtcnn</th>\n",
" <td>98.1</td>\n",
" <td>97.2</td>\n",
" <td>95.8</td>\n",
" <td>96.4</td>\n",
" <td>91.0</td>\n",
" <td>89.5</td>\n",
" <td>90.0</td>\n",
" <td>69.4</td>\n",
" <td>67.4</td>\n",
" <td>63.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>dlib</th>\n",
" <td>97.0</td>\n",
" <td>92.6</td>\n",
" <td>94.5</td>\n",
" <td>95.1</td>\n",
" <td>96.4</td>\n",
" <td>63.3</td>\n",
" <td>69.8</td>\n",
" <td>75.8</td>\n",
" <td>66.5</td>\n",
" <td>58.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yolov8</th>\n",
" <td>97.3</td>\n",
" <td>95.7</td>\n",
" <td>95.0</td>\n",
" <td>95.5</td>\n",
" <td>88.8</td>\n",
" <td>88.9</td>\n",
" <td>91.9</td>\n",
" <td>68.7</td>\n",
" <td>67.5</td>\n",
" <td>65.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>yunet</th>\n",
" <td>97.9</td>\n",
" <td>97.4</td>\n",
" <td>96.0</td>\n",
" <td>96.7</td>\n",
" <td>91.6</td>\n",
" <td>89.1</td>\n",
" <td>91.0</td>\n",
" <td>70.9</td>\n",
" <td>66.5</td>\n",
" <td>63.5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>centerface</th>\n",
" <td>97.7</td>\n",
" <td>96.8</td>\n",
" <td>95.7</td>\n",
" <td>96.5</td>\n",
" <td>90.9</td>\n",
" <td>87.5</td>\n",
" <td>89.3</td>\n",
" <td>68.9</td>\n",
" <td>67.8</td>\n",
" <td>63.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>mediapipe</th>\n",
" <td>96.1</td>\n",
" <td>90.6</td>\n",
" <td>92.9</td>\n",
" <td>90.3</td>\n",
" <td>92.6</td>\n",
" <td>64.3</td>\n",
" <td>75.4</td>\n",
" <td>78.7</td>\n",
" <td>64.8</td>\n",
" <td>63.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>ssd</th>\n",
" <td>88.7</td>\n",
" <td>87.5</td>\n",
" <td>87.0</td>\n",
" <td>86.2</td>\n",
" <td>83.3</td>\n",
" <td>82.2</td>\n",
" <td>84.5</td>\n",
" <td>66.8</td>\n",
" <td>63.8</td>\n",
" <td>62.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>opencv</th>\n",
" <td>87.6</td>\n",
" <td>84.9</td>\n",
" <td>87.2</td>\n",
" <td>84.6</td>\n",
" <td>84.0</td>\n",
" <td>85.0</td>\n",
" <td>83.6</td>\n",
" <td>66.2</td>\n",
" <td>63.7</td>\n",
" <td>60.1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>skip</th>\n",
" <td>91.4</td>\n",
" <td>67.6</td>\n",
" <td>90.6</td>\n",
" <td>54.8</td>\n",
" <td>69.3</td>\n",
" <td>78.4</td>\n",
" <td>83.4</td>\n",
" <td>57.4</td>\n",
" <td>62.6</td>\n",
" <td>61.1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
},
{
"data": {
"text/html": [
"<hr>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"for align in alignment:\n",
" for metric in distance_metrics:\n",
" df = pd.read_csv(f\"results/pivot_{metric}_with_alignment_{align}.csv\")\n",
" df = df.rename(columns = {'Unnamed: 0': 'detector'})\n",
" df = df.set_index('detector')\n",
"\n",
" print(f\"{metric} for alignment {align}\")\n",
" display(HTML(df.to_html()))\n",
" display(HTML(\"<hr>\"))"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "aef6dc64",
"metadata": {},
"outputs": [],
"source": [
"def create_github_table():\n",
" for metric in distance_metrics:\n",
" for align in [True, False]:\n",
" df = pd.read_csv(f\"results/pivot_{metric}_with_alignment_{align}.csv\")\n",
" df = df.rename(columns = {'Unnamed: 0': 'detector'})\n",
" df = df.set_index('detector')\n",
" \n",
" print(f\"Performance Matrix for {metric} while alignment is {align} \\n\")\n",
" header = \"| | \"\n",
" for col_name in df.columns.tolist():\n",
" header += f\"{col_name} |\"\n",
" print(header)\n",
" # -------------------------------\n",
" seperator = \"| --- | \"\n",
" for col_name in df.columns.tolist():\n",
" seperator += \" --- |\"\n",
" print(seperator)\n",
" # -------------------------------\n",
" for index, instance in df.iterrows():\n",
" line = f\"| {instance.name} |\"\n",
" for i in instance.values:\n",
" if i < 97.5:\n",
" line += f\"{i} |\"\n",
" else:\n",
" line += f\"**{i}** |\"\n",
" print(line)\n",
" \n",
" print(\"\\n---------------------------\")"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "5004caaa",
"metadata": {},
"outputs": [],
"source": [
"# create_github_table()"
]
},
{
"cell_type": "markdown",
"id": "965c655f",
"metadata": {},
"source": [
"# Alignment impact"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "6ce20a58",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>max_alignment_impact</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>ArcFace</th>\n",
" <td>6.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>DeepFace</th>\n",
" <td>3.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>GhostFaceNet</th>\n",
" <td>2.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Facenet</th>\n",
" <td>2.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>SFace</th>\n",
" <td>2.1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Dlib</th>\n",
" <td>1.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>DeepID</th>\n",
" <td>1.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>OpenFace</th>\n",
" <td>1.1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Facenet512</th>\n",
" <td>0.5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>VGG-Face</th>\n",
" <td>0.4</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"align_df = None\n",
"\n",
"for distance_metric in distance_metrics:\n",
" df1 = (\n",
" pd.read_csv(f\"results/pivot_{distance_metric}_with_alignment_True.csv\")\n",
" .rename(columns = {'Unnamed: 0': 'detector'})\n",
" .set_index('detector')\n",
" )\n",
" df2 = (\n",
" pd.read_csv(f\"results/pivot_{distance_metric}_with_alignment_False.csv\")\n",
" .rename(columns = {'Unnamed: 0': 'detector'})\n",
" .set_index('detector')\n",
" )\n",
" df1 = df1[df1.index != \"skip\"]\n",
" df2 = df2[df2.index != \"skip\"]\n",
" pivot_df = df1.subtract(df2)\n",
" \n",
" pivot_df = pivot_df.max()\n",
" pivot_df = pd.DataFrame(pivot_df, columns=[f'alignment_impact_of_{distance_metric}'])\n",
" # display(HTML(pivot_df.to_html()))\n",
"\n",
" if align_df is None:\n",
" align_df = pivot_df.copy()\n",
" else:\n",
" align_df = align_df.merge(pivot_df, left_index=True, right_index=True)\n",
"\n",
"# display(HTML(align_df.to_html()))\n",
"align_df = pd.DataFrame(align_df.max(axis=1), columns = [\"max_alignment_impact\"])\n",
"align_df = align_df.sort_values(by=[\"max_alignment_impact\"], ascending=False)\n",
"display(HTML(align_df.to_html()))"
]
},
{
"cell_type": "markdown",
"id": "f66e349f",
"metadata": {},
"source": [
"## Detection impact"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "34eca61b",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>max_detection_impact</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>ArcFace</th>\n",
" <td>41.8</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Facenet</th>\n",
" <td>32.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Dlib</th>\n",
" <td>27.3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>OpenFace</th>\n",
" <td>20.2</td>\n",
" </tr>\n",
" <tr>\n",
" <th>GhostFaceNet</th>\n",
" <td>15.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>SFace</th>\n",
" <td>9.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>DeepFace</th>\n",
" <td>7.6</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Facenet512</th>\n",
" <td>6.9</td>\n",
" </tr>\n",
" <tr>\n",
" <th>VGG-Face</th>\n",
" <td>6.1</td>\n",
" </tr>\n",
" <tr>\n",
" <th>DeepID</th>\n",
" <td>5.6</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>"
],
"text/plain": [
"<IPython.core.display.HTML object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"detect_df = None\n",
"for distance_metric in distance_metrics:\n",
" tmp_df = (\n",
" pd.read_csv(f\"results/pivot_{distance_metric}_with_alignment_False.csv\")\n",
" .rename(columns = {'Unnamed: 0': 'detector'})\n",
" .set_index('detector')\n",
" )\n",
" ref_df = tmp_df[tmp_df.index == \"skip\"]\n",
" \n",
" j = []\n",
" for i in range(0, len(detectors) - 1):\n",
" j.append(ref_df)\n",
" minus_df = pd.concat(j)\n",
" \n",
" tmp_df = tmp_df[tmp_df.index != \"skip\"]\n",
" minus_df.index = tmp_df.index\n",
" \n",
" # print(\"performance with no detection\")\n",
" # display(HTML(ref_df.to_html()))\n",
" \n",
" # print(\"pivot\")\n",
" tmp_df = tmp_df.subtract(minus_df)\n",
" # display(HTML(tmp_df.to_html()))\n",
" \n",
" # print(\"avg of detector impact for models\")\n",
" # avg_df = pd.DataFrame(tmp_df.mean()).T\n",
" avg_df = pd.DataFrame(tmp_df.max(), columns=[f\"detection_impact_of_{distance_metric}\"])\n",
" # display(HTML(avg_df.to_html()))\n",
"\n",
" if detect_df is None:\n",
" detect_df = avg_df.copy()\n",
" else:\n",
" detect_df = detect_df.merge(avg_df, left_index=True, right_index=True)\n",
"\n",
"# display(HTML(detect_df.to_html()))\n",
"detect_df = pd.DataFrame(detect_df.max(axis=1), columns = [\"max_detection_impact\"])\n",
"detect_df = detect_df.sort_values(by=[\"max_detection_impact\"], ascending=False)\n",
"display(HTML(detect_df.to_html()))\n"
]
},
{
"cell_type": "markdown",
"id": "1bdf64a3",
"metadata": {},
"source": [
"# facial recognition model's best scores"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "0cb1f232",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>best_accuracy_score</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>Facenet512</th>\n",
" <td>98.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Human-beings</th>\n",
" <td>97.5</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Facenet</th>\n",
" <td>97.4</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Dlib</th>\n",
" <td>96.8</td>\n",
" </tr>\n",
" <tr>\n",
" <th>VGG-Face</th>\n",
" <td>96.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>ArcFace</th>\n",
" <td>96.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>GhostFaceNet</th>\n",
" <td>93.3</td>\n",
" </tr>\n",
" <tr>\n",
" <th>SFace</th>\n",
" <td>93.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>OpenFace</th>\n",
" <td>78.7</td>\n",
" </tr>\n",
" <tr>\n",
" <th>DeepFace</th>\n",
" <td>69.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>DeepID</th>\n",
" <td>66.5</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" best_accuracy_score\n",
"Facenet512 98.4\n",
"Human-beings 97.5\n",
"Facenet 97.4\n",
"Dlib 96.8\n",
"VGG-Face 96.7\n",
"ArcFace 96.7\n",
"GhostFaceNet 93.3\n",
"SFace 93.0\n",
"OpenFace 78.7\n",
"DeepFace 69.0\n",
"DeepID 66.5"
]
},
"execution_count": 9,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df = pd.DataFrame()\n",
"for align in alignment:\n",
" for distance_metric in distance_metrics:\n",
" tmp_df = (\n",
" pd.read_csv(f\"results/pivot_{distance_metric}_with_alignment_{align}.csv\")\n",
" .rename(columns = {'Unnamed: 0': 'detector'})\n",
" .set_index('detector')\n",
" )\n",
" df = pd.concat([df, tmp_df])\n",
"\n",
"pivot_df = pd.DataFrame(df.max(), columns = [\"best_accuracy_score\"])\n",
"\n",
"# add human comparison\n",
"pivot_df.loc[\"Human-beings\"] = 97.5\n",
"\n",
"pivot_df = pivot_df.sort_values(by = [\"best_accuracy_score\"], ascending = False)\n",
"pivot_df"
]
},
{
"cell_type": "markdown",
"id": "b81ebe92",
"metadata": {},
"source": [
"# ROC Curves"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "bcb4db0a",
"metadata": {},
"outputs": [],
"source": [
"def plot_roc(model_name, detector_backend, distance_metric, align):\n",
" alignment_text = \"aligned\" if align == True else \"unaligned\"\n",
"\n",
" df = pd.read_csv(f\"outputs/test/{model_name}_{detector_backend}_{distance_metric}_{alignment_text}.csv\")\n",
" \n",
" #normalize\n",
" df[\"distances_normalized\"] = df[\"distances\"] / df[\"distances\"].max()\n",
" df[\"actuals_normalized\"] = 0\n",
" idx = df[df[\"actuals\"] == False].index\n",
" df.loc[idx, \"actuals_normalized\"] = 1\n",
" \n",
" y_actual = df[\"actuals_normalized\"].values.tolist()\n",
" y_pred_proba = df[\"distances_normalized\"].values.tolist()\n",
" \n",
" fpr, tpr, _ = metrics.roc_curve(y_actual, y_pred_proba)\n",
" auc = metrics.roc_auc_score(y_actual, y_pred_proba)\n",
" auc = round(auc, 4)\n",
"\n",
" # best accuracy score\n",
" result_path = f\"results/pivot_{distance_metric}_with_alignment_{align}.csv\"\n",
" result_df = pd.read_csv(result_path)\n",
" acc = result_df[result_df[\"Unnamed: 0\"] == detector_backend][model_name].values[0]\n",
"\n",
" label = f\"{model_name}_{detector_backend}_{distance_metric}_{alignment_text} (acc: {acc}, auc: {auc})\"\n",
"\n",
" return acc, auc, fpr, tpr, label"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "84b3d5b5",
"metadata": {
"scrolled": false
},
"outputs": [
{
"data": {
"image/png": "iVBORw0KGgoAAAANSUhEUgAABXwAAAKnCAYAAAA1L4U5AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOzdeXhU9dn/8c/M5MxkJoGoYFgkgopFERfAQt21UnF5XH5txV2aujxabW1pXHhE0LpXq9hqpS7BrZaqtWqt1SotbqWILAqKWlwwlaVQKyFMmJnMfH9/UCIhs5zJnDmzvV/Xlesi8z1zzp2FZOaTe+6vxxhjBAAAAAAAAAAoed5CFwAAAAAAAAAAcAaBLwAAAAAAAACUCQJfAAAAAAAAACgTBL4AAAAAAAAAUCYIfAEAAAAAAACgTBD4AgAAAAAAAECZIPAFAAAAAAAAgDJB4AsAAAAAAAAAZaKq0AW4LZFIaOXKlerVq5c8Hk+hywEAAAAAAABQAYwx2rBhgwYOHCivN399uBUX+K5cuVINDQ2FLgMAAAAAAABABWppadGgQYPydv6KC3x79eolafMntnfv3gWuBgAAAAAAAEAlaG1tVUNDQ2c+mS8VF/huGePQu3dvAl8AAAAAAAAArsr3mFk2bQMAAAAAAACAMkHgCwAAAAAAAABlgsAXAAAAAAAAAMoEgS8AAAAAAAAAlAkCXwAAAAAAAAAoEwS+AAAAAAAAAFAmCHwBAAAAAAAAoEwQ+AIAAAAAAABAmSDwBQAAAAAAAIAyQeALAAAAAAAAAGWCwBcAAAAAAAAAygSBLwAAAAAAAACUCQJfAAAAAAAAACgTBL4AAAAAAAAAUCYIfAEAAAAAAACgTBD4AgAAAAAAAECZIPAFAAAAAAAAgDJB4AsAAAAAAAAAZYLAFwAAAAAAAADKBIEvAAAAAAAAAJSJgga+r7zyio4//ngNHDhQHo9HTz31VMb7zJkzR6NGjVIgENDQoUP1wAMP5L1OAAAAAAAAACgFBQ18N27cqH333Vd33XWXreM//vhjHXfccTriiCO0ePFi/fCHP9S5556rF154Ic+VAgAAAAAAAEDxqyrkxY855hgdc8wxto+fMWOGdtllF/3sZz+TJO2555567bXXdPvtt2v8+PH5KhMAAAAAAAAASkJBA99szZ07V+PGjety2/jx4/XDH/6wMAUBAFBGjDHqiCYKXQYAVBZjpFi40FW4zhij9nik0GUAQEUxxshENhW6jIrWumGDK9cpqcB39erV6tevX5fb+vXrp9bWVrW3tysYDHa7TyQSUSTy5QOJ1tbWvNcJAICbnAhqjTH6/a0Lte6fbQ5VBQAAYI8xRlJHocsAgJz5Mqy3Rze6UkdJBb49ceONN+qaa64pdBkAkBVjjNpj8UKXgRJgjNFz09/Svz7mD5oAuiNEAVD8jKIbfisTX1voQgCUKJ/HKnQJ8kj6+oAztH2gX9rjNkSMfuxCPSUV+Pbv319r1qzpctuaNWvUu3fvpN29kjR58mRNmjSp8/3W1lY1NDTktU4A7iq3cNQY6eQZc/XuKgI8/FcioZpE8pf7Wsaj8zb2dexS/w7+U88Nm+7Y+VDepsyKa8i/Cl1FeTKS4h5PbifxSG/sWq8NQb8jNQEAUGmKIUhEanZD1kpUUoHvAQccoOeee67LbS+++KIOOOCAlPcJBAIKBAL5Lg1AEm4EsYSj5cQoKJdm+Rm3fgEayRPLeEggnvoYj4zOX+tRNDgo49UOfv1y+eLRbIvswpuI6uQ/5XQKoKwZSXFvjkGsDX8fupNagzyGBYoJwU9+7TCoQf9zyf9tTnCAQjPSpsc+VWIts8aRHW+/oEJn7Z5yPbGhVZqe/zoKGvi2tbVp+fLlne9//PHHWrx4sXbYYQftvPPOmjx5sj777DM99NBDkqQLLrhAd955py677DJ997vf1V/+8hc99thj+uMf/1ioDwFwDF2q+eZimOiQPfr31iPnjFGuDV65MsZoUzz3wf7GGGlTqvMYWY+dqcTaj3K+TmYePf/Ftfq8Y1cXruWMaPIXsXRRt/5DWbE2niPBdYHtYhpy5Lq8PkE3Ruow3vxdwG4dkh775z5aG6ktdClZ6dMwWCc2TSNEQVExm/9jF7oMewh+XBO+6x+FLgFACbIG1GjHC/Ytisc6HssrT5on8V6/O0UWNPB98803dcQRR3S+v2X0wsSJE/XAAw9o1apV+vTTTzvXd9llF/3xj3/Uj370I91xxx0aNGiQ7rvvPo0fP9712oFsZApz7YejpRda2sipHJE+HDWqfvh/5F2zxKVqHPKFpJ+lXjaS2tP8Itn8PCr37rDz+9VrhTf3jpZ0L/02Ho8W7vcjtfVi5E4qgfYWfWvQFZIn+ZPjqr4ReYbmfp2gMcXwOAklxOMzOf1hKlOYayTN+mTfkgtZnbDj4CE6dco16f9PWiFl+gJUBQJpn3igPBhjZGK5beDpGiOtnfGWYqvc2bgGAHqqmIJEpJYpZK1EHrN5J4eK0draqrq6Oq1fv169e/cudDmoAMYYfXvGXC1Y8Z9cz6Qn/Ndof+8HjtSF0mYknT2gnxZXpwh0jXTSO5eo/4bS6WJ1W+2GFo1afLs8RfBrsOoru6nfL29LG9r03m57+XyZ9nwFSosxRrOunaqV/yit321OBbGZENTCLmOM1s54W9EVxfLKqvJE8ANUHoJEOM2tXLKkZvgChZDrqIVwNG477B0+oLcev+CA5M8PoxsVurW0nhAXnf57S43P5/wE3A3GGLWnGaPQ3tGuxb8/NuV6VcJfkmHvDgOqdfSFQ115IlVl7SOP53/yfp1gVXXGB4meYJAHknCVMUYdkcK/YiQW2WQ77N1xyK469Zqb5SmCpIUgFsXGxBIlGfaWWoBK8AMAKBUEvuhUbjNkneD0HNo3p4xTyJ+6Qy9o+dI8iNzqv2vTcskfcqSmSmGMUYepluTZ3B5bxIwxOu/P5+ntdW+nPa5Km3ddf/6bf1J1VdfhGR3RhGa9sUCSNOGq4aryJ3+59Jrzv6eO9+3NSttt9ovyVuc2pCNTsFnl54kUik+xhKNOMTKaNe1yrf3EjZnZ9l14zyOyAtUp1wlZYVdJjTZwiIl++Rh+wJSx8qR5vFlMCFABAMgPAl9IcnLsAFLZf/D26lPjd+ZBrT8k+WtyP08OjDHqiJbGkyljjH7/s4Va19JW6FJsG6OzNMbmsU+8sTTt+ppvjJcvEU25bucpYXDUKFX335EnZSg5uYa1xRqOlpuBw4Yr2LuOnzEFUlYBKbNh5fH75C2RwBcAAOQHgW+FyNS9m83YgUqUdtSCTem7d7W5nTgWTr0eTbPmsExhbikGqJWqbv2H8qYJeyUpsOeeGvLIw2lHXTByAKXIGKNZUy/Tyg+WFbqUosSIhOLmWghLQFpW/IN7y2Ol3gQRAABUBgLfCpBt926msQOVKGNYmytjpObxUss8B06VW+dtOYe5xbRJl1uCw3bTLgveJMxFyXFijEIsssmxsLeYwlGnELIWLzbgyl2pzYZ1CiMSAACAROBbEdpj9rt3HR07APtiYVthrzFSx06HSImAFOnesW2M0e9vXah1/3QnrC21ANWbiBbN8z5rj2Hq/+B9SvdM1M5mX5kQ5qIU5aMzN9N82EwIRwurrEYO2GCicdfD3nILSAk+AQBAJSPwrTC5bRpWoTKNWnBCNLw5zDUB6ZIlkr/7xlhG0u+nv6t1C8PSwlfyW4/shbl2AlQ74wJyZYxRe8emvJ1/a+e/dL4++Pz9jMfNmfCyQlbqDc4IYoHUOiIRR8Ne5sMWt4xhboWPHHBrAy4CUgAAgPJB4FthQn6fQn6+7LY5OGoh02We/PwGrY7tKV35Ts7nc6Lz1k6Ya6dL1ROs1qY8P4Gc+Px39N7n7+X1Gl340388I+tHqqb3DjxxBlLINK4hFvnyDzi5duZKdOcWM0YXpOcf3FveGovvXwAAAGSF5A9Ix+aohVx1mMDmsNcGu523X3n9NXmDSTqFjdF5fz5Pb697O8OVMncTRazl0lNHZDyu3Oyxwx568OgHU64Hq+jeBVLJdlyDFaiWVZ1b4Iv
"text/plain": [
"<Figure size 1700x800 with 1 Axes>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"# to show all models in same graph\n",
"plt.figure(figsize=(17, 8))\n",
"\n",
"for model_name in models:\n",
" # to show graphs model by model\n",
" # plt.figure(figsize=(17, 8))\n",
" accs = []\n",
" aucs = []\n",
" fprs = []\n",
" tprs = []\n",
" labels = []\n",
" for distance_metric in distance_metrics:\n",
" # for detector_backend in robust_face_detectors:\n",
" for detector_backend in detectors:\n",
" for align in alignment:\n",
" if detector_backend == \"skip\" and align is True:\n",
" continue\n",
" acc, auc, fpr, tpr, label = plot_roc(model_name, detector_backend, distance_metric, align)\n",
" accs.append(acc)\n",
" aucs.append(auc)\n",
" fprs.append(fpr)\n",
" tprs.append(tpr)\n",
" labels.append(label)\n",
" # ---------------------------------\n",
" #sort by auc\n",
" df = pd.DataFrame({\"acc\": accs, \"auc\": aucs, \"fpr\": fprs, \"tpr\": tprs, \"label\": labels})\n",
" # df = df.sort_values(by = [\"auc\"], ascending = False).reset_index()\n",
" df = df.sort_values(by = [\"acc\"], ascending = False).reset_index()\n",
" \n",
" for index, instance in df.iterrows():\n",
" fpr = instance[\"fpr\"]\n",
" tpr = instance[\"tpr\"]\n",
" auc = instance[\"auc\"]\n",
" acc = instance[\"acc\"]\n",
" label = instance[\"label\"]\n",
" \n",
" plt.plot(fpr, tpr, label=label)\n",
" plt.ylabel(\"True Positive Rate\")\n",
" plt.xlabel(\"False Positive Rate\")\n",
" plt.legend(loc=\"lower center\", ncol=2)\n",
" # normally this should be [0, 1] but that scale makes graphs not legible\n",
" # plt.xlim([0, 1])\n",
" plt.xlim([0, 0.3])\n",
"\n",
" # to show the best auc value\n",
" break\n",
" \n",
" # to show graphs model by model\n",
" # plt.show()\n",
" # print(\"----------------\")\n",
"\n",
"# to show all models in same graph\n",
"plt.show()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "661c5236",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.16"
}
},
"nbformat": 4,
"nbformat_minor": 5
}