Merge pull request #5 from creativeIKEP/develop

v1.1.0

Assets/Scenes/Sample.unity

@@ -409,7 +409,6 @@ MonoBehaviour:
   faceLineTemplateMesh: {fileID: 4300000, guid: 4d1b450bb6d281641af6c72e8b37cfcb, type: 2}
   handShader: {fileID: 4800000, guid: 1b792b8e71d565242ae238f771a1db59, type: 3}
   handScoreThreshold: 0.5
-  holisticResource: {fileID: 11400000, guid: 3d7456c7320edba4caea63df7db64fb7, type: 2}
   holisticInferenceType: 0
 --- !u!114 &1716695272
 MonoBehaviour:
@@ -448,7 +447,7 @@ VideoPlayer:
   m_PrefabAsset: {fileID: 0}
   m_GameObject: {fileID: 1716695270}
   m_Enabled: 1
-  m_VideoClip: {fileID: 32900000, guid: fe097d08b4bc3d547819e723bcc49b71, type: 3}
+  m_VideoClip: {fileID: 32900000, guid: c15773ede1129c044a1cdcd7fe466cc7, type: 3}
   m_TargetCameraAlpha: 1
   m_TargetCamera3DLayout: 0
   m_TargetCamera: {fileID: 0}

Assets/Scripts/Visuallizer.cs

@@ -15,8 +15,6 @@ public class Visuallizer : MonoBehaviour
     [SerializeField] Mesh faceLineTemplateMesh;
     [SerializeField] Shader handShader;
     [SerializeField, Range(0, 1)] float handScoreThreshold = 0.5f;
-    // Set "Packages/HolisticBarracuda/ResourceSet/Holistic.asset" on the Unity Editor.
-    [SerializeField] HolisticResource holisticResource;
     // Select inference type with pull down on the Unity Editor.
     [SerializeField] HolisticInferenceType holisticInferenceType = HolisticInferenceType.full;
 
@@ -42,7 +40,7 @@ public class Visuallizer : MonoBehaviour
     void Start()
     {
         // Make instance of HolisticPipeline
-        holisticPipeline = new HolisticPipeline(holisticResource);
+        holisticPipeline = new HolisticPipeline();
 
         poseMaterial = new Material(poseShader);
         faceMeshMaterial = new Material(faceShader);

Packages/HolisticBarracuda/CHANGELOG.md

@@ -1,2 +1,10 @@
+### [1.1.0] - 2022-09-25
+- Update BlazePose version
+- Fixed an issue that estimation was not performed correctly when the color space was Liner.
+- Automatically load `HolisticResource` asset data. The constructor arguments are not required.
+- Add new methods (`GetPoseLandmark`, `GetPoseWorldLandmark`, `GetFaceLandmark`, `GetLeftEyeLandmark`, `GetRightEyeLandmark`, `GetLeftHandLandmark` and `GetRightHandLandmark`) for accessing data with CPU (C#).
+- Add detection score variables (`faceDetectionScore`, `leftHandDetectionScore` and `rightHandDetectionScore`)
+- Improve the stability of hand estimation.
+
 ### [1.0.1] - 2021-10-06
 This is the first release of `creativeIKEP/HolisticBarracuda`(`jp.ikep.mediapipe.holistic`).

Packages/HolisticBarracuda/ComputeShader/CommonProcess.compute

@@ -1,6 +1,8 @@
 #pragma kernel LetterBoxImage
 
+#include "UnityCG.cginc"
 
+uint _isLinerColorSpace;
 float2 _spadScale;
 uint _letterboxWidth;
 sampler2D _letterboxInput;
@@ -28,5 +30,7 @@ void LetterBoxImage(uint2 id : SV_DispatchThreadID)
     // Bounding
     rgb *= all(uv > 0) && all (uv < 1);
 
+    if(_isLinerColorSpace) rgb = LinearToGammaSpace(rgb);
+
     _letterboxTexture[id] = float4(rgb, 1);
 }

Packages/HolisticBarracuda/ComputeShader/HandProcess.compute

@@ -11,36 +11,48 @@
 #define HAND_KEYPOINT_COUNT 21
 
 
-uint _detectionCount;
+ByteAddressBuffer _detectionCount;
 float _regionDetectDt;
+StructuredBuffer<float4> _poseInput;
 StructuredBuffer<PalmDetection> _palmDetections;
 RWStructuredBuffer<HandRegion> _handsRegionFromPalm;
 
 [numthreads(1, 1, 1)]
 void HandRegionDetectFromPalm(uint id : SV_DispatchThreadID)
 {
-    HandRegion pRegion0 = _handsRegionFromPalm[0];
-    HandRegion pRegion1 = _handsRegionFromPalm[1];
+    uint detectionCount = _detectionCount.Load(0);
+    if(detectionCount == 0) return;
+
+    HandRegion pRegionL = _handsRegionFromPalm[0];
+    HandRegion pRegionR = _handsRegionFromPalm[1];
 
-    for(uint i = 0; i<_detectionCount; i++){
+    float4 index_l = _poseInput[19];
+    float4 pinky_l = _poseInput[17];
+    float2 boundingBoxCenter_l = (2.0f * index_l.xy + pinky_l.xy) / 3.0f;
 
+    float4 index_r = _poseInput[20];
+    float4 pinky_r = _poseInput[18];
+    float2 boundingBoxCenter_r = (2.0f * index_r.xy + pinky_r.xy) / 3.0f;
+
+    for(uint i = 0; i < detectionCount; i++){
         // Input from the palm detection model
         const PalmDetection palm = _palmDetections[i];
 
         // Palm region size (squarified and enlarged bounding box)
-        float size = max(palm.extent.x, palm.extent.y) * 3;
+        float size = max(palm.extent.x, palm.extent.y) * 3.05f;
 
         // Palm angle
         const float2 up = palm.keyPoints[3] - palm.keyPoints[0];
         float angle = atan2(up.y, up.x) - PI / 2;
 
         const float4 box = float4(palm.center, size, angle);
 
-        float distance0 = distance(box.xy, pRegion0.box.xy);
-        float distance1 = distance(box.xy, pRegion1.box.xy);
+        float distanceFromL = distance(box.xy, boundingBoxCenter_l.xy);
+        float distanceFromR = distance(box.xy, boundingBoxCenter_r.xy);
 
-        HandRegion region = pRegion0;
-        if(distance0 > distance1) region = pRegion1;
+        uint isRight = distanceFromL > distanceFromR;
+        HandRegion region = pRegionL;
+        if(isRight) region = pRegionR;
 
         // Low pass filter parameters and input vector
         const float3 lpf_params = float3(2, 1.5f, _regionDetectDt);
@@ -52,18 +64,17 @@ void HandRegionDetectFromPalm(uint id : SV_DispatchThreadID)
         float4x4 m2 = makeScalingMatrix(region.box.z);
         float4x4 m3 = makeTranslationMatrix(0.5);
         float4x4 m4 = makeRotationMatrix(region.box.w);
-        float4x4 m5 = makeTranslationMatrix(-0.5 + float2(0, 0.15));
+        float4x4 m5 = makeTranslationMatrix(-0.5 + float2(0, 0.05f));
         region.cropMatrix = mul(mul(mul(mul(m1, m2), m3), m4), m5);
 
         // Compute buffer update
-        _handsRegionFromPalm[i] = region;
+        _handsRegionFromPalm[isRight] = region;
     }
 }
 
 
 int _isRight;
 float _bboxDt;
-StructuredBuffer<float4> _poseInput;
 RWStructuredBuffer<HandRegion> _bboxRegion;
 
 [numthreads(1,1,1)]
@@ -79,7 +90,7 @@ void HandRegionDetectFromPose(uint3 id : SV_DispatchThreadID)
 
     // bounding box
     float2 boundingBoxCenter = middle;
-    float boundingBoxSize = boxSize * 2.0f;
+    float boundingBoxSize = boxSize * 1.5f;
 
     // bounding box angle
     float target = PI * 0.5;

Packages/HolisticBarracuda/Documentation~/HolisticBarracuda.md

@@ -26,7 +26,7 @@ To the `scopedRegistries` section:
 ```
 To the `dependencies` section:
 ```
-"jp.ikep.mediapipe.holistic": "1.0.1"
+"jp.ikep.mediapipe.holistic": "1.1.0"
 ```
 Finally, the manifest file looks like below:
 ```
@@ -44,7 +44,7 @@ Finally, the manifest file looks like below:
         }
     ],
     "dependencies": {
-        "jp.ikep.mediapipe.holistic": "1.0.1",
+        "jp.ikep.mediapipe.holistic": "1.1.0",
         ...
     }
 }

Packages/HolisticBarracuda/README.md

@@ -1,5 +1,23 @@
 # HolisticBarracuda
-![demo](https://user-images.githubusercontent.com/34697515/136178988-9a6c37cb-09a2-43e4-9f05-f8c4908b8665.gif)
+`full`
+
+![full](https://user-images.githubusercontent.com/34697515/192131544-97d0aedb-bd4b-477c-a367-4c7f22f8f7cb.gif)
+
+`pose_and_face` 
+
+![pose_and_face](https://user-images.githubusercontent.com/34697515/192131548-66a26715-cc9d-4a1c-a391-3ecd0d648f02.gif)
+
+`pose_and_hand`
+
+![pose_and_hand](https://user-images.githubusercontent.com/34697515/192131549-b5929bd0-de56-4938-9cb2-a816987a639b.gif)
+
+`pose_only`
+
+![pose_only](https://user-images.githubusercontent.com/34697515/192131552-6b2948a5-93f2-47b4-bd45-d11bffe5a58c.gif)
+
+`face_only`
+
+![face_only](https://user-images.githubusercontent.com/34697515/192131531-2b46cfb9-d6b8-4668-81a6-93d6e4595b3f.gif)
 
 **HolisticBarracuda** is the Unity Package that simultaneously estimates 33 pose, 21 per-hand, and 468 facial landmarks with a monocular color camera only.
 
@@ -37,7 +55,7 @@ To the `scopedRegistries` section:
 ```
 To the `dependencies` section:
 ```
-"jp.ikep.mediapipe.holistic": "1.0.1"
+"jp.ikep.mediapipe.holistic": "1.1.0"
 ```
 Finally, the manifest file looks like below:
 ```
@@ -55,7 +73,7 @@ Finally, the manifest file looks like below:
         }
     ],
     "dependencies": {
-        "jp.ikep.mediapipe.holistic": "1.0.1",
+        "jp.ikep.mediapipe.holistic": "1.1.0",
         ...
     }
 }
@@ -67,7 +85,12 @@ This repository has the demo that inference pose, face and hands landmarks, and
 Check a Unity [scene](https://github.com/creativeIKEP/HolisticBarracuda/blob/main/Assets/Scenes/Sample.unity), [scripts](https://github.com/creativeIKEP/HolisticBarracuda/tree/main/Assets/Scripts) and [shaders](https://github.com/creativeIKEP/HolisticBarracuda/tree/main/Assets/Shaders) in the ["/Assets"](https://github.com/creativeIKEP/HolisticBarracuda/tree/main/Assets) directory.
 
 ## Demo image
-Videos for demoe scene (["/Assets/Scenes/Sample.unity"](https://github.com/creativeIKEP/HolisticBarracuda/blob/main/Assets/Scenes/Sample.unity)) was downloaded from [here](https://www.pexels.com/ja-jp/video/7559286/).
+Videos for demo was downloaded from [pexels](https://www.pexels.com/ja-jp/).
+- https://www.pexels.com/ja-jp/video/5089491/
+- https://www.pexels.com/ja-jp/video/4492700/
+- https://www.pexels.com/ja-jp/video/8627747/
+- https://www.pexels.com/ja-jp/video/2795750/
+- https://www.pexels.com/ja-jp/video/6985340/
 
 ## Author
 [IKEP](https://ikep.jp)

Packages/HolisticBarracuda/Script/FaceMesh/Script/FacePipeline_Process.cs

@@ -14,6 +14,7 @@ partial class FacePipeline
     FaceRegion _faceRegion = new FaceRegion();
     EyeRegion _leyeRegion = new EyeRegion();
     EyeRegion _reyeRegion = new EyeRegion(true);
+    float _faceDetectionScore;
 
     // Vertex retrieval from the face landmark detector
     float4 GetFaceVertex(int index)
@@ -26,7 +27,8 @@ void RunPipeline(Texture input)
 
         // Cancel if the face detection score is too low.
         var face = _faceDetector.Detections.FirstOrDefault();
-        if (face.score < 0.5f) return;
+        _faceDetectionScore = face.score;
+        if (_faceDetectionScore < 0.5f) return;
 
         // Try updating the face region with the detection result. It's
         // actually updated only when there is a noticeable jump from the last

Packages/HolisticBarracuda/Script/FaceMesh/Script/FacePipeline_Public.cs

@@ -51,6 +51,9 @@ public float4x4 RightEyeCropMatrix
 
     #endregion
 
+    public float FaceDetectionScore
+      => _faceDetectionScore;
+
     #region Public methods
 
     public FacePipeline(ResourceSet resources)

Packages/HolisticBarracuda/Script/FaceMesh/Shader/Preprocess.shader

@@ -16,7 +16,15 @@ Shader "Hidden/MediaPipe/FaceMesh/Preprocess"
                     float2 uv : TEXCOORD0) : SV_Target
     {
         uv = mul(_Xform, float4(uv, 0, 1)).xy;
-        return tex2D(_MainTex, uv);
+        float4 color = tex2D(_MainTex, uv);
+
+        // The Shader will display the appropriate colors even in the liner color space, 
+        // so the color representation will be wrong, but we will convert it for better estimation accuracy.
+        #if !UNITY_COLORSPACE_GAMMA
+        color.rgb = LinearToGammaSpace(color.rgb);
+        #endif
+
+        return color;
     }
 
     ENDCG