Skip to content

Commit 5dae993

Browse files
tucan9389tucan9389
committed
Support simple baseline pose estimation model provied by imgclsmob repo (#59)
1 parent a35166b commit 5dae993

File tree

7 files changed

+297
-14
lines changed

7 files changed

+297
-14
lines changed

Diff for: ‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,6 @@ iOSInjectionProject/
9191

9292
Pods/*
9393
Podfile.lock
94+
95+
# TensorFlow Lite Model
96+
*.tflite

Diff for: ‎PoseEstimation-TFLiteSwift.xcodeproj/project.pbxproj

+68
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,21 @@
2727
712A7FD12426691B00B043F9 /* PEFMCPMPoseEstimator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 712A7FD02426691B00B043F9 /* PEFMCPMPoseEstimator.swift */; };
2828
712A7FD324266EC700B043F9 /* pefm_hourglass_v1.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 712A7FD224266EC700B043F9 /* pefm_hourglass_v1.tflite */; };
2929
7138DCCF242142FE0048E1D2 /* TFLiteFlatArray.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7138DCCE242142FE0048E1D2 /* TFLiteFlatArray.swift */; };
30+
713A0479247905AC00F90C77 /* simplepose_mobile_mobilenetv3_small_w1_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A046B2479056E00F90C77 /* simplepose_mobile_mobilenetv3_small_w1_coco.tflite */; };
31+
713A047A247905AC00F90C77 /* simplepose_resnet50b_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A046C2479057300F90C77 /* simplepose_resnet50b_coco.tflite */; };
32+
713A047B247905AC00F90C77 /* alphapose_fastseresnet101b_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A046D2479057B00F90C77 /* alphapose_fastseresnet101b_coco.tflite */; };
33+
713A047C247905AC00F90C77 /* simplepose_mobile_resnet18_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A046E2479057D00F90C77 /* simplepose_mobile_resnet18_coco.tflite */; };
34+
713A047D247905AC00F90C77 /* simplepose_resnet152b_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A046F2479058600F90C77 /* simplepose_resnet152b_coco.tflite */; };
35+
713A047E247905AC00F90C77 /* simplepose_mobile_resnet50b_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A04702479058A00F90C77 /* simplepose_mobile_resnet50b_coco.tflite */; };
36+
713A047F247905AC00F90C77 /* simplepose_resnet101b_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A04712479059200F90C77 /* simplepose_resnet101b_coco.tflite */; };
37+
713A0480247905AC00F90C77 /* simplepose_resneta152b_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A04722479059B00F90C77 /* simplepose_resneta152b_coco.tflite */; };
38+
713A0481247905AC00F90C77 /* simplepose_resneta50b_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A0473247905A000F90C77 /* simplepose_resneta50b_coco.tflite */; };
39+
713A0482247905AC00F90C77 /* simplepose_resneta101b_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A0474247905A700F90C77 /* simplepose_resneta101b_coco.tflite */; };
40+
713A0483247905AC00F90C77 /* simplepose_mobile_mobilenetv2b_w1_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A0475247905A800F90C77 /* simplepose_mobile_mobilenetv2b_w1_coco.tflite */; };
41+
713A0484247905AC00F90C77 /* simplepose_mobile_mobilenetv3_large_w1_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A0476247905A900F90C77 /* simplepose_mobile_mobilenetv3_large_w1_coco.tflite */; };
42+
713A0485247905AC00F90C77 /* simplepose_resnet18_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A0477247905AB00F90C77 /* simplepose_resnet18_coco.tflite */; };
43+
713A0486247905AC00F90C77 /* simplepose_mobile_mobilenet_w1_coco.tflite in Resources */ = {isa = PBXBuildFile; fileRef = 713A0478247905AC00F90C77 /* simplepose_mobile_mobilenet_w1_coco.tflite */; };
44+
713A04882479071D00F90C77 /* IMGCLSPoseEstimator.swift in Sources */ = {isa = PBXBuildFile; fileRef = 713A04872479071D00F90C77 /* IMGCLSPoseEstimator.swift */; };
3045
71A1ED1F24527D55001F796C /* PoseConfidenceMapDrawingView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 71A1ED1E24527D55001F796C /* PoseConfidenceMapDrawingView.swift */; };
3146
71A1ED4124574F2E001F796C /* StillImageHeatmapViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 71A1ED4024574F2E001F796C /* StillImageHeatmapViewController.swift */; };
3247
71A1ED43245971AC001F796C /* LICENSE in Resources */ = {isa = PBXBuildFile; fileRef = 71A1ED42245971AC001F796C /* LICENSE */; };
@@ -64,6 +79,21 @@
6479
712A7FD02426691B00B043F9 /* PEFMCPMPoseEstimator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PEFMCPMPoseEstimator.swift; sourceTree = "<group>"; };
6580
712A7FD224266EC700B043F9 /* pefm_hourglass_v1.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = pefm_hourglass_v1.tflite; sourceTree = "<group>"; };
6681
7138DCCE242142FE0048E1D2 /* TFLiteFlatArray.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TFLiteFlatArray.swift; sourceTree = "<group>"; };
82+
713A046B2479056E00F90C77 /* simplepose_mobile_mobilenetv3_small_w1_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_mobile_mobilenetv3_small_w1_coco.tflite; sourceTree = "<group>"; };
83+
713A046C2479057300F90C77 /* simplepose_resnet50b_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_resnet50b_coco.tflite; sourceTree = "<group>"; };
84+
713A046D2479057B00F90C77 /* alphapose_fastseresnet101b_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = alphapose_fastseresnet101b_coco.tflite; sourceTree = "<group>"; };
85+
713A046E2479057D00F90C77 /* simplepose_mobile_resnet18_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_mobile_resnet18_coco.tflite; sourceTree = "<group>"; };
86+
713A046F2479058600F90C77 /* simplepose_resnet152b_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_resnet152b_coco.tflite; sourceTree = "<group>"; };
87+
713A04702479058A00F90C77 /* simplepose_mobile_resnet50b_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_mobile_resnet50b_coco.tflite; sourceTree = "<group>"; };
88+
713A04712479059200F90C77 /* simplepose_resnet101b_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_resnet101b_coco.tflite; sourceTree = "<group>"; };
89+
713A04722479059B00F90C77 /* simplepose_resneta152b_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_resneta152b_coco.tflite; sourceTree = "<group>"; };
90+
713A0473247905A000F90C77 /* simplepose_resneta50b_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_resneta50b_coco.tflite; sourceTree = "<group>"; };
91+
713A0474247905A700F90C77 /* simplepose_resneta101b_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_resneta101b_coco.tflite; sourceTree = "<group>"; };
92+
713A0475247905A800F90C77 /* simplepose_mobile_mobilenetv2b_w1_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_mobile_mobilenetv2b_w1_coco.tflite; sourceTree = "<group>"; };
93+
713A0476247905A900F90C77 /* simplepose_mobile_mobilenetv3_large_w1_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_mobile_mobilenetv3_large_w1_coco.tflite; sourceTree = "<group>"; };
94+
713A0477247905AB00F90C77 /* simplepose_resnet18_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_resnet18_coco.tflite; sourceTree = "<group>"; };
95+
713A0478247905AC00F90C77 /* simplepose_mobile_mobilenet_w1_coco.tflite */ = {isa = PBXFileReference; lastKnownFileType = file; path = simplepose_mobile_mobilenet_w1_coco.tflite; sourceTree = "<group>"; };
96+
713A04872479071D00F90C77 /* IMGCLSPoseEstimator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IMGCLSPoseEstimator.swift; sourceTree = "<group>"; };
6797
71A1ED1E24527D55001F796C /* PoseConfidenceMapDrawingView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PoseConfidenceMapDrawingView.swift; sourceTree = "<group>"; };
6898
71A1ED4024574F2E001F796C /* StillImageHeatmapViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = StillImageHeatmapViewController.swift; sourceTree = "<group>"; };
6999
71A1ED42245971AC001F796C /* LICENSE */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = LICENSE; sourceTree = "<group>"; };
@@ -154,6 +184,7 @@
154184
7105C92E241D0235001A4325 /* PoseEstimator.swift */,
155185
7105C932241D0651001A4325 /* TFLiteImageInterpreter.swift */,
156186
7138DCCE242142FE0048E1D2 /* TFLiteFlatArray.swift */,
187+
71FE62A62478EFB800425708 /* imgclsmob-Pose */,
157188
712A7FC22424FEC900B043F9 /* PoseNet */,
158189
712A7FC7242667A400B043F9 /* PoseEstimationForMobile */,
159190
71E8D9152438BA5B0081DD6E /* OpenPose */,
@@ -207,6 +238,28 @@
207238
path = OpenPose;
208239
sourceTree = "<group>";
209240
};
241+
71FE62A62478EFB800425708 /* imgclsmob-Pose */ = {
242+
isa = PBXGroup;
243+
children = (
244+
713A046D2479057B00F90C77 /* alphapose_fastseresnet101b_coco.tflite */,
245+
713A0478247905AC00F90C77 /* simplepose_mobile_mobilenet_w1_coco.tflite */,
246+
713A0475247905A800F90C77 /* simplepose_mobile_mobilenetv2b_w1_coco.tflite */,
247+
713A046B2479056E00F90C77 /* simplepose_mobile_mobilenetv3_small_w1_coco.tflite */,
248+
713A0476247905A900F90C77 /* simplepose_mobile_mobilenetv3_large_w1_coco.tflite */,
249+
713A046E2479057D00F90C77 /* simplepose_mobile_resnet18_coco.tflite */,
250+
713A04702479058A00F90C77 /* simplepose_mobile_resnet50b_coco.tflite */,
251+
713A0477247905AB00F90C77 /* simplepose_resnet18_coco.tflite */,
252+
713A046C2479057300F90C77 /* simplepose_resnet50b_coco.tflite */,
253+
713A04712479059200F90C77 /* simplepose_resnet101b_coco.tflite */,
254+
713A046F2479058600F90C77 /* simplepose_resnet152b_coco.tflite */,
255+
713A0473247905A000F90C77 /* simplepose_resneta50b_coco.tflite */,
256+
713A0474247905A700F90C77 /* simplepose_resneta101b_coco.tflite */,
257+
713A04722479059B00F90C77 /* simplepose_resneta152b_coco.tflite */,
258+
713A04872479071D00F90C77 /* IMGCLSPoseEstimator.swift */,
259+
);
260+
path = "imgclsmob-Pose";
261+
sourceTree = "<group>";
262+
};
210263
C0C61619494007101B3B8411 /* Pods */ = {
211264
isa = PBXGroup;
212265
children = (
@@ -284,15 +337,29 @@
284337
buildActionMask = 2147483647;
285338
files = (
286339
712A7FD324266EC700B043F9 /* pefm_hourglass_v1.tflite in Resources */,
340+
713A047D247905AC00F90C77 /* simplepose_resnet152b_coco.tflite in Resources */,
341+
713A0481247905AC00F90C77 /* simplepose_resneta50b_coco.tflite in Resources */,
287342
7105C922241CE9B7001A4325 /* LaunchScreen.storyboard in Resources */,
343+
713A047A247905AC00F90C77 /* simplepose_resnet50b_coco.tflite in Resources */,
288344
7105C91F241CE9B7001A4325 /* Assets.xcassets in Resources */,
345+
713A0479247905AC00F90C77 /* simplepose_mobile_mobilenetv3_small_w1_coco.tflite in Resources */,
289346
712A7FCF2426690A00B043F9 /* pefm_cpm.tflite in Resources */,
347+
713A0486247905AC00F90C77 /* simplepose_mobile_mobilenet_w1_coco.tflite in Resources */,
290348
71A1ED43245971AC001F796C /* LICENSE in Resources */,
349+
713A0483247905AC00F90C77 /* simplepose_mobile_mobilenetv2b_w1_coco.tflite in Resources */,
291350
71E8D9172438BAC10081DD6E /* openpose_ildoonet.tflite in Resources */,
351+
713A0480247905AC00F90C77 /* simplepose_resneta152b_coco.tflite in Resources */,
292352
7105C91D241CE9B6001A4325 /* Main.storyboard in Resources */,
293353
71E8D93B243CC5330081DD6E /* README.md in Resources */,
354+
713A0482247905AC00F90C77 /* simplepose_resneta101b_coco.tflite in Resources */,
355+
713A0485247905AC00F90C77 /* simplepose_resnet18_coco.tflite in Resources */,
356+
713A047C247905AC00F90C77 /* simplepose_mobile_resnet18_coco.tflite in Resources */,
357+
713A047F247905AC00F90C77 /* simplepose_resnet101b_coco.tflite in Resources */,
294358
712A7FCD242668FF00B043F9 /* pefm_hourglass_v2.tflite in Resources */,
359+
713A047B247905AC00F90C77 /* alphapose_fastseresnet101b_coco.tflite in Resources */,
360+
713A0484247905AC00F90C77 /* simplepose_mobile_mobilenetv3_large_w1_coco.tflite in Resources */,
295361
7105C93A241E7624001A4325 /* posenet_mobilenet_v1_100_257x257_multi_kpt_stripped.tflite in Resources */,
362+
713A047E247905AC00F90C77 /* simplepose_mobile_resnet50b_coco.tflite in Resources */,
296363
);
297364
runOnlyForDeploymentPostprocessing = 0;
298365
};
@@ -330,6 +397,7 @@
330397
files = (
331398
7105C933241D0651001A4325 /* TFLiteImageInterpreter.swift in Sources */,
332399
71A1ED1F24527D55001F796C /* PoseConfidenceMapDrawingView.swift in Sources */,
400+
713A04882479071D00F90C77 /* IMGCLSPoseEstimator.swift in Sources */,
333401
7105C93C241E8CE3001A4325 /* CVPixelBufferExtension.swift in Sources */,
334402
712A7FC9242667C900B043F9 /* PEFMHourglassPoseEstimator.swift in Sources */,
335403
71B07B97245E5C6C001FD385 /* NumericExtension.swift in Sources */,

Diff for: ‎PoseEstimation-TFLiteSwift/LiveLineHeatmapViewController.swift

+1-1
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ class LiveLineHeatmapViewController: UIViewController {
122122
humanType: humanType)
123123
}
124124

125-
let poseEstimator: PoseEstimator = OpenPosePoseEstimator()
125+
let poseEstimator: PoseEstimator = IMGCLSPoseEstimator()
126126

127127
override func viewDidLoad() {
128128
super.viewDidLoad()

Diff for: ‎PoseEstimation-TFLiteSwift/OpenPose/OpenPosePoseEstimator.swift

+18-11
Original file line numberDiff line numberDiff line change
@@ -230,10 +230,10 @@ private extension PoseEstimationOutput {
230230

231231
switch postprocessOptions.humanType {
232232
case .singlePerson:
233-
// <#TODO#> - use partThreshold & don't use `convertToKeypoints` and `makeLines`
234-
let keypoints = convertToKeypoints(from: outputs)
235-
let lines = makeLines(with: keypoints)
236-
humans = [Human(keypoints: keypoints, lines: lines)]
233+
let human = parseSinglePerson(outputs,
234+
partIndex: postprocessOptions.bodyPart,
235+
partThreshold: postprocessOptions.partThreshold)
236+
humans = [human]
237237
case .multiPerson(let pairThreshold, let nmsFilterSize, let maxHumanNumber):
238238
humans = parseMultiHuman(outputs,
239239
partIndex: postprocessOptions.bodyPart,
@@ -244,8 +244,9 @@ private extension PoseEstimationOutput {
244244
}
245245
}
246246

247-
func convertToKeypoints(from outputs: [TFLiteFlatArray<Float32>]) -> [Keypoint] {
248-
let output = outputs[0] // openpose_ildoonet.tflite only use the first output
247+
func parseSinglePerson(_ outputs: [TFLiteFlatArray<Float32>], partIndex: Int?, partThreshold: Float?) -> Human {
248+
// openpose_ildoonet.tflite only use the first output
249+
let output = outputs[0]
249250

250251
// get (col, row)s from heatmaps
251252
let keypointIndexInfos: [(row: Int, col: Int, val: Float32)] = (0..<OpenPosePoseEstimator.Output.ConfidenceMap.count).map { heatmapIndex in
@@ -270,19 +271,25 @@ private extension PoseEstimationOutput {
270271
return (point: CGPoint(x: x, y: y), score: score)
271272
}
272273

273-
return keypointInfos.map { keypointInfo in Keypoint(position: keypointInfo.point, score: keypointInfo.score) }
274-
}
275-
276-
func makeLines(with keypoints: [Keypoint]) -> [Human.Line] {
274+
let keypoints: [Keypoint?] = keypointInfos
275+
.map { keypointInfo -> Keypoint? in Keypoint(position: keypointInfo.point, score: keypointInfo.score) }
276+
.map { keypointInfo -> Keypoint? in
277+
guard let score = keypointInfo?.score, let partThreshold = partThreshold else { return keypointInfo }
278+
return (score > partThreshold) ? keypointInfo : nil
279+
}
280+
281+
// lines
277282
var keypointWithBodyPart: [OpenPosePoseEstimator.Output.BodyPart: Keypoint] = [:]
278283
OpenPosePoseEstimator.Output.BodyPart.allCases.enumerated().forEach { (index, bodyPart) in
279284
keypointWithBodyPart[bodyPart] = keypoints[index]
280285
}
281-
return OpenPosePoseEstimator.Output.BodyPart.lines.compactMap { line in
286+
let lines: [Human.Line] = OpenPosePoseEstimator.Output.BodyPart.lines.compactMap { line in
282287
guard let fromKeypoint = keypointWithBodyPart[line.from],
283288
let toKeypoint = keypointWithBodyPart[line.to] else { return nil }
284289
return (from: fromKeypoint, to: toKeypoint)
285290
}
291+
292+
return Human(keypoints: keypoints, lines: lines)
286293
}
287294

288295
func parseMultiHuman(_ outputs: [TFLiteFlatArray<Float32>], partIndex: Int?, partThreshold: Float?, pairThreshold: Float?, nmsFilterSize: Int, maxHumanNumber: Int?) -> [Human] {

Diff for: ‎PoseEstimation-TFLiteSwift/StillImageHeatmapViewController.swift

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ class StillImageHeatmapViewController: UIViewController {
5353
@IBOutlet weak var pafSegment: UISegmentedControl?
5454

5555
// MARK: - ML Property
56-
let poseEstimator: PoseEstimator = OpenPosePoseEstimator()
56+
let poseEstimator: PoseEstimator = IMGCLSPoseEstimator()
5757
var modelOutput: TFLiteFlatArray<Float32>? {
5858
didSet {
5959
updateHeatmapOverlayView()

Diff for: ‎PoseEstimation-TFLiteSwift/StillImageLineViewController.swift

+1-1
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ class StillImageLineViewController: UIViewController {
9797
}
9898

9999
// MARK: - ML Property
100-
let poseEstimator: PoseEstimator = OpenPosePoseEstimator()
100+
let poseEstimator: PoseEstimator = IMGCLSPoseEstimator()
101101
var outputHumans: [PoseEstimationOutput.Human] = [] {
102102
didSet {
103103
updateOverlayView()

0 commit comments

Comments
 (0)